added icon downloader

pipeline/03_icon_download/log.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+)
+
+type LogWriter struct {
+	file       *os.File
+	mu         sync.Mutex
+	errorsOnly bool
+}
+
+func NewLogWriter(path string, errorsOnly bool) (*LogWriter, error) {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		return nil, err
+	}
+	return &LogWriter{file: f, errorsOnly: errorsOnly}, nil
+}
+
+func (lw *LogWriter) Write(line string, isError bool) {
+	if lw.errorsOnly && !isError {
+		return
+	}
+	lw.mu.Lock()
+	defer lw.mu.Unlock()
+	fmt.Fprintln(lw.file, line)
+}
+
+func (lw *LogWriter) Close() error {
+	return lw.file.Close()
+}
+
+func formatLogLine(icon IconRow, result DownloadResult) string {
+	if result.Err != "" {
+		return fmt.Sprintf("icon: %s err:%s %s", icon.URL, result.ErrType, result.Err)
+	}
+
+	dedup := ""
+	if result.Dedup {
+		dedup = " dedup"
+	}
+
+	dims := ""
+	if result.Width > 0 && result.Height > 0 {
+		dims = fmt.Sprintf(" %dx%d", result.Width, result.Height)
+	}
+
+	return fmt.Sprintf("icon: %s %s%s %.1fKB%s ok",
+		icon.URL, result.ContentType, dims,
+		float64(result.FileSize)/1024, dedup)
+}
+
+func writeStats(stats *Stats) {
+	finishedAt := time.Now()
+	duration := finishedAt.Sub(stats.StartedAt)
+
+	data := map[string]interface{}{
+		"started_at":       stats.StartedAt.Format(time.RFC3339),
+		"finished_at":      finishedAt.Format(time.RFC3339),
+		"duration_seconds": int(duration.Seconds()),
+		"processed":        stats.Processed.Load(),
+		"completed":        stats.Completed.Load(),
+		"failed":           stats.Failed.Load(),
+		"failed_dns":       stats.DNSErrors.Load(),
+		"failed_timeout":   stats.Timeouts.Load(),
+		"failed_http":      stats.HTTPErrors.Load(),
+		"failed_invalid":   stats.InvalidImg.Load(),
+		"failed_too_large": stats.TooLarge.Load(),
+		"dedup_hits":       stats.DedupHits.Load(),
+		"db_errors":        stats.DBErrors.Load(),
+		"panics":           stats.Panics.Load(),
+		"bytes_downloaded":  stats.BytesDown.Load(),
+	}
+
+	os.MkdirAll("stats", 0755)
+	f, err := os.Create("stats/03_icon_download.json")
+	if err != nil {
+		fmt.Printf("Failed to write stats: %v\n", err)
+		return
+	}
+	defer f.Close()
+
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "  ")
+	enc.Encode(data)
+	fmt.Println("Stats written to stats/03_icon_download.json")
+}