joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve stats generation

Browse source
This commit is contained in:
Joe Lothan 2026-05-20 00:31:38 -04:00
parent 0c9ad5bfd6
commit a8177a1583
5 changed files with 61 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

1
pipeline/02_warc_parse/log.go
View file

@ -71,6 +71,7 @@ func writeStats(stats *Stats, cfg Config) {
"duration_seconds": int(duration.Seconds()),
"processed": stats.Processed.Load(),
"titles_found": stats.TitlesFound.Load(),
"no_title": stats.NoTitle.Load(),
"icons_found": stats.IconsFound.Load(),
"iframe_blocked": stats.IframeBlocked.Load(),
"fetch_errors": stats.FetchErrors.Load(),

19
pipeline/02_warc_parse/main.go
View file

@ -26,6 +26,7 @@ type Config struct {
type Stats struct {
Processed atomic.Int64
TitlesFound atomic.Int64
NoTitle atomic.Int64
IconsFound atomic.Int64
IframeBlocked atomic.Int64
ParseErrors atomic.Int64
@ -167,12 +168,17 @@ func main() {
// Update stats
stats.Processed.Add(1)
if result.Title != "" {
stats.TitlesFound.Add(1)
}
stats.IconsFound.Add(int64(len(result.Icons)))
if result.Err == nil && !result.IframeAllowed {
stats.IframeBlocked.Add(1)
if result.Err == nil {
if result.Title != "" {
stats.TitlesFound.Add(1)
} else {
stats.NoTitle.Add(1)
}
// +1 for the /favicon.ico entry added per host
stats.IconsFound.Add(int64(len(result.Icons) + 1))
if !result.IframeAllowed {
stats.IframeBlocked.Add(1)
}
}
if result.Err != nil {
if result.FetchErr {
@ -195,6 +201,7 @@ func main() {
fmt.Printf("Duration: %s\n", duration.Round(time.Second))
fmt.Printf("Processed: %d\n", stats.Processed.Load())
fmt.Printf("Titles found: %d\n", stats.TitlesFound.Load())
fmt.Printf("No title: %d\n", stats.NoTitle.Load())
fmt.Printf("Icons found: %d\n", stats.IconsFound.Load())
fmt.Printf("Iframe blocked: %d\n", stats.IframeBlocked.Load())
fmt.Printf("Fetch errors: %d\n", stats.FetchErrors.Load())