revert to using host id ordering for bundle generation
This commit is contained in:
parent
9266c1417d
commit
1343df1a41
3 changed files with 20 additions and 62 deletions
|
|
@ -128,9 +128,7 @@ func main() {
|
|||
// Stage 1: DB fetcher — continuously fetches pages into hostCh
|
||||
go func() {
|
||||
defer close(hostCh)
|
||||
var lastTime *time.Time
|
||||
var lastID int64
|
||||
iconPhase := true // first: hosts with icons (by download time), then: hosts without
|
||||
pageSize := 50000
|
||||
fetched := 0
|
||||
for {
|
||||
|
|
@ -145,26 +143,14 @@ func main() {
|
|||
}
|
||||
}
|
||||
fetchStart := time.Now()
|
||||
hosts, err := fetchHostsPage(ctx, pool, lastTime, lastID, limit)
|
||||
hosts, err := fetchHostsPage(ctx, pool, lastID, limit)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to fetch hosts: %v", err)
|
||||
}
|
||||
if len(hosts) == 0 {
|
||||
if iconPhase {
|
||||
// Switch to no-icon hosts
|
||||
iconPhase = false
|
||||
lastTime = nil
|
||||
lastID = 0
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
last := hosts[len(hosts)-1]
|
||||
if last.IconDownloadedAt != nil {
|
||||
lastTime = last.IconDownloadedAt
|
||||
} else {
|
||||
lastID = last.ID
|
||||
}
|
||||
lastID = hosts[len(hosts)-1].ID
|
||||
fmt.Printf("[fetcher] %d hosts in %dms (hostCh: %d/%d)\n",
|
||||
len(hosts), time.Since(fetchStart).Milliseconds(), len(hostCh), cap(hostCh))
|
||||
for _, h := range hosts {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue