joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bump up s3 warc retries to 6 to avoid 503 errors

Browse source
This commit is contained in:
Joe Lothan 2026-05-20 01:30:46 -04:00
parent 081866f62e
commit ec33b2e857
1 changed files with 7 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
pipeline/02_warc_parse/warc.go
View file

@ -10,6 +10,7 @@ import (
"strings" "strings"
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/nlnwa/gowarc/v3" "github.com/nlnwa/gowarc/v3"
@ -20,7 +21,12 @@ const ccBucket = "commoncrawl"
var s3Client *s3.Client var s3Client *s3.Client
func initS3() error { func initS3() error {
cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion("us-east-1")) cfg, err := config.LoadDefaultConfig(context.Background(),
config.WithRegion("us-east-1"),
config.WithRetryer(func() aws.Retryer {
return retry.AddWithMaxAttempts(retry.NewStandard(), 6)
}),
)
if err != nil { if err != nil {
return fmt.Errorf("load AWS config: %w", err) return fmt.Errorf("load AWS config: %w", err)
} }