From 27a22ba023ffb9040a4eeddc0cfdb3b2c95147c5 Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Thu, 11 May 2023 21:15:29 +0000 Subject: [PATCH] Dynamic rate limiter --- distro/debian/snapshot/mr/mr.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/distro/debian/snapshot/mr/mr.go b/distro/debian/snapshot/mr/mr.go index 53ce74e..198d9ca 100644 --- a/distro/debian/snapshot/mr/mr.go +++ b/distro/debian/snapshot/mr/mr.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "github.com/rs/zerolog/log" @@ -14,7 +15,18 @@ import ( const apiURL = "https://snapshot.debian.org/mr" -var Limiter = rate.NewLimiter(rate.Every(time.Second), 3) +var ( + limiterTimeout time.Duration = time.Second + limiterBurst int = 3 + + Limiter = rate.NewLimiter(rate.Every(limiterTimeout), limiterBurst) +) + +func lowerLimit() { + limiterTimeout = limiterTimeout * 2 + log.Info().Msgf("limiter timeout set to %v", limiterTimeout) + Limiter.SetLimit(rate.Every(limiterTimeout)) +} // Retries in case of 5xx errors var Retries = 10 @@ -90,6 +102,11 @@ func getJson(query string, target interface{}) (err error) { flog.Trace().Msg("start") resp, err = http.Get(query) if err != nil { + if strings.Contains(err.Error(), "reset by peer") { + flog.Debug().Err(err).Msg("") + lowerLimit() + continue + } flog.Error().Err(err).Msg("") return }