refactor: http retry for retryable errors (#20)

cache/cache.go

@@ -25,7 +25,7 @@ func New(path string) (*Client, error) {
 		return nil, fmt.Errorf("open: %w", err)
 	}
 
-	log := logger.New("trace").With().Logger()
+	log := logger.New("").With().Logger()
 
 	// start cleaner
 	st, tail := state.WithShutdown()

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/dgraph-io/badger/v3 v3.2011.1
 	github.com/goccy/go-yaml v1.8.8
 	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
 	github.com/lefelys/state v1.1.0
 	github.com/lucperkins/rek v0.1.3

go.sum

@@ -153,6 +153,12 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=

media/omdb/media.go

@@ -28,13 +28,8 @@ func (c *Client) GetItem(imdbId string) (*Item, error) {
 		return nil, fmt.Errorf("generate lookup request url: %w", err)
 	}
 
-	c.log.Trace().
-		Str("url", reqUrl).
-		Msg("Searching omdb")
-
 	// send request
-	c.rl.Take()
+	resp, err := rek.Get(reqUrl, rek.Client(c.http))
-	resp, err := rek.Get(reqUrl, rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request lookup: %w", err)
 	}

media/omdb/omdb.go

@@ -2,27 +2,31 @@ package omdb
 
 import (
 	"github.com/l3uddz/nabarr/logger"
+	"github.com/l3uddz/nabarr/util"
 	"github.com/rs/zerolog"
 	"go.uber.org/ratelimit"
+	"net/http"
 	"time"
 )
 
 type Client struct {
-	apiKey string
 	log  zerolog.Logger
-	rl     ratelimit.Limiter
+	http *http.Client
 
+	apiKey string
 	apiURL string
-	apiTimeout time.Duration
 }
 
 func New(cfg *Config) *Client {
-	return &Client{
+	l := logger.New(cfg.Verbosity).With().
-		apiKey: cfg.ApiKey,
+		Str("media", "omdb").
-		log:    logger.New(cfg.Verbosity).With().Logger(),
+		Logger()
-		rl:     ratelimit.New(1, ratelimit.WithoutSlack),
 
+	return &Client{
+		log:  l,
+		http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
+
+		apiKey: cfg.ApiKey,
 		apiURL: "https://www.omdbapi.com",
-		apiTimeout: 30 * time.Second,
 	}
 }

media/trakt/media.go

@@ -23,13 +23,8 @@ func (c *Client) GetShow(tvdbId string) (*Show, error) {
 		return nil, fmt.Errorf("generate lookup show request url: %w", err)
 	}
 
-	c.log.Trace().
-		Str("url", reqUrl).
-		Msg("Searching trakt")
-
 	// send request
-	c.rl.Take()
+	resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.getAuthHeaders()))
-	resp, err := rek.Get(reqUrl, rek.Headers(c.getAuthHeaders()), rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request show: %w", err)
 	}
@@ -67,13 +62,8 @@ func (c *Client) GetMovie(imdbId string) (*Movie, error) {
 		return nil, fmt.Errorf("generate lookup movie request url: %w", err)
 	}
 
-	c.log.Trace().
-		Str("url", reqUrl).
-		Msg("Searching trakt")
-
 	// send request
-	c.rl.Take()
+	resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.getAuthHeaders()))
-	resp, err := rek.Get(reqUrl, rek.Headers(c.getAuthHeaders()), rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request movie: %w", err)
 	}

media/trakt/trakt.go

@@ -2,34 +2,38 @@ package trakt
 
 import (
 	"github.com/l3uddz/nabarr/logger"
+	"github.com/l3uddz/nabarr/util"
 	"github.com/rs/zerolog"
 	"go.uber.org/ratelimit"
+	"net/http"
 	"time"
 )
 
 type Client struct {
-	clientId string
 	log  zerolog.Logger
-	rl       ratelimit.Limiter
+	http *http.Client
 
+	apiKey string
 	apiURL string
-	apiTimeout time.Duration
 }
 
 func New(cfg *Config) *Client {
-	return &Client{
+	l := logger.New(cfg.Verbosity).With().
-		clientId: cfg.ClientId,
+		Str("media", "trakt").
-		log:      logger.New(cfg.Verbosity).With().Logger(),
+		Logger()
-		rl:       ratelimit.New(1, ratelimit.WithoutSlack),
 
+	return &Client{
+		log:  l,
+		http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
+
+		apiKey: cfg.ClientId,
 		apiURL: "https://api.trakt.tv",
-		apiTimeout: 30 * time.Second,
 	}
 }
 
 func (c *Client) getAuthHeaders() map[string]string {
 	return map[string]string{
-		"trakt-api-key":     c.clientId,
+		"trakt-api-key":     c.apiKey,
 		"trakt-api-version": "2",
 	}
 }

media/tvdb/media.go

@@ -20,13 +20,9 @@ func (c *Client) GetItem(tvdbId string) (*Item, error) {
 
 	// prepare request
 	reqUrl := util.JoinURL(c.apiURL, "series", tvdbId)
-	c.log.Trace().
-		Str("url", reqUrl).
-		Msg("Searching tvdb")
 
 	// send request
-	c.rl.Take()
+	resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
-	resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request lookup: %w", err)
 	}

media/tvdb/tvdb.go

@@ -3,31 +3,35 @@ package tvdb
 import (
 	"fmt"
 	"github.com/l3uddz/nabarr/logger"
+	"github.com/l3uddz/nabarr/util"
 	"github.com/rs/zerolog"
 	"go.uber.org/ratelimit"
+	"net/http"
 	"time"
 )
 
 type Client struct {
-	apiKey string
 	log  zerolog.Logger
-	rl     ratelimit.Limiter
+	http *http.Client
 
+	apiKey     string
 	apiURL     string
 	apiHeaders map[string]string
-	apiTimeout time.Duration
 }
 
 func New(cfg *Config) *Client {
-	return &Client{
+	l := logger.New(cfg.Verbosity).With().
-		apiKey: cfg.ApiKey,
+		Str("media", "tvdb").
-		log:    logger.New(cfg.Verbosity).With().Logger(),
+		Logger()
-		rl:     ratelimit.New(1, ratelimit.WithoutSlack),
 
+	return &Client{
+		log:  l,
+		http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
+
+		apiKey: cfg.ApiKey,
 		apiURL: "https://api.thetvdb.com",
 		apiHeaders: map[string]string{
 			"Authorization": fmt.Sprintf("Bearer %s", cfg.ApiKey),
 		},
-		apiTimeout: 30 * time.Second,
 	}
 }

nabarr.go

@@ -33,16 +33,3 @@ func NewExprEnv(media *media.Item) *ExprEnv {
 		Now:  func() time.Time { return time.Now().UTC() },
 	}
 }
-
-func StringOrDefault(currentValue *string, defaultValue string) string {
-	if currentValue == nil {
-		return defaultValue
-	}
-	return *currentValue
-}
-func Uint64OrDefault(currentValue *uint64, defaultValue uint64) uint64 {
-	if currentValue == nil {
-		return defaultValue
-	}
-	return *currentValue
-}

pvr.go

@@ -34,8 +34,8 @@ type PvrFilters struct {
 type PvrOption func(options *PvrOptions)
 
 type PvrOptions struct {
-	// the seriesType returned from the lookup before adding (sonarr)
+	// seriesType returned from the lookup before adding (sonarr)
-	LookupType string
+	SeriesType string
 
 	AddMonitored  bool
 	SearchMissing bool
@@ -53,7 +53,7 @@ func BuildPvrOptions(opts ...PvrOption) (*PvrOptions, error) {
 
 func WithSeriesType(seriesType string) PvrOption {
 	return func(opts *PvrOptions) {
-		opts.LookupType = seriesType
+		opts.SeriesType = seriesType
 	}
 }
 

radarr/api.go

@@ -19,8 +19,7 @@ var (
 
 func (c *Client) getSystemStatus() (*systemStatus, error) {
 	// send request
-	resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Headers(c.apiHeaders),
+	resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Client(c.http), rek.Headers(c.apiHeaders))
-		rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request system status: %w", err)
 	}
@@ -42,8 +41,7 @@ func (c *Client) getSystemStatus() (*systemStatus, error) {
 
 func (c *Client) getQualityProfileId(profileName string) (int, error) {
 	// send request
-	resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Headers(c.apiHeaders),
+	resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Client(c.http), rek.Headers(c.apiHeaders))
-		rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return 0, fmt.Errorf("request quality profiles: %w", err)
 	}
@@ -89,7 +87,7 @@ func (c *Client) lookupMediaItem(item *media.Item) (*lookupRequest, error) {
 	}
 
 	// send request
-	resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
+	resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
 	if err != nil {
 		return nil, fmt.Errorf("request movie lookup: %w", err)
 	}
@@ -150,8 +148,8 @@ func (c *Client) AddMediaItem(item *media.Item, opts ...nabarr.PvrOption) error
 	}
 
 	// send request
-	resp, err := rek.Post(util.JoinURL(c.apiURL, "movie"), rek.Headers(c.apiHeaders), rek.Json(req),
+	resp, err := rek.Post(util.JoinURL(c.apiURL, "movie"), rek.Client(c.http), rek.Headers(c.apiHeaders),
-		rek.Timeout(c.apiTimeout))
+		rek.Json(req))
 	if err != nil {
 		return fmt.Errorf("request add movie: %w", err)
 	}

radarr/radarr.go

@@ -8,6 +8,7 @@ import (
 	"github.com/l3uddz/nabarr/media"
 	"github.com/l3uddz/nabarr/util"
 	"github.com/rs/zerolog"
+	"net/http"
 	"strings"
 	"time"
 )
@@ -26,7 +27,6 @@ type Client struct {
 
 	apiURL     string
 	apiHeaders map[string]string
-	apiTimeout time.Duration
 
 	cache             *cache.Client
 	cacheTempDuration time.Duration
@@ -35,6 +35,7 @@ type Client struct {
 	queue chan *media.FeedItem
 
 	m           *media.Client
+	http        *http.Client
 	log         zerolog.Logger
 	ignoresExpr []*nabarr.ExprProgram
 }
@@ -81,9 +82,9 @@ func New(c nabarr.PvrConfig, mode string, m *media.Client, cc *cache.Client) (*C
 
 		apiURL:     apiURL,
 		apiHeaders: apiHeaders,
-		apiTimeout: 60 * time.Second,
 
 		m:    m,
+		http: util.NewRetryableHttpClient(60*time.Second, nil, &l),
 		log:  l,
 	}
 

rss/job.go

@@ -3,6 +3,7 @@ package rss
 import (
 	"fmt"
 	"github.com/l3uddz/nabarr/cmd/nabarr/pvr"
+	"github.com/l3uddz/nabarr/util"
 	"github.com/robfig/cron/v3"
 	"time"
 )
@@ -17,12 +18,17 @@ func (c *Client) AddJob(feed feedItem) error {
 		feed.CacheDuration = (24 * time.Hour) * 28
 	}
 
+	l := c.log.With().
+		Str("feed_name", feed.Name).
+		Logger()
+
 	// create job
 	job := &rssJob{
 		name: feed.Name,
-		log:     c.log.With().Str("feed_name", feed.Name).Logger(),
+		log:  l,
+		http: util.NewRetryableHttpClient(30*time.Second, nil, &l),
+
 		url:  feed.URL,
-		timeout: 30 * time.Second,
 		pvrs: make(map[string]pvr.PVR, 0),
 
 		attempts: 0,

rss/process.go

@@ -48,7 +48,7 @@ func (j *rssJob) queueItemWithPvrs(item *media.FeedItem) {
 
 func (j *rssJob) getFeed() ([]media.FeedItem, error) {
 	// request feed
-	res, err := rek.Get(j.url, rek.Timeout(j.timeout))
+	res, err := rek.Get(j.url, rek.Client(j.http))
 	if err != nil {
 		return nil, fmt.Errorf("request feed: %w", err)
 	}

rss/struct.go

@@ -5,6 +5,7 @@ import (
 	"github.com/l3uddz/nabarr/cmd/nabarr/pvr"
 	"github.com/robfig/cron/v3"
 	"github.com/rs/zerolog"
+	"net/http"
 	"time"
 )
 
@@ -25,8 +26,9 @@ type Config struct {
 type rssJob struct {
 	name string
 	log  zerolog.Logger
+	http *http.Client
+
 	url  string
-	timeout time.Duration
 	pvrs map[string]pvr.PVR
 
 	attempts int

sonarr/api.go

@@ -19,8 +19,7 @@ var (
 
 func (c *Client) getSystemStatus() (*systemStatus, error) {
 	// send request
-	resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Headers(c.apiHeaders),
+	resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Client(c.http), rek.Headers(c.apiHeaders))
-		rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return nil, fmt.Errorf("request system status: %w", err)
 	}
@@ -42,8 +41,7 @@ func (c *Client) getSystemStatus() (*systemStatus, error) {
 
 func (c *Client) getQualityProfileId(profileName string) (int, error) {
 	// send request
-	resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Headers(c.apiHeaders),
+	resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Client(c.http), rek.Headers(c.apiHeaders))
-		rek.Timeout(c.apiTimeout))
 	if err != nil {
 		return 0, fmt.Errorf("request quality profiles: %w", err)
 	}
@@ -79,7 +77,7 @@ func (c *Client) lookupMediaItem(item *media.Item) (*lookupRequest, error) {
 	}
 
 	// send request
-	resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
+	resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
 	if err != nil {
 		return nil, fmt.Errorf("request series lookup: %w", err)
 	}
@@ -134,14 +132,14 @@ func (c *Client) AddMediaItem(item *media.Item, opts ...nabarr.PvrOption) error
 			IgnoreEpisodesWithoutFiles: false,
 		},
 		Seasons:      []string{},
-		SeriesType:   util.StringOrDefault(o.LookupType, "standard"),
+		SeriesType:   util.StringOrDefault(o.SeriesType, "standard"),
 		SeasonFolder: true,
 		TvdbId:       tvdbId,
 	}
 
 	// send request
-	resp, err := rek.Post(util.JoinURL(c.apiURL, "series"), rek.Headers(c.apiHeaders), rek.Json(req),
+	resp, err := rek.Post(util.JoinURL(c.apiURL, "series"), rek.Client(c.http), rek.Headers(c.apiHeaders),
-		rek.Timeout(c.apiTimeout))
+		rek.Json(req))
 	if err != nil {
 		return fmt.Errorf("request add series: %w", err)
 	}

sonarr/queue.go

@@ -163,11 +163,11 @@ func (c *Client) queueProcessor(tail state.ShutdownTail) {
 			// set appropriate series type
 			switch {
 			case util.StringSliceContains(mediaItem.Genres, "anime"), util.StringSliceContains(mediaItem.Tvdb.Genre, "anime"):
-				s.Type = "anime"
+				s.SeriesType = "anime"
 			}
 
 			// check if item should be skipped (skip options)
-			if c.skipAnime && strings.EqualFold(s.Type, "anime") {
+			if c.skipAnime && strings.EqualFold(s.SeriesType, "anime") {
 				c.log.Debug().
 					Str("trakt_title", mediaItem.Title).
 					Str("trakt_tvdb_id", mediaItem.TvdbId).
@@ -202,7 +202,7 @@ func (c *Client) queueProcessor(tail state.ShutdownTail) {
 			}
 
 			opts := []nabarr.PvrOption{
-				nabarr.WithSeriesType(s.Type),
+				nabarr.WithSeriesType(s.SeriesType),
 				nabarr.WithAddMonitored(c.addMonitored),
 				nabarr.WithSearchMissing(c.searchMissing),
 			}

sonarr/sonarr.go

@@ -8,6 +8,7 @@ import (
 	"github.com/l3uddz/nabarr/media"
 	"github.com/l3uddz/nabarr/util"
 	"github.com/rs/zerolog"
+	"net/http"
 	"strings"
 	"time"
 )
@@ -27,7 +28,6 @@ type Client struct {
 
 	apiURL     string
 	apiHeaders map[string]string
-	apiTimeout time.Duration
 
 	cache             *cache.Client
 	cacheTempDuration time.Duration
@@ -36,6 +36,7 @@ type Client struct {
 	queue chan *media.FeedItem
 
 	m           *media.Client
+	http        *http.Client
 	log         zerolog.Logger
 	ignoresExpr []*nabarr.ExprProgram
 }
@@ -84,9 +85,9 @@ func New(c nabarr.PvrConfig, mode string, m *media.Client, cc *cache.Client) (*C
 
 		apiURL:     apiURL,
 		apiHeaders: apiHeaders,
-		apiTimeout: 60 * time.Second,
 
 		m:    m,
+		http: util.NewRetryableHttpClient(60*time.Second, nil, &l),
 		log:  l,
 	}
 

sonarr/struct.go

@@ -15,7 +15,7 @@ type lookupRequest struct {
 	TitleSlug  string `json:"titleSlug"`
 	Year       int    `json:"year,omitempty"`
 	TvdbId     int    `json:"tvdbId"`
-	Type      string `json:"seriesType"`
+	SeriesType string `json:"seriesType"`
 }
 
 type addRequest struct {

util/http.go

@@ -0,0 +1,42 @@
+package util
+
+import (
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/rs/zerolog"
+	"go.uber.org/ratelimit"
+	"net/http"
+	"time"
+)
+
+func NewRetryableHttpClient(timeout time.Duration, rl ratelimit.Limiter, log *zerolog.Logger) *http.Client {
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = 10
+	retryClient.RetryWaitMin = 1 * time.Second
+	retryClient.RetryWaitMax = 10 * time.Second
+	retryClient.RequestLogHook = func(l retryablehttp.Logger, request *http.Request, i int) {
+		// rate limit
+		if rl != nil {
+			rl.Take()
+		}
+
+		// log
+		if log != nil && request != nil && request.URL != nil {
+			switch i {
+			case 0:
+				// first
+				log.Trace().
+					Str("url", request.URL.String()).
+					Msg("Sending request")
+			default:
+				// retry
+				log.Debug().
+					Str("url", request.URL.String()).
+					Int("attempt", i).
+					Msg("Retrying failed request")
+			}
+		}
+	}
+	retryClient.HTTPClient.Timeout = timeout
+	retryClient.Logger = nil
+	return retryClient.StandardClient()
+}