laurent/nabarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: http retry for retryable errors (#20)

Browse Source
This commit is contained in:
l3uddz
2021-02-21 14:01:21 +00:00
committed by GitHub
parent a2848439b9
commit db9fdc97a2
21 changed files with 150 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cache/cache.go vendored
View File

@@ -25,7 +25,7 @@ func New(path string) (*Client, error) {
return nil, fmt.Errorf("open: %w", err)
}
log := logger.New("trace").With().Logger()
log := logger.New("").With().Logger()
// start cleaner
st, tail := state.WithShutdown()

1
go.mod
View File

@@ -9,6 +9,7 @@ require (
github.com/dgraph-io/badger/v3 v3.2011.1
github.com/goccy/go-yaml v1.8.8
github.com/golang/protobuf v1.4.3 // indirect
github.com/hashicorp/go-retryablehttp v0.6.8
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
github.com/lefelys/state v1.1.0
github.com/lucperkins/rek v0.1.3

6
go.sum
View File

@@ -153,6 +153,12 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=

7
media/omdb/media.go
View File

@@ -28,13 +28,8 @@ func (c *Client) GetItem(imdbId string) (*Item, error) {
return nil, fmt.Errorf("generate lookup request url: %w", err)
}
c.log.Trace().
Str("url", reqUrl).
Msg("Searching omdb")
// send request
c.rl.Take()
resp, err := rek.Get(reqUrl, rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http))
if err != nil {
return nil, fmt.Errorf("request lookup: %w", err)
}

26
media/omdb/omdb.go
View File

@@ -2,27 +2,31 @@ package omdb
import (
"github.com/l3uddz/nabarr/logger"
"github.com/l3uddz/nabarr/util"
"github.com/rs/zerolog"
"go.uber.org/ratelimit"
"net/http"
"time"
)
type Client struct {
apiKey string
log zerolog.Logger
rl ratelimit.Limiter
log zerolog.Logger
http *http.Client
apiURL string
apiTimeout time.Duration
apiKey string
apiURL string
}
func New(cfg *Config) *Client {
return &Client{
apiKey: cfg.ApiKey,
log: logger.New(cfg.Verbosity).With().Logger(),
rl: ratelimit.New(1, ratelimit.WithoutSlack),
l := logger.New(cfg.Verbosity).With().
Str("media", "omdb").
Logger()
apiURL: "https://www.omdbapi.com",
apiTimeout: 30 * time.Second,
return &Client{
log: l,
http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
apiKey: cfg.ApiKey,
apiURL: "https://www.omdbapi.com",
}
}

14
media/trakt/media.go
View File

@@ -23,13 +23,8 @@ func (c *Client) GetShow(tvdbId string) (*Show, error) {
return nil, fmt.Errorf("generate lookup show request url: %w", err)
}
c.log.Trace().
Str("url", reqUrl).
Msg("Searching trakt")
// send request
c.rl.Take()
resp, err := rek.Get(reqUrl, rek.Headers(c.getAuthHeaders()), rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.getAuthHeaders()))
if err != nil {
return nil, fmt.Errorf("request show: %w", err)
}
@@ -67,13 +62,8 @@ func (c *Client) GetMovie(imdbId string) (*Movie, error) {
return nil, fmt.Errorf("generate lookup movie request url: %w", err)
}
c.log.Trace().
Str("url", reqUrl).
Msg("Searching trakt")
// send request
c.rl.Take()
resp, err := rek.Get(reqUrl, rek.Headers(c.getAuthHeaders()), rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.getAuthHeaders()))
if err != nil {
return nil, fmt.Errorf("request movie: %w", err)
}

28
media/trakt/trakt.go
View File

@@ -2,34 +2,38 @@ package trakt
import (
"github.com/l3uddz/nabarr/logger"
"github.com/l3uddz/nabarr/util"
"github.com/rs/zerolog"
"go.uber.org/ratelimit"
"net/http"
"time"
)
type Client struct {
clientId string
log zerolog.Logger
rl ratelimit.Limiter
log zerolog.Logger
http *http.Client
apiURL string
apiTimeout time.Duration
apiKey string
apiURL string
}
func New(cfg *Config) *Client {
return &Client{
clientId: cfg.ClientId,
log: logger.New(cfg.Verbosity).With().Logger(),
rl: ratelimit.New(1, ratelimit.WithoutSlack),
l := logger.New(cfg.Verbosity).With().
Str("media", "trakt").
Logger()
apiURL: "https://api.trakt.tv",
apiTimeout: 30 * time.Second,
return &Client{
log: l,
http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
apiKey: cfg.ClientId,
apiURL: "https://api.trakt.tv",
}
}
func (c *Client) getAuthHeaders() map[string]string {
return map[string]string{
"trakt-api-key": c.clientId,
"trakt-api-key": c.apiKey,
"trakt-api-version": "2",
}
}

6
media/tvdb/media.go
View File

@@ -20,13 +20,9 @@ func (c *Client) GetItem(tvdbId string) (*Item, error) {
// prepare request
reqUrl := util.JoinURL(c.apiURL, "series", tvdbId)
c.log.Trace().
Str("url", reqUrl).
Msg("Searching tvdb")
// send request
c.rl.Take()
resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return nil, fmt.Errorf("request lookup: %w", err)
}

22
media/tvdb/tvdb.go
View File

@@ -3,31 +3,35 @@ package tvdb
import (
"fmt"
"github.com/l3uddz/nabarr/logger"
"github.com/l3uddz/nabarr/util"
"github.com/rs/zerolog"
"go.uber.org/ratelimit"
"net/http"
"time"
)
type Client struct {
apiKey string
log zerolog.Logger
rl ratelimit.Limiter
log zerolog.Logger
http *http.Client
apiKey string
apiURL string
apiHeaders map[string]string
apiTimeout time.Duration
}
func New(cfg *Config) *Client {
return &Client{
apiKey: cfg.ApiKey,
log: logger.New(cfg.Verbosity).With().Logger(),
rl: ratelimit.New(1, ratelimit.WithoutSlack),
l := logger.New(cfg.Verbosity).With().
Str("media", "tvdb").
Logger()
return &Client{
log: l,
http: util.NewRetryableHttpClient(30*time.Second, ratelimit.New(1, ratelimit.WithoutSlack), &l),
apiKey: cfg.ApiKey,
apiURL: "https://api.thetvdb.com",
apiHeaders: map[string]string{
"Authorization": fmt.Sprintf("Bearer %s", cfg.ApiKey),
},
apiTimeout: 30 * time.Second,
}
}

13
nabarr.go
View File

@@ -33,16 +33,3 @@ func NewExprEnv(media *media.Item) *ExprEnv {
Now: func() time.Time { return time.Now().UTC() },
}
}
func StringOrDefault(currentValue *string, defaultValue string) string {
if currentValue == nil {
return defaultValue
}
return *currentValue
}
func Uint64OrDefault(currentValue *uint64, defaultValue uint64) uint64 {
if currentValue == nil {
return defaultValue
}
return *currentValue
}

6
pvr.go
View File

@@ -34,8 +34,8 @@ type PvrFilters struct {
type PvrOption func(options *PvrOptions)
type PvrOptions struct {
// the seriesType returned from the lookup before adding (sonarr)
LookupType string
// seriesType returned from the lookup before adding (sonarr)
SeriesType string
AddMonitored bool
SearchMissing bool
@@ -53,7 +53,7 @@ func BuildPvrOptions(opts ...PvrOption) (*PvrOptions, error) {
func WithSeriesType(seriesType string) PvrOption {
return func(opts *PvrOptions) {
opts.LookupType = seriesType
opts.SeriesType = seriesType
}
}

12
radarr/api.go
View File

@@ -19,8 +19,7 @@ var (
func (c *Client) getSystemStatus() (*systemStatus, error) {
// send request
resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Headers(c.apiHeaders),
rek.Timeout(c.apiTimeout))
resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return nil, fmt.Errorf("request system status: %w", err)
}
@@ -42,8 +41,7 @@ func (c *Client) getSystemStatus() (*systemStatus, error) {
func (c *Client) getQualityProfileId(profileName string) (int, error) {
// send request
resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Headers(c.apiHeaders),
rek.Timeout(c.apiTimeout))
resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return 0, fmt.Errorf("request quality profiles: %w", err)
}
@@ -89,7 +87,7 @@ func (c *Client) lookupMediaItem(item *media.Item) (*lookupRequest, error) {
}
// send request
resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return nil, fmt.Errorf("request movie lookup: %w", err)
}
@@ -150,8 +148,8 @@ func (c *Client) AddMediaItem(item *media.Item, opts ...nabarr.PvrOption) error
}
// send request
resp, err := rek.Post(util.JoinURL(c.apiURL, "movie"), rek.Headers(c.apiHeaders), rek.Json(req),
rek.Timeout(c.apiTimeout))
resp, err := rek.Post(util.JoinURL(c.apiURL, "movie"), rek.Client(c.http), rek.Headers(c.apiHeaders),
rek.Json(req))
if err != nil {
return fmt.Errorf("request add movie: %w", err)
}

9
radarr/radarr.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/l3uddz/nabarr/media"
"github.com/l3uddz/nabarr/util"
"github.com/rs/zerolog"
"net/http"
"strings"
"time"
)
@@ -26,7 +27,6 @@ type Client struct {
apiURL string
apiHeaders map[string]string
apiTimeout time.Duration
cache *cache.Client
cacheTempDuration time.Duration
@@ -35,6 +35,7 @@ type Client struct {
queue chan *media.FeedItem
m *media.Client
http *http.Client
log zerolog.Logger
ignoresExpr []*nabarr.ExprProgram
}
@@ -81,10 +82,10 @@ func New(c nabarr.PvrConfig, mode string, m *media.Client, cc *cache.Client) (*C
apiURL: apiURL,
apiHeaders: apiHeaders,
apiTimeout: 60 * time.Second,
m: m,
log: l,
m: m,
http: util.NewRetryableHttpClient(60*time.Second, nil, &l),
log: l,
}
// compile expressions

16
rss/job.go
View File

@@ -3,6 +3,7 @@ package rss
import (
"fmt"
"github.com/l3uddz/nabarr/cmd/nabarr/pvr"
"github.com/l3uddz/nabarr/util"
"github.com/robfig/cron/v3"
"time"
)
@@ -17,13 +18,18 @@ func (c *Client) AddJob(feed feedItem) error {
feed.CacheDuration = (24 * time.Hour) * 28
}
l := c.log.With().
Str("feed_name", feed.Name).
Logger()
// create job
job := &rssJob{
name: feed.Name,
log: c.log.With().Str("feed_name", feed.Name).Logger(),
url: feed.URL,
timeout: 30 * time.Second,
pvrs: make(map[string]pvr.PVR, 0),
name: feed.Name,
log: l,
http: util.NewRetryableHttpClient(30*time.Second, nil, &l),
url: feed.URL,
pvrs: make(map[string]pvr.PVR, 0),
attempts: 0,
errors: make([]error, 0),

2
rss/process.go
View File

@@ -48,7 +48,7 @@ func (j *rssJob) queueItemWithPvrs(item *media.FeedItem) {
func (j *rssJob) getFeed() ([]media.FeedItem, error) {
// request feed
res, err := rek.Get(j.url, rek.Timeout(j.timeout))
res, err := rek.Get(j.url, rek.Client(j.http))
if err != nil {
return nil, fmt.Errorf("request feed: %w", err)
}

12
rss/struct.go
View File

@@ -5,6 +5,7 @@ import (
"github.com/l3uddz/nabarr/cmd/nabarr/pvr"
"github.com/robfig/cron/v3"
"github.com/rs/zerolog"
"net/http"
"time"
)
@@ -23,11 +24,12 @@ type Config struct {
}
type rssJob struct {
name string
log zerolog.Logger
url string
timeout time.Duration
pvrs map[string]pvr.PVR
name string
log zerolog.Logger
http *http.Client
url string
pvrs map[string]pvr.PVR
attempts int
errors []error

14
sonarr/api.go
View File

@@ -19,8 +19,7 @@ var (
func (c *Client) getSystemStatus() (*systemStatus, error) {
// send request
resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Headers(c.apiHeaders),
rek.Timeout(c.apiTimeout))
resp, err := rek.Get(util.JoinURL(c.apiURL, "system", "status"), rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return nil, fmt.Errorf("request system status: %w", err)
}
@@ -42,8 +41,7 @@ func (c *Client) getSystemStatus() (*systemStatus, error) {
func (c *Client) getQualityProfileId(profileName string) (int, error) {
// send request
resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Headers(c.apiHeaders),
rek.Timeout(c.apiTimeout))
resp, err := rek.Get(util.JoinURL(c.apiURL, "profile"), rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return 0, fmt.Errorf("request quality profiles: %w", err)
}
@@ -79,7 +77,7 @@ func (c *Client) lookupMediaItem(item *media.Item) (*lookupRequest, error) {
}
// send request
resp, err := rek.Get(reqUrl, rek.Headers(c.apiHeaders), rek.Timeout(c.apiTimeout))
resp, err := rek.Get(reqUrl, rek.Client(c.http), rek.Headers(c.apiHeaders))
if err != nil {
return nil, fmt.Errorf("request series lookup: %w", err)
}
@@ -134,14 +132,14 @@ func (c *Client) AddMediaItem(item *media.Item, opts ...nabarr.PvrOption) error
IgnoreEpisodesWithoutFiles: false,
},
Seasons: []string{},
SeriesType: util.StringOrDefault(o.LookupType, "standard"),
SeriesType: util.StringOrDefault(o.SeriesType, "standard"),
SeasonFolder: true,
TvdbId: tvdbId,
}
// send request
resp, err := rek.Post(util.JoinURL(c.apiURL, "series"), rek.Headers(c.apiHeaders), rek.Json(req),
rek.Timeout(c.apiTimeout))
resp, err := rek.Post(util.JoinURL(c.apiURL, "series"), rek.Client(c.http), rek.Headers(c.apiHeaders),
rek.Json(req))
if err != nil {
return fmt.Errorf("request add series: %w", err)
}

6
sonarr/queue.go
View File

@@ -163,11 +163,11 @@ func (c *Client) queueProcessor(tail state.ShutdownTail) {
// set appropriate series type
switch {
case util.StringSliceContains(mediaItem.Genres, "anime"), util.StringSliceContains(mediaItem.Tvdb.Genre, "anime"):
s.Type = "anime"
s.SeriesType = "anime"
}
// check if item should be skipped (skip options)
if c.skipAnime && strings.EqualFold(s.Type, "anime") {
if c.skipAnime && strings.EqualFold(s.SeriesType, "anime") {
c.log.Debug().
Str("trakt_title", mediaItem.Title).
Str("trakt_tvdb_id", mediaItem.TvdbId).
@@ -202,7 +202,7 @@ func (c *Client) queueProcessor(tail state.ShutdownTail) {
}
opts := []nabarr.PvrOption{
nabarr.WithSeriesType(s.Type),
nabarr.WithSeriesType(s.SeriesType),
nabarr.WithAddMonitored(c.addMonitored),
nabarr.WithSearchMissing(c.searchMissing),
}

9
sonarr/sonarr.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/l3uddz/nabarr/media"
"github.com/l3uddz/nabarr/util"
"github.com/rs/zerolog"
"net/http"
"strings"
"time"
)
@@ -27,7 +28,6 @@ type Client struct {
apiURL string
apiHeaders map[string]string
apiTimeout time.Duration
cache *cache.Client
cacheTempDuration time.Duration
@@ -36,6 +36,7 @@ type Client struct {
queue chan *media.FeedItem
m *media.Client
http *http.Client
log zerolog.Logger
ignoresExpr []*nabarr.ExprProgram
}
@@ -84,10 +85,10 @@ func New(c nabarr.PvrConfig, mode string, m *media.Client, cc *cache.Client) (*C
apiURL: apiURL,
apiHeaders: apiHeaders,
apiTimeout: 60 * time.Second,
m: m,
log: l,
m: m,
http: util.NewRetryableHttpClient(60*time.Second, nil, &l),
log: l,
}
// compile expressions

12
sonarr/struct.go
View File

@@ -10,12 +10,12 @@ type qualityProfile struct {
}
type lookupRequest struct {
Id int `json:"id,omitempty"`
Title string `json:"title"`
TitleSlug string `json:"titleSlug"`
Year int `json:"year,omitempty"`
TvdbId int `json:"tvdbId"`
Type string `json:"seriesType"`
Id int `json:"id,omitempty"`
Title string `json:"title"`
TitleSlug string `json:"titleSlug"`
Year int `json:"year,omitempty"`
TvdbId int `json:"tvdbId"`
SeriesType string `json:"seriesType"`
}
type addRequest struct {

42
util/http.go Normal file
View File

@@ -0,0 +1,42 @@
package util
import (
"github.com/hashicorp/go-retryablehttp"
"github.com/rs/zerolog"
"go.uber.org/ratelimit"
"net/http"
"time"
)
func NewRetryableHttpClient(timeout time.Duration, rl ratelimit.Limiter, log *zerolog.Logger) *http.Client {
retryClient := retryablehttp.NewClient()
retryClient.RetryMax = 10
retryClient.RetryWaitMin = 1 * time.Second
retryClient.RetryWaitMax = 10 * time.Second
retryClient.RequestLogHook = func(l retryablehttp.Logger, request *http.Request, i int) {
// rate limit
if rl != nil {
rl.Take()
}
// log
if log != nil && request != nil && request.URL != nil {
switch i {
case 0:
// first
log.Trace().
Str("url", request.URL.String()).
Msg("Sending request")
default:
// retry
log.Debug().
Str("url", request.URL.String()).
Int("attempt", i).
Msg("Retrying failed request")
}
}
}
retryClient.HTTPClient.Timeout = timeout
retryClient.Logger = nil
return retryClient.StandardClient()
}