Browse Source

feat: supports timeout and retry on download

feature/assets
Dnomd343 1 year ago
parent
commit
f981880488
  1. 1
      go.mod
  2. 2
      go.sum
  3. 4
      next/assets/assets.go
  4. 42
      next/assets/remote.go

1
go.mod

@ -17,6 +17,7 @@ require (
) )
require ( require (
github.com/avast/retry-go v3.0.0+incompatible // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
go.uber.org/multierr v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect
golang.org/x/net v0.8.0 // indirect golang.org/x/net v0.8.0 // indirect

2
go.sum

@ -2,6 +2,8 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

4
next/assets/assets.go

@ -5,9 +5,11 @@ import (
"os" "os"
) )
// updateRemoteAsset will download remote asset via the optional proxy and
// save them locally. Local files will be overwritten if they exist.
func updateRemoteAsset(file string, url string, proxy string) error { func updateRemoteAsset(file string, url string, proxy string) error {
logger.Debugf("Start downloading remote asset `%s` to `%s`", url, file) logger.Debugf("Start downloading remote asset `%s` to `%s`", url, file)
asset, date, err := download(url, proxy) asset, date, err := downloadAsset(url, proxy)
if err != nil { if err != nil {
logger.Errorf("Failed to download remote asset `%s`", url) logger.Errorf("Failed to download remote asset `%s`", url)
return err return err

42
next/assets/remote.go

@ -5,6 +5,7 @@ import (
"bytes" "bytes"
"errors" "errors"
"github.com/andybalholm/brotli" "github.com/andybalholm/brotli"
"github.com/avast/retry-go"
"github.com/go-http-utils/headers" "github.com/go-http-utils/headers"
"github.com/klauspost/compress/flate" "github.com/klauspost/compress/flate"
"github.com/klauspost/compress/gzip" "github.com/klauspost/compress/gzip"
@ -14,6 +15,9 @@ import (
"time" "time"
) )
const DownloadRetry = 3 // max retry times
const DownloadTimeout = 480 // max request seconds
// broltiDecode handles brolti encoding in http responses. // broltiDecode handles brolti encoding in http responses.
func broltiDecode(stream io.Reader) ([]byte, error) { func broltiDecode(stream io.Reader) ([]byte, error) {
var buffer bytes.Buffer var buffer bytes.Buffer
@ -68,7 +72,9 @@ func nonDecode(stream io.Reader) ([]byte, error) {
func createClient(remoteUrl string, proxyUrl string) (http.Client, error) { func createClient(remoteUrl string, proxyUrl string) (http.Client, error) {
if proxyUrl == "" { if proxyUrl == "" {
logger.Infof("Downloading `%s` without proxy", remoteUrl) logger.Infof("Downloading `%s` without proxy", remoteUrl)
return http.Client{}, nil return http.Client{
Timeout: DownloadTimeout * time.Second,
}, nil
} }
logger.Infof("Downloading `%s` via `%s`", remoteUrl, proxyUrl) logger.Infof("Downloading `%s` via `%s`", remoteUrl, proxyUrl)
proxy, err := url.Parse(proxyUrl) proxy, err := url.Parse(proxyUrl)
@ -80,9 +86,35 @@ func createClient(remoteUrl string, proxyUrl string) (http.Client, error) {
Transport: &http.Transport{ Transport: &http.Transport{
Proxy: http.ProxyURL(proxy), Proxy: http.ProxyURL(proxy),
}, },
Timeout: DownloadTimeout * time.Second,
}, nil }, nil
} }
func doRequest(client http.Client, req *http.Request) (*http.Response, error) {
maxRetry := retry.Attempts(DownloadRetry)
maxDelay := retry.MaxDelay(DownloadTimeout * time.Second)
onRetry := retry.OnRetry(func(n uint, err error) {
logger.Errorf("Failed to execute http request -> %v", err)
if DownloadRetry != n+1 {
logger.Infof("Download retry on the %d times, remain %d times...", n+1, DownloadRetry-n-1)
}
})
delay := retry.Delay(4 * time.Second) // backoff start at 4s
var resp *http.Response
retryFunc := func() error {
var err error
if resp, err = client.Do(req); err != nil {
return err
}
return nil
}
if err := retry.Do(retryFunc, delay, maxDelay, onRetry, maxRetry); err != nil {
return nil, err
}
return resp, nil
}
// assetDate attempts to obtain the last modification time of the remote // assetDate attempts to obtain the last modification time of the remote
// file and returns nil if it does not exist or is invalid. // file and returns nil if it does not exist or is invalid.
func assetDate(resp *http.Response) *time.Time { func assetDate(resp *http.Response) *time.Time {
@ -95,9 +127,9 @@ func assetDate(resp *http.Response) *time.Time {
return &date return &date
} }
// download obtains resource file from the remote server, gets its // downloadAsset obtains resource file from the remote server, gets its
// modification time, and supports proxy acquisition. // modification time, and supports proxy acquisition.
func download(url string, proxy string) ([]byte, *time.Time, error) { func downloadAsset(url string, proxy string) ([]byte, *time.Time, error) {
client, err := createClient(url, proxy) client, err := createClient(url, proxy)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@ -109,9 +141,9 @@ func download(url string, proxy string) ([]byte, *time.Time, error) {
return nil, nil, err return nil, nil, err
} }
req.Header.Set(headers.AcceptEncoding, "gzip, deflate, br") req.Header.Set(headers.AcceptEncoding, "gzip, deflate, br")
resp, err := client.Do(req)
resp, err := doRequest(client, req)
if err != nil { if err != nil {
logger.Errorf("Failed to execute http request -> %v", err)
return nil, nil, err return nil, nil, err
} }
defer resp.Body.Close() defer resp.Body.Close()

Loading…
Cancel
Save