1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
// Copyright 2021 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"errors"
"io"
"net"
"strings"
"time"
"github.com/googleapis/gax-go/v2"
"google.golang.org/api/googleapi"
)
// Backoff is an interface around gax.Backoff's Pause method, allowing tests to provide their
// own implementation.
type Backoff interface {
Pause() time.Duration
}
// These are declared as global variables so that tests can overwrite them.
var (
// Default per-chunk deadline for resumable uploads.
defaultRetryDeadline = 32 * time.Second
// Default backoff timer.
backoff = func() Backoff {
return &gax.Backoff{Initial: 100 * time.Millisecond}
}
// syscallRetryable is a platform-specific hook, specified in retryable_linux.go
syscallRetryable func(error) bool = func(err error) bool { return false }
)
const (
// statusTooManyRequests is returned by the storage API if the
// per-project limits have been temporarily exceeded. The request
// should be retried.
// https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
statusTooManyRequests = 429
// statusRequestTimeout is returned by the storage API if the
// upload connection was broken. The request should be retried.
statusRequestTimeout = 408
)
// shouldRetry indicates whether an error is retryable for the purposes of this
// package, unless a ShouldRetry func is specified by the RetryConfig instead.
// It follows guidance from
// https://cloud.google.com/storage/docs/exponential-backoff .
func shouldRetry(status int, err error) bool {
if 500 <= status && status <= 599 {
return true
}
if status == statusTooManyRequests || status == statusRequestTimeout {
return true
}
if err == io.ErrUnexpectedEOF {
return true
}
// Transient network errors should be retried.
if syscallRetryable(err) {
return true
}
if err, ok := err.(interface{ Temporary() bool }); ok {
if err.Temporary() {
return true
}
}
var opErr *net.OpError
if errors.As(err, &opErr) {
if strings.Contains(opErr.Error(), "use of closed network connection") {
// TODO: check against net.ErrClosed (go 1.16+) instead of string
return true
}
}
// If Go 1.13 error unwrapping is available, use this to examine wrapped
// errors.
if err, ok := err.(interface{ Unwrap() error }); ok {
return shouldRetry(status, err.Unwrap())
}
return false
}
// RetryConfig allows configuration of backoff timing and retryable errors.
type RetryConfig struct {
Backoff *gax.Backoff
ShouldRetry func(err error) bool
}
// Get a new backoff object based on the configured values.
func (r *RetryConfig) backoff() Backoff {
if r == nil || r.Backoff == nil {
return backoff()
}
return &gax.Backoff{
Initial: r.Backoff.Initial,
Max: r.Backoff.Max,
Multiplier: r.Backoff.Multiplier,
}
}
// This is kind of hacky; it is necessary because ShouldRetry expects to
// handle HTTP errors via googleapi.Error, but the error has not yet been
// wrapped with a googleapi.Error at this layer, and the ErrorFunc type
// in the manual layer does not pass in a status explicitly as it does
// here. So, we must wrap error status codes in a googleapi.Error so that
// ShouldRetry can parse this correctly.
func (r *RetryConfig) errorFunc() func(status int, err error) bool {
if r == nil || r.ShouldRetry == nil {
return shouldRetry
}
return func(status int, err error) bool {
if status >= 400 {
return r.ShouldRetry(&googleapi.Error{Code: status})
}
return r.ShouldRetry(err)
}
}
|
