diff --git a/.chloggen/prw-error-permanent-classification.yaml b/.chloggen/prw-error-permanent-classification.yaml new file mode 100644 index 0000000000000..b5b8011053b88 --- /dev/null +++ b/.chloggen/prw-error-permanent-classification.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog) +component: exporter/prometheusremotewrite + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fix 5xx responses being incorrectly classified as `error_permanent="true"` on `otelcol_exporter_send_failed_metric_points_total` + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [48431] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index f94b030fb09cf..e62fd38e0c215 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -413,7 +413,7 @@ func (prwe *prwExporter) handleRequests(ctx context.Context, input chan *prompb. } if errExecute := prwe.execute(ctx, reqBuf); errExecute != nil { - errs = multierr.Append(errs, consumererror.NewPermanent(errExecute)) + errs = multierr.Append(errs, errExecute) } } } @@ -523,8 +523,10 @@ func (prwe *prwExporter) execute(ctx context.Context, buf []byte) error { } if err != nil { - // A permanent error is being returned here so we don't retry on context deadline exceeded. - return consumererror.NewPermanent(err) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return consumererror.NewPermanent(err) + } + return err } return nil diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index 105458cebfc82..3bdab0ee06750 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -1111,6 +1111,10 @@ func assertPermanentConsumerError(t assert.TestingT, err error, _ ...any) bool { return assert.True(t, consumererror.IsPermanent(err), "error should be consumererror.Permanent") } +func assertNonPermanentError(t assert.TestingT, err error, _ ...any) bool { + return assert.False(t, consumererror.IsPermanent(err), "error should not be consumererror.Permanent") +} + func TestRetries(t *testing.T) { tts := []struct { name string @@ -1118,6 +1122,7 @@ func TestRetries(t *testing.T) { expectedAttempts int httpStatus int RetryOnHTTP429 bool + retryEnabled bool assertError assert.ErrorAssertionFunc assertErrorType assert.ErrorAssertionFunc ctx context.Context @@ -1128,6 +1133,7 @@ func TestRetries(t *testing.T) { 4, http.StatusInternalServerError, false, + true, assert.NoError, assert.NoError, t.Context(), @@ -1138,6 +1144,7 @@ func TestRetries(t *testing.T) { 4, http.StatusTooManyRequests, true, + true, assert.NoError, assert.NoError, t.Context(), @@ -1148,6 +1155,7 @@ func TestRetries(t *testing.T) { 1, http.StatusTooManyRequests, false, + true, assert.Error, assertPermanentConsumerError, t.Context(), @@ -1158,6 +1166,7 @@ func TestRetries(t *testing.T) { 1, http.StatusBadRequest, false, + true, assert.Error, assertPermanentConsumerError, t.Context(), @@ -1168,10 +1177,33 @@ func TestRetries(t *testing.T) { 0, http.StatusInternalServerError, false, + true, assert.Error, assertPermanentConsumerError, canceledContext(), }, + { + "test 5xx with retry disabled returns non-permanent error", + 4, + 1, + http.StatusServiceUnavailable, + false, + false, + assert.Error, + assertNonPermanentError, + t.Context(), + }, + { + "test 4xx with retry disabled returns permanent error", + 4, + 1, + http.StatusBadRequest, + false, + false, + assert.Error, + assertPermanentConsumerError, + t.Context(), + }, } for _, tt := range tts { @@ -1202,7 +1234,7 @@ func TestRetries(t *testing.T) { client: http.DefaultClient, retryOnHTTP429: tt.RetryOnHTTP429, retrySettings: configretry.BackOffConfig{ - Enabled: true, + Enabled: tt.retryEnabled, }, settings: testTel.NewTelemetrySettings(), telemetry: telemetry,