From a36395500bc26186b09b20e3dbb4a7acd7178bb5 Mon Sep 17 00:00:00 2001 From: andriibeee <154226341+andriibeee@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:51:42 +0300 Subject: [PATCH] lib/awsapi: pre-populate credentials only for static creds without roleARN 0aaa741b5bd3ed97d4503c1c8f1e641ff72eb275 introduced a regression in lib/awsapi/config.go that causes empty credentials to be returned on the very first call to getFreshAPICredentials() when using EKS Pod Identity (or any container credential mechanism with no static access key). These empty credentials are then used for SigV4 signing -> 403 Forbidden on every remote write request. Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10815 --- docs/victoriametrics/changelog/CHANGELOG.md | 2 + lib/awsapi/config.go | 9 +- lib/awsapi/config_test.go | 105 ++++++++++++++++++++ 3 files changed, 113 insertions(+), 3 deletions(-) diff --git a/docs/victoriametrics/changelog/CHANGELOG.md b/docs/victoriametrics/changelog/CHANGELOG.md index 3ffad874a6..c8e39f803b 100644 --- a/docs/victoriametrics/changelog/CHANGELOG.md +++ b/docs/victoriametrics/changelog/CHANGELOG.md @@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel ## tip +* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `ec2_sd_configs` returning 401 `AuthFailure` from AWS when credentials are obtained via IRSA, instance role or `AWS_CONTAINER_CREDENTIALS_*` env vars. The regression was introduced in [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0). See [#10815](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10815). + ## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0) Released at 2026-04-10 diff --git a/lib/awsapi/config.go b/lib/awsapi/config.go index 151bd50e73..b887a10e80 100644 --- a/lib/awsapi/config.go +++ b/lib/awsapi/config.go @@ -91,9 +91,12 @@ func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, if len(secretKey) > 0 { cfg.defaultSecretKey = secretKey } - cfg.creds = &credentials{ - AccessKeyID: cfg.defaultAccessKey, - SecretAccessKey: cfg.defaultSecretKey, + if len(cfg.defaultAccessKey) > 0 && len(cfg.defaultSecretKey) > 0 && len(cfg.roleARN) == 0 { + // static credentials without roleARN never need refreshing; pre-populate them. + cfg.creds = &credentials{ + AccessKeyID: cfg.defaultAccessKey, + SecretAccessKey: cfg.defaultSecretKey, + } } return cfg, nil diff --git a/lib/awsapi/config_test.go b/lib/awsapi/config_test.go index d1727d4066..a8dd7d38cd 100644 --- a/lib/awsapi/config_test.go +++ b/lib/awsapi/config_test.go @@ -2,11 +2,13 @@ package awsapi import ( "fmt" + "io" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" + "strings" "testing" "time" @@ -469,6 +471,109 @@ func TestGetAPICredentialsWithProfile(t *testing.T) { } } +func TestGetFreshAPICredentialsFetchesWhenUnset(t *testing.T) { + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10815 + response := ` + + + + IRSAACCESSKEYID + IRSASECRETACCESSKEY + IRSATOKEN + 2026-01-01T00:00:00Z + + + test + +` + rt := &fakeRoundTripper{responses: make(map[string]*http.Response)} + recorder := httptest.NewRecorder() + recorder.WriteHeader(http.StatusOK) + _, _ = recorder.WriteString(response) + rt.responses["AssumeRoleWithWebIdentity"] = recorder.Result() + + tempDir := t.TempDir() + tokenPath := filepath.Join(tempDir, "token") + fs.MustWriteSync(tokenPath, []byte("webtoken")) + + cfg := &Config{ + stsEndpoint: "http://stsendpoint", + irsaRoleARN: "irsarole", + webTokenPath: tokenPath, + client: &http.Client{Transport: rt}, + } + creds, err := cfg.getFreshAPICredentials() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if creds.AccessKeyID != "IRSAACCESSKEYID" { + t.Fatalf("unexpected AccessKeyID; got %q, want %q", creds.AccessKeyID, "IRSAACCESSKEYID") + } +} + +func TestGetFreshAPICredentialsFetchesIMDS(t *testing.T) { + // verify that IMDS credentials are fetched when no static keys or IRSA config is set + // see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10817#issuecomment-4258125403 + imdsRT := &imdsRoundTripper{ + roleName: "test-role", + securityCredentials: `{ + "AccessKeyId": "IMDSACCESSKEYID", + "SecretAccessKey": "IMDSSECRETACCESSKEY", + "Token": "IMDSTOKEN", + "Expiration": "2026-01-01T00:00:00Z" + }`, + } + cfg := &Config{ + client: &http.Client{Transport: imdsRT}, + } + creds, err := cfg.getFreshAPICredentials() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if creds.AccessKeyID != "IMDSACCESSKEYID" { + t.Fatalf("unexpected AccessKeyID; got %q, want %q", creds.AccessKeyID, "IMDSACCESSKEYID") + } + if creds.SecretAccessKey != "IMDSSECRETACCESSKEY" { + t.Fatalf("unexpected SecretAccessKey; got %q, want %q", creds.SecretAccessKey, "IMDSSECRETACCESSKEY") + } + if creds.Token != "IMDSTOKEN" { + t.Fatalf("unexpected Token; got %q, want %q", creds.Token, "IMDSTOKEN") + } +} + +type imdsRoundTripper struct { + roleName string + securityCredentials string +} + +func (rt *imdsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodPut && strings.HasSuffix(req.URL.Path, "/api/token"): + // IMDSv2 session token request + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("fake-imds-token")), + Header: http.Header{"Content-Type": []string{"text/plain"}}, + }, nil + case strings.HasSuffix(req.URL.Path, "/meta-data/iam/security-credentials/"): + // Role name listing + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(rt.roleName)), + Header: http.Header{"Content-Type": []string{"text/plain"}}, + }, nil + case strings.HasSuffix(req.URL.Path, "/meta-data/iam/security-credentials/"+rt.roleName): + // Security credentials for the role + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(rt.securityCredentials)), + Header: http.Header{"Content-Type": []string{"application/json"}}, + }, nil + default: + return nil, fmt.Errorf("unexpected IMDS request: %s %s", req.Method, req.URL) + } +} + func mustParseRFC3339(s string) time.Time { expTime, err := time.Parse(time.RFC3339, s) if err != nil {