relabel debug: support multiple lines metrics input

Previously OPTIONS HTTP requests for CORS preflight checks would trigger
the original request handler. This pull request fixes that behavior to
align with https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5563

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

app/vmagent/datadogsketches/request_handler.go

@@ -49,6 +49,11 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 				Name:  "__name__",
 				Value: m.Name,
 			})
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			labels = append(labels, prompb.Label{
+				Name:  "host",
+				Value: sketch.Host,
+			})
 			for _, label := range m.Labels {
 				labels = append(labels, prompb.Label{
 					Name:  label.Name,
@@ -57,9 +62,6 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				labels = append(labels, prompb.Label{
 					Name:  name,
 					Value: value,

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,30 +83,58 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
+type UrlRelabelCfg struct {
+	Url           string `yaml:"url"`
+	RelabelConfig any    `yaml:"relabel_config"`
+
+	RelabelConfigStr string
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
-	p := remoteWriteURLRelabelConfigData.Load()
-	if p == nil {
+	cs := GetURLRelabelConfigData()
+	if cs == nil {
 		// Nothing to write to w
 		return
 	}
-	type urlRelabelCfg struct {
-		Url           string `yaml:"url"`
-		RelabelConfig any    `yaml:"relabel_config"`
+	d, _ := yaml.Marshal(cs)
+	_, _ = w.Write(d)
+}
+
+// GetURLRelabelConfigData is similar to WriteURLRelabelConfigData but returning data in []UrlRelabelCfg.
+func GetURLRelabelConfigData() []UrlRelabelCfg {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
 	}
-	var cs []urlRelabelCfg
+	var cs []UrlRelabelCfg
 	for i, url := range *remoteWriteURLs {
 		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
 		if !*showRemoteWriteURL {
 			url = fmt.Sprintf("%d:secret-url", i+1)
 		}
-		cs = append(cs, urlRelabelCfg{
+		cs = append(cs, UrlRelabelCfg{
 			Url:           url,
 			RelabelConfig: cfgData,
+
+			RelabelConfigStr: string(cfgDataBytes),
 		})
 	}
-	d, _ := yaml.Marshal(cs)
-	_, _ = w.Write(d)
+	return cs
 }
 
 func reloadRelabelConfigs() {

app/vmalert/config/config.go

@@ -81,12 +81,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.Interval.Duration() < 0 {
 		return fmt.Errorf("interval shouldn't be lower than 0")
 	}
-	if g.EvalOffset.Duration() < 0 {
-		return fmt.Errorf("eval_offset shouldn't be lower than 0")
-	}
-	// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
-	if g.EvalOffset.Duration() > g.Interval.Duration() {
-		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
+	// if `eval_offset` is set, the group interval must be specified explicitly(instead of inherited from global evaluationInterval flag) and must bigger than offset.
+	if g.EvalOffset.Duration().Abs() > g.Interval.Duration() {
+		return fmt.Errorf("the abs value of eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
 	}
 	if g.EvalOffset != nil && g.EvalDelay != nil {
 		return fmt.Errorf("eval_offset cannot be used with eval_delay")

app/vmalert/config/config_test.go

@@ -176,11 +176,17 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, false, "interval shouldn't be lower than 0")
 
 	f(&Group{
-		Name:       "wrong eval_offset",
+		Name:       "too big eval_offset",
 		Interval:   promutil.NewDuration(time.Minute),
 		EvalOffset: promutil.NewDuration(2 * time.Minute),
 	}, false, "eval_offset should be smaller than interval")
 
+	f(&Group{
+		Name:       "too big negative eval_offset",
+		Interval:   promutil.NewDuration(time.Minute),
+		EvalOffset: promutil.NewDuration(-2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
 	limit := -1
 	f(&Group{
 		Name:  "wrong limit",

app/vmalert/remotewrite/client.go

@@ -186,6 +186,11 @@ func (c *Client) run(ctx context.Context) {
 				return
 			case <-ticker.C:
 				c.flush(ctx, wr)
+				// drain the potential stale tick to avoid small or empty flushes after a slow flush.
+				select {
+				case <-ticker.C:
+				default:
+				}
 			case ts, ok := <-c.input:
 				if !ok {
 					continue

app/vmalert/rule/group.go

@@ -484,8 +484,15 @@ func (g *Group) UpdateWith(newGroup *Group) {
 // delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
 func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
 	if g.EvalOffset != nil {
+		offset := *g.EvalOffset
+		// adjust the offset for negative evalOffset, the rule is:
+		// `eval_offset: -x` is equivalent to `eval_offset: y` for `interval: x+y`.
+		// For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
+		if offset < 0 {
+			offset += g.Interval
+		}
 		// if offset is specified, ignore the maxDelay and return a duration aligned with offset
-		currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
+		currentOffsetPoint := ts.Truncate(g.Interval).Add(offset)
 		if currentOffsetPoint.Before(ts) {
 			// wait until the next offset point
 			return currentOffsetPoint.Add(g.Interval).Sub(ts)

app/vmalert/rule/group_test.go

@@ -606,6 +606,15 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
 	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
 
+	// test group with negative offset -2min, which is equivalent to 3min offset for 5min interval
+	offset = -2 * time.Minute
+	g.EvalOffset = &offset
+
+	f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
+	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
+
 	maxDelay = time.Minute * 1
 	g.EvalOffset = nil
 

app/vmauth/auth_config.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -28,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -90,6 +92,8 @@ type UserInfo struct {
 
 	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
 
+	AccessLog *AccessLog `yaml:"access_log,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
@@ -102,6 +106,31 @@ type UserInfo struct {
 	requestsDuration *metrics.Summary
 }
 
+// AccessLog represents configuration for access log settings.
+type AccessLog struct {
+	Filters *AccessLogFilters `yaml:"filters"`
+}
+
+// AccessLogFilters represents list of filters for access logs printing
+type AccessLogFilters struct {
+	// SkipStatusCodes is a list of HTTP status codes for which access logs will be skipped
+	SkipStatusCodes []int `yaml:"skip_status_codes"`
+}
+
+func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int) {
+	filters := ui.AccessLog.Filters
+	if filters != nil && len(filters.SkipStatusCodes) > 0 {
+		if slices.Contains(filters.SkipStatusCodes, statusCode) {
+			return
+		}
+	}
+
+	remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+	requestURI := httpserver.GetRequestURI(r)
+	logger.Infof("access_log request_host=%q request_uri=%q status_code=%d remote_addr=%s user_agent=%q referer=%q username=%q",
+		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), userName)
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`

app/vmauth/auth_config_test.go

@@ -681,6 +681,31 @@ users:
 			URLPrefix: mustParseURL("http://aaa:343/bbb"),
 		},
 	}, nil)
+
+	// Multiple users with access logs enabled
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo
+  access_log: {}
+- username: bar
+  url_prefix: https://bar/x/
+  access_log:
+    filters:
+      skip_status_codes: [404]
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo", ""): {
+			Username:  "foo",
+			URLPrefix: mustParseURL("http://foo"),
+			AccessLog: &AccessLog{},
+		},
+		getHTTPAuthBasicToken("bar", ""): {
+			Username:  "bar",
+			URLPrefix: mustParseURL("https://bar/x/"),
+			AccessLog: &AccessLog{Filters: &AccessLogFilters{SkipStatusCodes: []int{404}}},
+		},
+	}, nil)
+
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {

app/vmauth/example_config.yml

@@ -125,3 +125,8 @@ unauthorized_user:
   - http://vmselect-az1/?deny_partial_response=1
   - http://vmselect-az2/?deny_partial_response=1
   retry_status_codes: [503, 500]
+  # log access for requests routed to this user
+  access_log:
+    filters:
+      # except requests with Status Codes below
+      skip_status_codes: [200, 202]

app/vmauth/main.go

@@ -226,6 +226,36 @@ func getUserInfoByAuthTokens(ats []string) *UserInfo {
 	return nil
 }
 
+// responseWriterWithStatus is a wrapper around http.ResponseWriter that captures the status code written to the response.
+type responseWriterWithStatus struct {
+	http.ResponseWriter
+	status int
+}
+
+// WriteHeader records the status so it can be easily retrieved later
+func (rws *responseWriterWithStatus) WriteHeader(status int) {
+	rws.status = status
+	rws.ResponseWriter.WriteHeader(status)
+}
+
+// Flush implements net/http.Flusher interface
+//
+// This is needed for the copyStreamToClient()
+func (rws *responseWriterWithStatus) Flush() {
+	flusher, ok := rws.ResponseWriter.(http.Flusher)
+	if !ok {
+		logger.Panicf("BUG: it is expected http.ResponseWriter (%T) supports http.Flusher interface", rws.ResponseWriter)
+	}
+	flusher.Flush()
+}
+
+// Unwrap returns the original ResponseWriter wrapped by rws.
+//
+// This is needed for the net/http.ResponseController - see https://pkg.go.dev/net/http#NewResponseController
+func (rws *responseWriterWithStatus) Unwrap() http.ResponseWriter {
+	return rws.ResponseWriter
+}
+
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
@@ -235,6 +265,19 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+
+	if ui.AccessLog != nil {
+		w = &responseWriterWithStatus{ResponseWriter: w}
+		defer func() {
+			rws := w.(*responseWriterWithStatus)
+			ui.logRequest(r, userName, rws.status)
+		}()
+	}
+
 	// Acquire global concurrency limit.
 	if err := beginConcurrencyLimit(ctx); err != nil {
 		handleConcurrencyLimitError(w, r, err)
@@ -253,10 +296,6 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 	}
 
 	// Read the initial chunk for the request body.
-	userName := ui.name()
-	if userName == "" {
-		userName = "unauthorized"
-	}
 	bb, err := bufferRequestBody(ctx, r.Body, userName)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)

app/vminsert/datadogsketches/request_handler.go

@@ -45,15 +45,14 @@ func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompb.Label)
 		ms := sketch.ToSummary()
 		for _, m := range ms {
 			ctx.Labels = ctx.Labels[:0]
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			ctx.AddLabel("host", sketch.Host) // newly added
 			ctx.AddLabel("", m.Name)
 			for _, label := range m.Labels {
 				ctx.AddLabel(label.Name, label.Value)
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				ctx.AddLabel(name, value)
 			}
 			for j := range extraLabels {

app/vmselect/main.go

@@ -321,19 +321,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/tags/tagSeries":
 		graphiteTagsTagSeriesRequests.Inc()
-		if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags/tagMultiSeries":
 		graphiteTagsTagMultiSeriesRequests.Inc()
-		if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagMultiSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagMultiSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags":
 		graphiteTagsRequests.Inc()

apptest/model.go

@@ -33,6 +33,8 @@ type PrometheusQuerier interface {
 	// separate interface or rename this interface to allow for multiple querier
 	// types.
 	GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
+	GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
+	GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
 }
 
 // Writer contains methods for writing new data

apptest/tests/graphite_test.go

@@ -60,3 +60,60 @@ func TestClusterMetricsIndex(t *testing.T) {
 
 	testMetricsIndex(tc.T(), sut)
 }
+
+// testTagSeries tests the registration of new time series in index.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb.
+func testTagSeries(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, getStorageMetric func(string) int) {
+	t := tc.T()
+
+	assertNewTimeseriesCreatedTotal := func(want int) {
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected vm_new_timeseries_created_total",
+			Got: func() any {
+				return getStorageMetric("vm_new_timeseries_created_total")
+			},
+			Want: want,
+		})
+	}
+
+	rec := "disk.used;rack=a1;datacenter=dc1;server=web01"
+	sut.GraphiteTagsTagSeries(t, rec, apptest.QueryOpts{})
+	assertNewTimeseriesCreatedTotal(0)
+
+	recs := []string{
+		"metric.yyy;t2=a;t1=b;t3=c",
+		"metric.zzz;t5=d;t4=e;t6=f",
+		"metric.xxx;t8=g;t7=h;t9=i",
+	}
+	sut.GraphiteTagsTagMultiSeries(t, recs, apptest.QueryOpts{})
+	assertNewTimeseriesCreatedTotal(0)
+}
+
+func TestSingleTagSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultVmsingle()
+	getStorageMetric := func(name string) int {
+		return sut.GetIntMetric(t, name)
+	}
+
+	testTagSeries(tc, sut, getStorageMetric)
+}
+
+func TestClusterTagSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultCluster()
+	getStorageMetric := func(name string) int {
+		var v int
+		for _, s := range sut.Vmstorages {
+			v += s.GetIntMetric(t, name)
+		}
+		return v
+	}
+
+	testTagSeries(tc, sut, getStorageMetric)
+}

apptest/tests/per_day_index_test.go

@@ -61,8 +61,8 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
 
 type startSUTFunc func(name string, disablePerDayIndex bool) apptest.PrometheusWriteQuerier
 
-// testDisablePerDayIndex_Search shows what search results to expect when data
-// is first inserted with per-day index enabled and then with per-day index
+// testSearchWithDisabledPerDayIndex shows what search results to expect when
+// data is first inserted with per-day index enabled and then with per-day index
 // disabled.
 //
 // The data inserted with enabled per-day index must be searchable with disabled
@@ -112,8 +112,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
 		})
 	}
 
-	// Start vmsingle with enabled per-day index, insert sample1, and confirm it
-	// is searchable.
+	// Start SUT with enabled per-day index, insert sample1, and confirm it is
+	// searchable.
 	sut := start("with-per-day-index", false)
 	sample1 := []string{"metric1 111 1704067200000"} // 2024-01-01T00:00:00Z
 	sut.PrometheusAPIV1ImportPrometheus(t, sample1, apptest.QueryOpts{})
@@ -130,8 +130,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
 		},
 	})
 
-	// Restart vmsingle with disabled per-day index, insert sample2, and confirm
-	// that both sample1 and sample2 is searchable.
+	// Restart SUT with disabled per-day index, insert sample2, and confirm that
+	// both sample1 and sample2 is searchable.
 	tc.StopPrometheusWriteQuerier(sut)
 	sut = start("without-per-day-index", true)
 	sample2 := []string{"metric2 222 1704067200000"} // 2024-01-01T00:00:00Z
@@ -156,8 +156,8 @@ func testSearchWithDisabledPerDayIndex(tc *apptest.TestCase, start startSUTFunc)
 		},
 	})
 
-	// Insert sample1 but for a different date, restart vmsingle with enabled
-	// per-day index and confirm that:
+	// Insert sample1 but for a different date, restart SUT with enabled per-day
+	// index and confirm that:
 	// - sample1 is searchable within the time range of Jan 1st
 	// - sample1 is not searchable within the time range of Jan 20th
 	// - sample1 is searchable within the time range of Jan 1st-20th (because

apptest/vminsert.go

@@ -298,13 +298,14 @@ func (app *Vminsert) String() string {
 func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
 	t.Helper()
 
+	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
+
 	send()
 
 	const (
 		retries = 20
 		period  = 100 * time.Millisecond
 	)
-	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
 	for range retries {
 		d := app.rpcRowsSentTotal(t)
 		if d >= wantRowsSentCount {

apptest/vmselect.go

@@ -307,6 +307,37 @@ func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) Graphite
 	return index
 }
 
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagMultiSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
 // APIV1AdminTenants sends a query to a /admin/tenants endpoint
 func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
 	t.Helper()

apptest/vmsingle.go

@@ -414,6 +414,37 @@ func (app *Vmsingle) GraphiteMetricsIndex(t *testing.T, _ QueryOpts) GraphiteMet
 	return index
 }
 
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmsingle) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmsingle) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagMultiSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
 // APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
 // and returns the statistics response for given params.
 //

dashboards/vm/vmauth.json

@@ -1612,7 +1612,7 @@
                 "type": "victoriametrics-metrics-datasource",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
@@ -1624,7 +1624,7 @@
                 "type": "victoriametrics-metrics-datasource",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

dashboards/vmauth.json

@@ -1611,7 +1611,7 @@
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
@@ -1623,7 +1623,7 @@
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     depends_on:
       - "vmauth"
     ports:
@@ -38,14 +38,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.136.0-cluster
+    image: victoriametrics/vmstorage:v1.137.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.136.0-cluster
+    image: victoriametrics/vmstorage:v1.137.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -55,7 +55,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.136.0-cluster
+    image: victoriametrics/vminsert:v1.137.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -64,7 +64,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.136.0-cluster
+    image: victoriametrics/vminsert:v1.137.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -76,7 +76,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.136.0-cluster
+    image: victoriametrics/vmselect:v1.137.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -86,7 +86,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.136.0-cluster
+    image: victoriametrics/vmselect:v1.137.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -101,7 +101,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.136.0
+    image: victoriametrics/vmauth:v1.137.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -115,7 +115,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.136.0
+    image: victoriametrics/vmalert:v1.137.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.136.0
+    image: victoriametrics/victoria-metrics:v1.137.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -55,7 +55,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.136.0
+    image: victoriametrics/vmalert:v1.137.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.136.0
+    image: victoriametrics/victoria-metrics:v1.137.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.136.0
+    image: victoriametrics/vmalert:v1.137.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,9 +10,9 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.136.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.136.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.136.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.137.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.137.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.137.0)
 - [Grafana](https://grafana.com/) (v.10.2.1)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -340,7 +340,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.136.0
+    image: victoriametrics/victoria-metrics:v1.137.0
     ports:
       - 8428:8428
     volumes:
@@ -373,7 +373,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.136.0
+    image: victoriametrics/vmalert:v1.137.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/getting-started-with-opentelemetry/README.md

@@ -316,4 +316,4 @@ using query `service.name: unknown_service:otel`.
 ## Limitations
 
 - VictoriaMetrics and VictoriaLogs do not support experimental JSON encoding [format](https://github.com/open-telemetry/opentelemetry-proto/blob/main/examples/metrics.json).
-- VictoriaMetrics supports only the `AggregationTemporalityCumulative` type for [histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#histogram) and [summary](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#summary-legacy). Either consider using cumulative temporality or use the [`delta-to-cumulative processor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) to convert to cumulative temporality in OpenTelemetry Collector.
+

docs/guides/grafana-vmgateway-openid-configuration/README.md

@@ -249,27 +249,27 @@ services:
       - grafana_data:/var/lib/grafana/
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.136.0
+    image: victoriametrics/victoria-metrics:v1.137.0
     command:
       - -httpListenAddr=0.0.0.0:8429
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.136.0-cluster
+    image: victoriametrics/vmstorage:v1.137.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.136.0-cluster
+    image: victoriametrics/vminsert:v1.137.0-cluster
     command:
       - -storageNode=vmstorage:8400
       - -httpListenAddr=0.0.0.0:8480
 
   vmselect:
-    image: victoriametrics/vmselect:v1.136.0-cluster
+    image: victoriametrics/vmselect:v1.137.0-cluster
     command:
       - -storageNode=vmstorage:8401
       - -httpListenAddr=0.0.0.0:8481
 
   vmagent:
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -278,7 +278,7 @@ services:
       - -remoteWrite.url=http://vmsingle:8429/api/v1/write
 
   vmgateway-cluster:
-    image: victoriametrics/vmgateway:v1.136.0-enterprise
+    image: victoriametrics/vmgateway:v1.137.0-enterprise
     ports:
       - 8431:8431
     volumes:
@@ -294,7 +294,7 @@ services:
       - -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
 
   vmgateway-single:
-    image: victoriametrics/vmgateway:v1.136.0-enterprise
+    image: victoriametrics/vmgateway:v1.137.0-enterprise
     ports:
       - 8432:8431
     volumes:
@@ -405,7 +405,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
 
 ```yaml
   vmagent:
-    image: victoriametrics/vmagent:v1.136.0
+    image: victoriametrics/vmagent:v1.137.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
       - ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

docs/victoriametrics/FAQ.md

@@ -557,8 +557,8 @@ and proportionally to the total length of all the labels seen across all the reg
 
 Typical monitoring in Kubernetes generates moderate-to-high churn rate for time series because every restart of the `pod` creates a new set of time series
 for all the [metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#what-is-a-metric) exposed by that pod, with a new `pod` label.
-The number of labels and the summary length of `label=value` pairs per every time series in Kubernetes is quite large
-(~30-40 labels with ~1KB summary length of `label=value` pairs per time series). This contributes to quick growth of the `indexdb` over time,
+The number of labels and the total length of `label=value` pairs per every time series in Kubernetes is quite large
+(~30-40 labels with ~1KB total length of `label=value` pairs per time series). This contributes to quick growth of the `indexdb` over time,
 so its' size may exceed the size of the `data` folder by up to 2x in typical production cases.
 
 There are the following workarounds, which can reduce the growth rate of the `indexdb`:

docs/victoriametrics/Quick-Start.md

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.136.0
+docker pull victoriametrics/victoria-metrics:v1.137.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.136.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.137.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -708,7 +708,7 @@ Using the delete API is not recommended in the following cases, since it brings
   time series occupy disk space until the next merge operation, which can never occur when deleting too old data.
   [Forced merge](#forced-merge) may be used for freeing up disk space occupied by old data.
   Note that VictoriaMetrics doesn't delete entries from [IndexDB](#indexdb) for the deleted time series.
-  IndexDB is cleaned up once per the configured [retention](#retention).
+  IndexDB is cleaned up along with the corresponding data partition once it becomes outside the [-retentionPeriod](#retention).
 
 It's better to use the `-retentionPeriod` command-line flag for efficient pruning of old data.
 
@@ -1419,21 +1419,22 @@ See also [how to work with snapshots](#how-to-work-with-snapshots) and [IndexDB]
 ## IndexDB
 
 VictoriaMetrics identifies
-[time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) by
-`TSID` (time series ID) and stores
-[raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) sorted
-by TSID (see [Storage](#storage)). Thus, the TSID is a primary index and could
-be used for searching and retrieving raw samples. However, the TSID is never
-exposed to the clients, i.e. it is for internal use only.
+[time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
+by `TSID` (time series ID) and stores
+[raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+sorted by TSID (see [Storage](#storage)). Thus, the TSID is a primary index and
+could be used for searching and retrieving raw samples. However, the TSID is
+never exposed to the clients, i.e. it is for internal use only.
 
-Instead, VictoriaMetrics maintains an **inverted index** that enables searching
-the raw samples by metric name, label name, and label value by mapping these
-values to the corresponding TSIDs.
+Instead, VictoriaMetrics maintains an **inverted index** (known as `indexDB`)
+that enables searching the raw samples by metric name, label name, and label
+value by mapping these values to the corresponding TSIDs. Every data
+[partition](#storage) has its own indexDB.
 
 VictoriaMetrics uses two types of inverted indexes:
 
 * Global index. Searches using this index is performed across the entire
-  retention period.
+  partition time range.
 * Per-day index. This index stores mappings similar to ones in global index
   but also includes the date in each mapping. This speeds up data retrieval
   for queries within a shorter time range (which is often just the last day).
@@ -1441,19 +1442,18 @@ VictoriaMetrics uses two types of inverted indexes:
 When the search query is executed, VictoriaMetrics decides which index to use
 based on the time range of the query:
 
-* Per-day index is used if the search time range is 40 days or less.
-* Global index is used for search queries with a time range greater than 40
-  days.
+* Per-day index is used if the search time range is less than the partition time range.
+* Global index is used for search queries with a time range that matches exactly
+  or greater than the partition time range.
 
 Mappings are added to the indexes during the data ingestion:
 
-* In global index each mapping is created only once per retention period.
+* In global index each mapping is created only once per partition.
 * In the per-day index each mapping is created for each unique date that
   has been seen in the samples for the corresponding time series.
 
-IndexDB respects [retention period](#retention) and once it is over, the indexes
-are dropped. For the new retention period, the indexes are gradually populated
-again as the new samples arrive.
+Since indexDB is a part of a partition, it is dropped along with it as it
+becomes outside the [retention period](#retention).
 
 See also [Why IndexDB size is so large?](https://docs.victoriametrics.com/victoriametrics/faq/#why-indexdb-size-is-so-large).
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,16 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: all VictoriaMetrics components: implement proper CORS preflight handling by responding 204 No Content to HTTP OPTIONS requests. See [#5563](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5563).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `access_log` configuration option for each user that will log requests to stdout, and support filtering by HTTP status codes. See more in [docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). See [#5936](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5936).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support negative values for the group `eval_offset` option, which allows starting group evaluation at `groupInterval-abs(eval_offset)` within `[0...groupInterval]`. See [#10424](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10424).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): enhance metrics relabel debug by adding remote write relabel configs to the relabel configs input. See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918). 
+
+* BUGFIX: [dashboards/vmauth](https://grafana.com/grafana/dashboards/21394): fix `requested from system` and `heap inuse` expressions in the memory usage panel. See [#10574](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10574).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not enable ACL when uploading backups to S3-compatible endpoints by default. ACL is not always supported by S3-compatible endpoints and it is not recommended to use ACLs to limit access to objects. See [#10539](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10539) for more details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly attach `host` label to the time series ingested via [/datadog/api/beta/sketches](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/#) API. See [#10557](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557).
+
 ## [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.137.0)
 
 Released at 2026-02-27
@@ -35,7 +45,7 @@ It enables back `Discovered targets` debug UI by default.
 
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/): can now copy backups between different storage backends, such as from s3 to local disk or gcs to s3. See [#10401](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10401). Thanks to @BenNF for the contribution.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add JWT token authentication support with signature verification based on provided `public_keys`. Read more about configuration in [JWT Token auth proxy](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) documentation. See [#10445](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10445).
-* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/):  support dynamic rewriting of upstream URLs and request headers using placeholders populated from JWT `vm_access` claim fields. This allows routing requests to the correct tenant backend without maintaining a separate user config entry per tenant. Read more in [JWT claim-based request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) documentation. See [#10492](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10492).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): support dynamic rewriting of upstream URLs and request headers using placeholders populated from JWT `vm_access` claim fields. This allows routing requests to the correct tenant backend without maintaining a separate user config entry per tenant. Read more in [JWT claim-based request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) documentation. See [#10492](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10492).
 * FEATURE: all VictoriaMetrics components: expose `process_cpu_seconds_total`, `process_resident_memory_bytes`, and other process-level metrics when running on macOS. See [metrics#75](https://github.com/VictoriaMetrics/metrics/issues/75).
 * FEATURE: [dashboards/vmauth](https://grafana.com/grafana/dashboards/21394): add `Request body buffering duration` panel to the `Troubleshooting` section. This panel shows the time spent buffering incoming client request bodies, helping identify slow client uploads and potential concurrency issues. The panel is only available when `-requestBufferSize` is non-zero. See [#10309](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10309).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU and memory usage when `-promscrape.dropOriginalLabels` command-line flag is set. See [#9952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9952).
@@ -209,6 +219,19 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.16](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.16)
+
+Released at 2026-02-27
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic `error parsing regexp: expression nests too deeply` triggered by large repetition ranges in regex, for example `{"__name__"=~"a{0,1000}"}`. See [VictoriaLogs#1112](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1112).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly search tenants for [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) query request. See [#10422](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10422).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix escaping for label names with special characters. See [#10485](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10485).
+
 ## [v1.122.15](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.15)
 
 Released at 2026-02-13
@@ -361,6 +384,16 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11110)
 
+## [v1.110.31](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.31)
+
+Released at 2026-02-27
+
+**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11100) release**
+
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly search tenants for [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) query request. See [#10422](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10422).
+
 ## [v1.110.30](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.30)
 
 Released at 2026-02-13

docs/victoriametrics/enterprise.md

@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
 
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.136.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.136.0/victoria-metrics-linux-amd64-v1.136.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.136.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.136.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.137.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](#binary-releases) section.
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.136.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.137.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.136.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.137.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +177,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.136.0
+    image: victoriametrics/victoria-metrics:v1.137.0
     ports:
       - 8428:8428
     volumes:
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.136.0-enterprise
+    tag: v1.137.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.136.0-enterprise
+    tag: v1.137.0-enterprise
 
 license:
   secret:
@@ -270,7 +270,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.136.0-enterprise
+    tag: v1.137.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +287,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.136.0-enterprise
+    tag: v1.137.0-enterprise
 ```
 
 Example secret with license key:
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.136.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,7 +347,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.136.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.137.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## Monitoring license expiration
 

docs/victoriametrics/integrations/opentelemetry.md

@@ -13,10 +13,10 @@ It expects `protobuf`-encoded requests at `/opentelemetry/v1/metrics`. For gzip-
 
 See how to configure [OpenTelemetry Collector](https://docs.victoriametrics.com/victoriametrics/data-ingestion/opentelemetry-collector/) to push metrics to VictoriaMetrics.
 
-## Metric naming
+## Label sanitization
 
-By default, VictoriaMetrics stores the ingested OpenTelemetry [metric samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) as is **without any transformations**.
-The following label transformations can be enabled:
+By default, VictoriaMetrics stores the ingested OpenTelemetry [metric points](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#metric-points) as is **without any transformations**.
+The following label sanitization options can be enabled:
 * `-usePromCompatibleNaming` - replaces characters unsupported by Prometheus with `_` in metric names and labels **for all ingestion protocols**.
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time{service_name="foo"}`.
 * `-opentelemetry.usePrometheusNaming` - converts metric names and labels according to [OTLP Metric points to Prometheus specification](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.33.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus) for metrics ingested via OTLP.
@@ -26,10 +26,21 @@ The following label transformations can be enabled:
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 
+## Resource Attributes
+
+By default, VictoriaMetrics promotes all [OpenTelemetry resource](https://opentelemetry.io/docs/specs/otel/resource/data-model/) attributes to labels and attaches them to all ingested OTLP metrics.
+
 ## Exponential histograms
 
 OpenTelemetry [exponential histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram) is automatically converted
-to [VictoriaMetrics histogram format](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350).
+to [VictoriaMetrics histogram format](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) during ingestion. Since VictoriaMetrics histogram doesn't support negative observations, all buckets in the negative range are dropped.
+
+## Delta Temporality
+
+In OpenTelemetry, some metric types(including sums, histograms, and exponential histograms) support delta and cumulative aggregation temporality. VictoriaMetrics works best with cumulative temporality, and it's recommended to export metrics with cumulative temporality or convert delta to cumulative temporality using [OpenTelemetry Collector deltatocumulative processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) before sending to VictoriaMetrics.
+VictoriaMetrics stores delta temporality metric values as is {{% available_from "v1.132.0" %}}, they can be queried with [sum_over_time()](https://docs.victoriametrics.com/victoriametrics/metricsql/#sum_over_time) and [rate_over_sum()](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate_over_sum).
+
+> Do not apply [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling) to delta temporality metrics, since it might cause data loss.
 
 ## References
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.136.0/victoria-metrics-linux-amd64-v1.136.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.136.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.137.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.136.0/victoria-metrics-linux-amd64-v1.136.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.136.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.137.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/vmalert.md

@@ -144,8 +144,10 @@ name: <string>
 # Optional
 # Group will be evaluated at the exact offset in the range of [0...interval].
 # E.g. for Group with `interval: 1h` and `eval_offset: 5m` the evaluation will
-# start at 5th minute of the hour. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409
-# `interval` must be specified if `eval_offset` is used, and `eval_offset` cannot exceed `interval`.
+# start at 5th minute of the hour.
+# `eval_offset` also supports negative values, which means the evaluation will start at `interval-abs(eval_offset)` within [0...interval],
+# For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
+# `interval` must be specified if `eval_offset` is used, and the `abs(eval_offset)` cannot exceed `interval`.
 # `eval_offset` cannot be used with `eval_delay`, as group will be executed at the exact offset and `eval_delay` is ignored.
 [ eval_offset: <duration> ]
 

docs/victoriametrics/vmauth.md

@@ -1056,6 +1056,43 @@ unauthorized_user:
     url_prefix: 'http://victoria-logs:9428/?extra_filters={env="prod"}'
 ```
 
+## Access log
+
+vmauth allows configuring access logs {{% available_from "#" %}} printing per-user:
+```yaml
+unauthorized_user:
+  url_prefix: 'http://localhost:8428/'
+  # Log all requests to this user
+  access_log: {}
+```
+
+Access logs contain limited information to prevent exposing sensitive data. See an example of the printed access log below:
+```bash
+2026-02-26T15:00:00.207Z        info    VictoriaMetrics/app/vmauth/auth_config.go:134   access_log request_host="localhost:8427" request_uri="/prometheus/api/v1/query_range?query=1&start=1772116199.897&end=1772117999.897&step=5s" status_code=200 remote_addr="127.0.0.1:63425" user_agent="Mozilla/5.0..." referer="http://localhost:8427/vmui/?" username="unauthorized"
+```
+
+The printed log starts with `access_log` prefix and is followed with `request_host`, `request_uri`, `status_code`, `remote_addr`,
+`user_agent`, `referer` and `username` fields in [logfmt](https://brandur.org/logfmt) format. Such logs can be later
+analyzed in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs):
+```logsql
+access_log | extract 'access_log <access_log>' | unpack_logfmt from access_log
+| stats by(username, request_host, status_code) count()
+```
+
+Access logs can skip logging requests with specified status codes:
+```yaml
+users:
+- username: foo
+  password: bar
+  url_prefix: 'http://localhost:8428/'
+  access_log:
+    filters:
+      # except requests with HTTP status codes below
+      skip_status_codes: [200, 202]
+```
+
+Access logs can be enabled or disabled per-user with [hot config reload](https://docs.victoriametrics.com/victoriametrics/vmauth/#config-reload).
+
 ## Auth config
 
 `-auth.config` is represented in the following `yml` format:

docs/victoriametrics/vmbackup.md

@@ -492,7 +492,7 @@ Run `vmbackup -help` in order to see all the available options:
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -s3ACL string
-     ACL to be set for uploaded objects to S3. Supported values are: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control
+     ACL to be set for uploaded objects to S3. If not set, no ACL header is sent. Supported values are: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write.
   -s3ForcePathStyle
      Prefixing endpoint with bucket name when set false, true by default. (default true)
   -s3ObjectTags string

docs/victoriametrics/vmbackupmanager.md

@@ -680,7 +680,7 @@ command-line flags:
   -runOnStart
      Upload backups immediately after start of the service. Otherwise the backup starts on new hour
   -s3ACL string
-     ACL to be set for uploaded objects to S3. Supported values are: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control
+     ACL to be set for uploaded objects to S3. If not set, no ACL header is sent. Supported values are: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write.
   -s3ForcePathStyle
      Prefixing endpoint with bucket name when set false, true by default. (default true)
   -s3ObjectTags string

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.136.0/vmutils-darwin-arm64-v1.136.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/vmutils-darwin-arm64-v1.137.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.136.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.137.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

lib/backup/actions/util.go

@@ -28,7 +28,8 @@ var (
 	configProfile = flag.String("configProfile", "", "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), "+
 		"or if both not set, DefaultSharedConfigProfile is used")
 	customS3Endpoint = flag.String("customS3Endpoint", "", "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set")
-	s3ACL            = flag.String("s3ACL", "bucket-owner-full-control", "ACL to be set for uploaded objects to S3.")
+	s3ACL            = flag.String("s3ACL", "", "ACL to be set for uploaded objects to S3. If not set, no ACL header is sent. "+
+		"Supported values are: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write.")
 	s3ForcePathStyle = flag.Bool("s3ForcePathStyle", true, "Prefixing endpoint with bucket name when set false, true by default.")
 	s3StorageClass   = flag.String("s3StorageClass", "", "The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, "+
 		"DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.\n"+

lib/httpserver/httpserver.go

@@ -357,6 +357,12 @@ func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
 		r.URL.Path = path
 	}
 
+	if r.Method == http.MethodOptions {
+		EnableCORS(w, r)
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+
 	w = &responseWriterWithAbort{
 		ResponseWriter: w,
 	}
@@ -511,6 +517,8 @@ func EnableCORS(w http.ResponseWriter, _ *http.Request) {
 		return
 	}
 	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "*")
+	w.Header().Set("Access-Control-Allow-Headers", "*")
 }
 
 func pprofHandler(profileName string, w http.ResponseWriter, r *http.Request) {

lib/httpserver/httpserver_test.go

@@ -144,6 +144,55 @@ func TestAuthKeyMetrics(t *testing.T) {
 	tstWithOutAuthKey("wrong", "wrong", 401)
 }
 
+func TestHandlerWrapperOptionsRequest(t *testing.T) {
+	handlerCalled := false
+	rh := func(_ http.ResponseWriter, _ *http.Request) bool {
+		handlerCalled = true
+		return true
+	}
+	headersToCheck := []string{"Access-Control-Allow-Origin", "Access-Control-Allow-Headers"}
+	f := func(t *testing.T, corsDisabled bool) {
+		t.Helper()
+		handlerCalled = false
+
+		origDisableCORS := *disableCORS
+		*disableCORS = corsDisabled
+		defer func() {
+			*disableCORS = origDisableCORS
+		}()
+
+		wantCORSHeaderValue := "*"
+		if corsDisabled {
+			wantCORSHeaderValue = ""
+		}
+		req := httptest.NewRequest(http.MethodOptions, "/api/v1/query_range", nil)
+		w := httptest.NewRecorder()
+
+		handlerWrapper(w, req, rh)
+
+		res := w.Result()
+		_ = res.Body.Close()
+
+		if res.StatusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code; (-%d;+%d)", http.StatusNoContent, res.StatusCode)
+		}
+		if handlerCalled {
+			t.Fatalf("request handler must not be called for OPTIONS requests")
+		}
+		for _, h := range headersToCheck {
+			got := res.Header.Get(h)
+			if wantCORSHeaderValue != got {
+				t.Fatalf("unexpected header: %s value: (-%s;+%s)", h, wantCORSHeaderValue, got)
+			}
+		}
+	}
+
+	// CORS disabled
+	f(t, false)
+	// CORS enabled
+	f(t, true)
+}
+
 func TestHandlerWrapper(t *testing.T) {
 	const hstsHeader = "foo"
 	const frameOptionsHeader = "bar"

lib/jwt/jwt.go

@@ -3,12 +3,14 @@ package jwt
 import (
 	"bytes"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
+	"math"
 	"net/http"
-	"slices"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/valyala/fastjson"
 )
 
 const (
@@ -32,8 +34,8 @@ var (
 // Token represents jwt token
 // https://auth0.com/docs/tokens/json-web-tokens
 type Token struct {
-	header             *header
-	body               *body
+	header             header
+	body               body
 	payload, signature []byte
 }
 
@@ -41,55 +43,380 @@ type header struct {
 	Alg string `json:"alg"`
 	Typ string `json:"typ"`
 	Kid string `json:"kid"`
+
+	buf []byte
+	p   *fastjson.Parser
+}
+
+func (h *header) parse(src string) error {
+	var err error
+	h.buf, err = decodeB64(h.buf[:0], src)
+	if err != nil {
+		return err
+	}
+
+	h.p = parserPool.Get()
+	jv, err := h.p.ParseBytes(h.buf)
+	if err != nil {
+		return err
+	}
+	if jv == nil {
+		return fmt.Errorf("unexpected empty json")
+	}
+	if jv.Type() != fastjson.TypeObject {
+		return fmt.Errorf("unexpected non json object {} type: %q", jv.Type())
+	}
+	h.Alg, err = stringFromJSONValue(jv, "alg")
+	if err != nil {
+		return err
+	}
+	h.Typ, err = stringFromJSONValue(jv, "typ")
+	if err != nil {
+		return err
+	}
+	h.Kid, err = stringFromJSONValue(jv, "kid")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *header) reset() {
+	h.Alg = ""
+	h.Typ = ""
+	h.Kid = ""
+
+	h.buf = h.buf[:0]
+	if h.p != nil {
+		parserPool.Put(h.p)
+		h.p = nil
+	}
 }
 
 type body struct {
 	// expired at time unix_ts
 	Exp int64 `json:"exp"`
 	// issued at time unix_ts
-	Iat      int64          `json:"iat"`
-	Jti      string         `json:"jti,omitempty"`
-	Scope    string         `json:"scope,omitempty"`
-	VMAccess *VMAccessClaim `json:"vm_access"`
+	Iat           int64  `json:"iat"`
+	Jti           string `json:"jti,omitempty"`
+	Scope         string `json:"scope,omitempty"`
+	vmAccessClaim VMAccessClaim
+
+	buf []byte
+	p   *fastjson.Parser
+
+	// allClaims holds entire json body
+	// for the HasClaims() method
+	allClaims *fastjson.Value
+
+	// claimsParser holds optional parser for `vm_access` string representation
+	claimsParser *fastjson.Parser
 }
 
-// Labels defines labels added to filters or incoming time series.
-type Labels map[string]string
+func (b *body) parse(src string) error {
 
-// AsExtraLabels - converts labels to label=value pairs.
-func (l Labels) AsExtraLabels() []string {
-	if len(l) == 0 {
-		return nil
+	var err error
+	b.buf, err = decodeB64(b.buf[:0], src)
+	if err != nil {
+		return err
 	}
-	res := make([]string, 0, len(l))
-	for k, v := range l {
-		res = append(res, k+"="+v)
+	b.p = parserPool.Get()
+	jv, err := b.p.ParseBytes(b.buf)
+	if err != nil {
+		return err
 	}
-	// sort for consistent uri.
-	slices.Sort(res)
-	return res
+	if expObject := jv.Get("exp"); expObject != nil {
+		b.Exp, err = expObject.Int64()
+		if err != nil {
+			return fmt.Errorf("cannot parse `exp` field: %w", err)
+		}
+	}
+	if iatObject := jv.Get("iat"); iatObject != nil {
+		b.Iat, err = iatObject.Int64()
+		if err != nil {
+			return fmt.Errorf("cannot parse `iat` field: %w", err)
+		}
+	}
+	vaObject := jv.Get("vm_access")
+	if vaObject == nil {
+		return ErrVMAccessFieldMissing
+	}
+	// some IDPs encode custom claims as a string
+	// try parsing as an object and fallback to a string
+	switch vaObject.Type() {
+	case fastjson.TypeObject:
+		if err := b.vmAccessClaim.parseFrom(vaObject); err != nil {
+			return err
+		}
+	case fastjson.TypeString:
+		b.claimsParser = parserPool.Get()
+		va, err := b.claimsParser.ParseBytes(vaObject.GetStringBytes())
+		if err != nil {
+			return fmt.Errorf("cannot parse `vm_access` string json: %w", err)
+		}
+		if err := b.vmAccessClaim.parseFrom(va); err != nil {
+			return fmt.Errorf("cannot parse `vm_access` values from string json: %w", err)
+		}
+	case fastjson.TypeNull:
+		return ErrVMAccessFieldMissing
+	default:
+		return fmt.Errorf("unexpected type for `vm_access` field; got: %q, want object {}", vaObject.Type())
+	}
+	b.Jti = bytesutil.ToUnsafeString(jv.GetStringBytes("jti"))
+
+	if scopeObject := jv.Get("scope"); scopeObject != nil {
+		// some IDPs encode scope as a string and some as an array
+		switch scopeObject.Type() {
+		case fastjson.TypeString:
+			sb := scopeObject.GetStringBytes()
+			b.Scope = bytesutil.ToUnsafeString(sb)
+		case fastjson.TypeArray:
+			var sizeNeeded int
+			ss := scopeObject.GetArray()
+			for _, v := range ss {
+				sizeNeeded += len(v.GetStringBytes()) + 1
+			}
+			dst := make([]byte, 0, sizeNeeded)
+			for idx, v := range ss {
+				dst = append(dst, v.GetStringBytes()...)
+				if idx < len(ss)-1 {
+					dst = append(dst, ' ')
+				}
+			}
+			b.Scope = bytesutil.ToUnsafeString(dst)
+		default:
+			return fmt.Errorf("unexpected type for `scope` field; got %q, want String or []String", scopeObject.Type())
+		}
+	}
+	b.allClaims = jv
+
+	return nil
 }
 
+func (b *body) reset() {
+	b.Exp = 0
+	b.Iat = 0
+	b.Jti = ""
+	b.Scope = ""
+	b.buf = b.buf[:0]
+	b.allClaims = nil
+	b.vmAccessClaim.reset()
+	if b.p != nil {
+		parserPool.Put(b.p)
+		b.p = nil
+	}
+	if b.claimsParser != nil {
+		parserPool.Put(b.claimsParser)
+		b.claimsParser = nil
+	}
+
+}
+
+// Parse parses JWT token from given source string
+//
+// Token field is valid until src is reachable
+func (t *Token) Parse(src string, enforceAuthPrefix bool) error {
+	if enforceAuthPrefix && (len(src) < len(prefix) || !strings.EqualFold(src[:len(prefix)], prefix)) {
+		return fmt.Errorf("wrong format, prefix: %s is missing", prefix)
+	}
+	// While https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 states that only Bearer prefix is allowed,
+	// it claims to be conformant to the generic syntax defined in https://datatracker.ietf.org/doc/html/rfc2617#section-1.2
+	// which permits case-insensitive auth scheme.
+	// So we should be tolerant to different cases of "Bearer" prefix.
+	if len(src) >= len(prefix) && strings.EqualFold(src[:len(prefix)], prefix) {
+		src = src[len(prefix):]
+	}
+
+	// assume jwt token has the following structure:
+	// header.body.signature
+	var header, body, signature string
+	idx := strings.IndexByte(src, '.')
+	if idx <= 0 {
+		return ErrBadTokenFormat
+	}
+	header = src[:idx]
+	src = src[idx+1:]
+	idx = strings.IndexByte(src, '.')
+	if idx <= 0 {
+		return ErrBadTokenFormat
+	}
+	body = src[:idx]
+	signature = src[idx+1:]
+	if err := t.parse(header, body, signature); err != nil {
+		return err
+	}
+	return nil
+}
+
+// HasClaims checks if Token has all given claim key value pairs
+func (t *Token) HasClaims(claims map[string]string) bool {
+	for k, v := range claims {
+		gotV := t.body.allClaims.Get(k)
+		if gotV == nil || gotV.Type() != fastjson.TypeString {
+			return false
+		}
+		tcv := bytesutil.ToUnsafeString(gotV.GetStringBytes())
+		if tcv != v {
+			return false
+		}
+	}
+
+	return true
+}
+
+// VMAccess return a reference to the VMAccessClaim
+// all data are valid until Token is reachable
+func (t *Token) VMAccess() *VMAccessClaim {
+	return &t.body.vmAccessClaim
+}
+
+// Reset release memory used by token
+// Token cannot be used after this call
+func (t *Token) Reset() {
+	t.header.reset()
+	t.body.reset()
+	t.payload = t.payload[:0]
+	t.signature = t.signature[:0]
+}
+
+// VMAccessClaim represent JWT claim object
 type VMAccessClaim struct {
-	Tenant TenantID `json:"tenant_id"`
-	Labels Labels   `json:"extra_labels,omitempty"`
 	// promql filters applied to each select query
 	ExtraFilters []string `json:"extra_filters,omitempty"`
+
+	MetricsExtraFilters    []string `json:"metrics_extra_filters,omitempty"`
+	MetricsExtraLabels     []string `json:"metrics_extra_labels,omitempty"`
+	LogsExtraFilters       []string `json:"logs_extra_filters,omitempty"`
+	LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty"`
+
+	Labels []string `json:"extra_labels,omitempty"`
+	// labelsBuf holds allocated memory for Labels
+	labelsBuf []byte
+	Tenant    TenantID `json:"tenant_id"`
 	// role can be denied as 1 = read, 2 = write, 3 = read and write
 	// 0 = unconfigured - read and write
 	Mode int `json:"mode,omitempty"`
 
-	// TODO: use different claim struct for vmauth and vmgateway
-	// parsing must be dynamic based on provided hint
-	MetricsAccountID    uint32   `json:"metrics_account_id,omitempty"`
-	MetricsProjectID    uint32   `json:"metrics_project_id,omitempty"`
-	MetricsExtraFilters []string `json:"metrics_extra_filters,omitempty"`
-	MetricsExtraLabels  []string `json:"metrics_extra_labels,omitempty"`
+	MetricsAccountID uint32 `json:"metrics_account_id,omitempty"`
+	MetricsProjectID uint32 `json:"metrics_project_id,omitempty"`
 
-	LogsAccountID          uint32   `json:"logs_account_id,omitempty"`
-	LogsProjectID          uint32   `json:"logs_project_id,omitempty"`
-	LogsExtraFilters       []string `json:"logs_extra_filters,omitempty"`
-	LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty"`
+	LogsAccountID uint32 `json:"logs_account_id,omitempty"`
+	LogsProjectID uint32 `json:"logs_project_id,omitempty"`
+}
+
+func (vac *VMAccessClaim) reset() {
+	vac.Tenant.AccountID = 0
+	vac.Tenant.ProjectID = 0
+	clear(vac.Labels)
+	vac.Labels = vac.Labels[:0]
+	vac.labelsBuf = vac.labelsBuf[:0]
+	clear(vac.ExtraFilters)
+	vac.ExtraFilters = vac.ExtraFilters[:0]
+	vac.Mode = 0
+
+	vac.MetricsAccountID = 0
+	vac.MetricsProjectID = 0
+	clear(vac.MetricsExtraFilters)
+	vac.MetricsExtraFilters = vac.MetricsExtraFilters[:0]
+	clear(vac.MetricsExtraLabels)
+	vac.MetricsExtraLabels = vac.MetricsExtraLabels[:0]
+	vac.LogsAccountID = 0
+	vac.LogsProjectID = 0
+	clear(vac.LogsExtraFilters)
+	vac.LogsExtraFilters = vac.LogsExtraFilters[:0]
+	clear(vac.LogsExtraStreamFilters)
+	vac.LogsExtraStreamFilters = vac.LogsExtraStreamFilters[:0]
+}
+
+func (vac *VMAccessClaim) parseFrom(jv *fastjson.Value) error {
+
+	if err := vac.Tenant.parseFrom(jv); err != nil {
+		return err
+	}
+
+	var err error
+	vac.ExtraFilters, err = stringSliceFromJSONValue(vac.ExtraFilters, jv, "extra_filters")
+	if err != nil {
+		return err
+	}
+	efs := jv.Get("extra_labels")
+	if efs != nil {
+		efsO, err := efs.Object()
+		if err != nil {
+			return fmt.Errorf("cannot parse `extra_labels` field: %w", err)
+		}
+		buf := vac.labelsBuf[:0]
+		var visitErr error
+		efsO.Visit(func(key []byte, v *fastjson.Value) {
+			if visitErr != nil {
+				return
+			}
+			vs, err := v.StringBytes()
+			if err != nil {
+				visitErr = fmt.Errorf("unexpected value for key=%q: %w", string(key), err)
+			}
+			start := len(buf)
+			sizeNeeded := len(key) + 1 + len(vs)
+			if len(buf)+sizeNeeded >= cap(buf) {
+				// allocate new slice without memory fragmentation
+				// old slice will be referenced by vac.Labels
+				start = 0
+				buf = make([]byte, 0, len(buf)+sizeNeeded)
+			}
+			buf = append(buf, key...)
+			buf = append(buf, '=')
+			buf = append(buf, vs...)
+			ef := bytesutil.ToUnsafeString(buf[start:])
+			vac.Labels = append(vac.Labels, ef)
+		})
+		vac.labelsBuf = buf
+		if visitErr != nil {
+			return fmt.Errorf("cannot parse `extra_labels` field: %w", visitErr)
+		}
+	}
+	mode := jv.Get("mode")
+	if mode != nil {
+		vac.Mode, err = mode.Int()
+		if err != nil {
+			return fmt.Errorf("unexpected `mode` value: %w", err)
+		}
+	}
+	vac.MetricsAccountID, err = uint32FromJSONValue(jv, "metrics_account_id")
+	if err != nil {
+		return err
+	}
+	vac.MetricsProjectID, err = uint32FromJSONValue(jv, "metrics_project_id")
+	if err != nil {
+		return err
+	}
+
+	vac.MetricsExtraFilters, err = stringSliceFromJSONValue(vac.MetricsExtraFilters, jv, "metrics_extra_filters")
+	if err != nil {
+		return err
+	}
+	vac.MetricsExtraLabels, err = stringSliceFromJSONValue(vac.MetricsExtraLabels, jv, "metrics_extra_labels")
+	if err != nil {
+		return err
+	}
+	vac.LogsAccountID, err = uint32FromJSONValue(jv, "logs_account_id")
+	if err != nil {
+		return err
+	}
+	vac.LogsProjectID, err = uint32FromJSONValue(jv, "logs_project_id")
+	if err != nil {
+		return err
+	}
+	vac.LogsExtraFilters, err = stringSliceFromJSONValue(vac.LogsExtraFilters, jv, "logs_extra_filters")
+	if err != nil {
+		return err
+	}
+	vac.LogsExtraStreamFilters, err = stringSliceFromJSONValue(vac.LogsExtraStreamFilters, jv, "logs_extra_stream_filters")
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // TenantID represents tenantID.
@@ -98,12 +425,33 @@ type TenantID struct {
 	AccountID int32 `json:"account_id"`
 }
 
+func (tid *TenantID) parseFrom(jv *fastjson.Value) error {
+	tidObject := jv.Get("tenant_id")
+	if tidObject == nil {
+		return nil
+	}
+	var err error
+	tid.AccountID, err = int32FromJSONValue(tidObject, "account_id")
+	if err != nil {
+		return err
+	}
+	tid.ProjectID, err = int32FromJSONValue(tidObject, "project_id")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // String implements interface.
 func (tid TenantID) String() string {
 	return fmt.Sprintf("%d:%d", tid.AccountID, tid.ProjectID)
 }
 
 // NewToken creates token from raw string.
+//
+// Deprecated: allocates a new Token on every call.
+// Prefer acquiring a Token from a sync.Pool, calling t.Parse(), and returning it after use.
 func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
 	if enforceAuthPrefix && (len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix)) {
 		return nil, fmt.Errorf("wrong format, prefix: %s is missing", prefix)
@@ -122,10 +470,16 @@ func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
 		return nil, ErrBadTokenFormat
 	}
 	var t Token
-	return t.parse(jwt[0], jwt[1], jwt[2])
+	if err := t.parse(jwt[0], jwt[1], jwt[2]); err != nil {
+		return nil, err
+	}
+	return &t, nil
 }
 
 // NewTokenFromRequestWithCustomHeader return new jwt token from request by provided header
+//
+// Deprecated: allocates a new Token on every call.
+// Prefer acquiring a Token from a sync.Pool, calling t.Parse(), and returning it after use.
 func NewTokenFromRequestWithCustomHeader(r *http.Request, headerName string, enforceAuthPrefix bool) (*Token, error) {
 	auth := r.Header.Get(headerName)
 	if len(auth) == 0 {
@@ -134,28 +488,25 @@ func NewTokenFromRequestWithCustomHeader(r *http.Request, headerName string, enf
 	return NewToken(auth, enforceAuthPrefix)
 }
 
-func (t *Token) parse(header, body, signature string) (*Token, error) {
-	b, err := parseJWTBody(body)
-	if err != nil {
-		return nil, err
+func (t *Token) parse(header, body, signature string) error {
+	if err := t.body.parse(body); err != nil {
+		return fmt.Errorf("cannot parse token body: %w", err)
 	}
-	if b.VMAccess == nil {
-		return nil, ErrVMAccessFieldMissing
-	}
-	t.body = b
-	h, err := parseJWTHeader(header)
-	if err != nil {
-		return nil, err
-	}
-	t.header = h
-
-	t.payload = []byte(header + "." + body)
-	t.signature, err = decodeB64([]byte(signature))
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode signature as b64: %w", err)
+	if err := t.header.parse(header); err != nil {
+		return fmt.Errorf("cannot parse token header: %w", err)
 	}
 
-	return t, nil
+	t.payload = bytesutil.ResizeNoCopyNoOverallocate(t.payload, len(header)+len(body)+1)
+	t.payload = append(t.payload[:0], header...)
+	t.payload = append(t.payload, '.')
+	t.payload = append(t.payload, body...)
+	var err error
+	t.signature, err = decodeB64(t.signature[:0], signature)
+	if err != nil {
+		return fmt.Errorf("cannot decode token signature: %w", err)
+	}
+
+	return nil
 }
 
 // IsExpired checks if jwt token is expired.
@@ -166,10 +517,10 @@ func (t *Token) IsExpired(currentTime time.Time) bool {
 // CanWrite checks if token has write permissions.
 func (t *Token) CanWrite() bool {
 	// unconfigured
-	if t.body.VMAccess.Mode == 0 {
+	if t.body.vmAccessClaim.Mode == 0 {
 		return true
 	}
-	if write&t.body.VMAccess.Mode > 0 {
+	if write&t.body.vmAccessClaim.Mode > 0 {
 		return true
 	}
 	return false
@@ -178,10 +529,10 @@ func (t *Token) CanWrite() bool {
 // CanRead check if token has read permissions.
 func (t *Token) CanRead() bool {
 	// unconfigured
-	if t.body.VMAccess.Mode == 0 {
+	if t.body.vmAccessClaim.Mode == 0 {
 		return true
 	}
-	if read&t.body.VMAccess.Mode > 0 {
+	if read&t.body.vmAccessClaim.Mode > 0 {
 		return true
 	}
 	return false
@@ -189,101 +540,36 @@ func (t *Token) CanRead() bool {
 
 // AccessLabels returns vm_access labels for given JWT token,
 // in key=value format.
+//
+// Returned value is only valid until Token is reachable
 func (t *Token) AccessLabels() []string {
-	return t.body.VMAccess.Labels.AsExtraLabels()
+	return t.body.vmAccessClaim.Labels
 }
 
 // Tenant returns tenantID for token.
 func (t *Token) Tenant() TenantID {
-	return t.body.VMAccess.Tenant
+	return t.body.vmAccessClaim.Tenant
 }
 
 // ExtraFilters metricsql filters for select queries
+//
+// Returned value is only valid until Token is reachable
 func (t *Token) ExtraFilters() []string {
-	return t.body.VMAccess.ExtraFilters
+	return t.body.vmAccessClaim.ExtraFilters
 }
 
-func (t *Token) VMAccess() *VMAccessClaim {
-	return t.body.VMAccess
-}
-
-func parseJWTHeader(data string) (*header, error) {
-	var jh header
-	decoded, err := decodeB64([]byte(data))
-	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt header as b64: %w", err)
-	}
-	if err := json.Unmarshal(decoded, &jh); err != nil {
-		return nil, fmt.Errorf("cannot parse jwt header: %w", err)
-	}
-	return &jh, nil
-}
-
-func parseJWTBody(data string) (*body, error) {
-	type tbody struct {
-		// expired at time unix_ts
-		Exp int64 `json:"exp"`
-		// issued at time unix_ts
-		Iat   int64           `json:"iat"`
-		Jti   string          `json:"jti,omitempty"`
-		Scope json.RawMessage `json:"scope,omitempty"`
-		// store as raw message to support different types
-		VMAccess *json.RawMessage `json:"vm_access"`
-	}
-	var tb tbody
-
-	decoded, err := decodeB64([]byte(data))
-	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
-	}
-	if err := json.Unmarshal(decoded, &tb); err != nil {
-		return nil, fmt.Errorf("cannot parse jwt body: %w", err)
-	}
-
-	if tb.VMAccess == nil {
-		return nil, ErrVMAccessFieldMissing
-	}
-
-	// some IDPs encode custom claims as a string
-	// try parsing as an object and fallback to a string
-	var a VMAccessClaim
-	if err := json.Unmarshal(*tb.VMAccess, &a); err != nil {
-		var s string
-		if err := json.Unmarshal(*tb.VMAccess, &s); err != nil {
-			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
-		}
-
-		if err := json.Unmarshal([]byte(s), &a); err != nil {
-			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
-		}
-	}
-
-	// some IDPs encode scope as a string and some as an array
-	var scope string
-	if tb.Scope != nil {
-		if err := json.Unmarshal(tb.Scope, &scope); err != nil {
-			var scopeSlice []string
-			if err := json.Unmarshal(tb.Scope, &scopeSlice); err != nil {
-				return nil, fmt.Errorf("cannot parse jwt body scope: %w", err)
-			}
-			scope = strings.Join(scopeSlice, " ")
-		}
-	}
-
-	parsedBody := &body{
-		Exp:      tb.Exp,
-		Iat:      tb.Iat,
-		Jti:      tb.Jti,
-		Scope:    scope,
-		VMAccess: &a,
-	}
-	return parsedBody, nil
-}
-
-func decodeB64(data []byte) ([]byte, error) {
+func decodeB64(dst []byte, src string) ([]byte, error) {
+	data := bytesutil.ToUnsafeBytes(src)
 	idx := bytes.IndexAny(data, "+/")
 	// slow path, std base64 encoding convert it to url encoding
+	// it could be encoded with standard Base64 (+/) instead of Base64URL (-_).
 	if idx >= 0 {
+		// make a copy of provided input, src cannot be modified by parser
+		bb := decodeb64BufferPool.Get()
+		defer decodeb64BufferPool.Put(bb)
+		b := bb.B[:0]
+		b = append(b, data...)
+		data = b
 		for idx, c := range data {
 			switch c {
 			case '+':
@@ -292,12 +578,94 @@ func decodeB64(data []byte) ([]byte, error) {
 				data[idx] = '_'
 			}
 		}
-
 	}
-	dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(data)))
+	dst = bytesutil.ResizeNoCopyNoOverallocate(dst, base64.RawURLEncoding.DecodedLen(len(data)))
 	_, err := base64.RawURLEncoding.Decode(dst, data)
 	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
+		return nil, err
 	}
 	return dst, nil
 }
+
+// stringFromJSONValue is a helper with missing String parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func stringFromJSONValue(jv *fastjson.Value, key string) (string, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return "", nil
+	}
+	b, err := jvInner.StringBytes()
+	if err != nil {
+		return "", fmt.Errorf("unexpected non-string value for key=%q: %w", key, err)
+	}
+
+	return bytesutil.ToUnsafeString(b), nil
+}
+
+// uint32FromJSONValue is a helper for missing Uint32 parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func uint32FromJSONValue(jv *fastjson.Value, key string) (uint32, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return 0, nil
+	}
+	u64, err := jvInner.Uint64()
+	if err != nil {
+		return 0, fmt.Errorf("unexpected non-uint32 value for key=%q: %w", key, err)
+	}
+	if u64 > math.MaxUint32 {
+		return 0, fmt.Errorf("value cannot exceed uint32 for key=%q", key)
+	}
+
+	return uint32(u64), nil
+}
+
+// int32FromJSONValue is a helper for missing Int32 parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func int32FromJSONValue(jv *fastjson.Value, key string) (int32, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return 0, nil
+	}
+	i64, err := jvInner.Int64()
+	if err != nil {
+		return 0, fmt.Errorf("unexpected non-int32 value for key=%q: %w", key, err)
+	}
+	if i64 > math.MaxInt32 || i64 < math.MinInt32 {
+		return 0, fmt.Errorf("value cannot exceed int32 for key=%q", key)
+	}
+
+	return int32(i64), nil
+}
+
+// stringSliceFromJSONValue is a helper for missing StringArray parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func stringSliceFromJSONValue(dst []string, jv *fastjson.Value, key string) ([]string, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return dst, nil
+	}
+	if jvInner.Type() != fastjson.TypeArray {
+		return nil, fmt.Errorf("unexpected type for key=%q, got: %s, want: array string", key, jvInner.Type())
+	}
+	for _, ef := range jvInner.GetArray() {
+		if ef == nil {
+			continue
+		}
+		efs, err := ef.StringBytes()
+		if err != nil {
+			return nil, fmt.Errorf("unexpected non string array[] type for key=%q: %w", key, err)
+		}
+		dst = append(dst, bytesutil.ToUnsafeString(efs))
+
+	}
+	return dst, nil
+}
+
+var parserPool fastjson.ParserPool
+
+var decodeb64BufferPool bytesutil.ByteBufferPool

lib/jwt/jwt_test.go

@@ -17,54 +17,97 @@ func TestParseJWTHeader_Failure(t *testing.T) {
 			base64.RawURLEncoding.Encode(encoded, []byte(data))
 			data = string(encoded)
 		}
-		if _, err := parseJWTHeader(data); err != nil {
+		var h header
+		if err := h.parse(data); err != nil {
 			if err.Error() != expectedErr {
-				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+				t.Fatalf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
 			}
 		} else {
-			t.Errorf("expecting non-nil error")
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid input
 	f(
 		`bad input`,
-		`cannot decode jwt header as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		`illegal base64 data at input byte 3`,
 		false,
 	)
 
 	// invalid b644
 	f(
 		`YmFk`,
-		`cannot parse jwt header: invalid character 'b' looking for beginning of value`,
+		`cannot parse JSON: cannot parse number: unexpected char: "b"; unparsed tail: "bad"`,
 		false,
 	)
 
 	// invalid header json
 	f(`{]`,
-		`cannot parse jwt header: invalid character ']' looking for beginning of object key string`,
+		`cannot parse JSON: cannot parse object: cannot find opening '"" for object key; unparsed tail: "]"`,
 		true,
 	)
 
 	// invalid header type json
 	f(`[]`,
-		`cannot parse jwt header: json: cannot unmarshal array into Go value of type jwt.header`,
+		`unexpected non json object {} type: "array"`,
+		true,
+	)
+
+	// alg field is not a string
+	f(
+		`{"alg": 123, "typ": "JWT", "kid": "key-1"}`,
+		`unexpected non-string value for key="alg": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// typ field is not a string
+	f(
+		`{"alg": "RS256", "typ": 123, "kid": "key-1"}`,
+		`unexpected non-string value for key="typ": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// kid field is not a string
+	f(
+		`{"alg": "RS256", "typ": "JWT", "kid": 123}`,
+		`unexpected non-string value for key="kid": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// standard Base64 with + character (slow path in decodeB64)
+	f(
+		`{"alg": "RS256", "typ": "JWT/"}`,
+		`illegal base64 data at input byte 0`,
+		false,
+	)
+
+	// invalid header type json
+	f(`[]`,
+		`unexpected non json object {} type: "array"`,
 		true,
 	)
 }
 
 func TestParseJWTHeader_Success(t *testing.T) {
-	f := func(data string, expected *header) {
+	f := func(data string, expected header) {
 		t.Helper()
 		encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
 		encoded := make([]byte, encodedLen)
 		base64.RawURLEncoding.Encode(encoded, []byte(data))
-		header, err := parseJWTHeader(string(encoded))
+		var h header
+		err := h.parse(string(encoded))
 		if err != nil {
 			t.Fatalf("parseJWTHeader() error: %s", err)
 		}
-		if !reflect.DeepEqual(header, expected) {
-			t.Fatalf("unexpected token header;\ngot\n%v\nwant\n%v", header, expected)
+
+		if h.Alg != expected.Alg {
+			t.Fatalf("unexpected Alg:\ngot\n%s\nwant\n%s", h.Alg, expected.Alg)
+		}
+		if h.Typ != expected.Typ {
+			t.Fatalf("unexpected Typ:\ngot\n%s\nwant\n%s", h.Typ, expected.Typ)
+		}
+		if h.Kid != expected.Kid {
+			t.Fatalf("unexpected Kid:\ngot\n%s\nwant\n%s", h.Kid, expected.Kid)
 		}
 	}
 
@@ -77,7 +120,7 @@ func TestParseJWTHeader_Success(t *testing.T) {
 			"alg": %q,
 			"kid": "test"
 		}`, supportedAlgorithms[i]),
-			&header{
+			header{
 				Alg: supportedAlgorithms[i],
 				Kid: "test",
 			},
@@ -94,40 +137,41 @@ func TestParseJWTBody_Failure(t *testing.T) {
 			base64.RawURLEncoding.Encode(encoded, []byte(data))
 			data = string(encoded)
 		}
-		if _, err := parseJWTBody(data); err != nil {
+		var b body
+		if err := b.parse(data); err != nil {
 			if err.Error() != expectedErr {
-				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+				t.Fatalf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
 			}
 		} else {
-			t.Errorf("expecting non-nil error")
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid input
 	f(
 		`bad input`,
-		`cannot decode jwt body as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		`illegal base64 data at input byte 3`,
 		false,
 	)
 
 	// invalid b644
 	f(
 		`YmFk`,
-		`cannot parse jwt body: invalid character 'b' looking for beginning of value`,
+		`cannot parse JSON: cannot parse number: unexpected char: "b"; unparsed tail: "bad"`,
 		false,
 	)
 
 	// invalid body json
 	f(
 		`{]`,
-		`cannot parse jwt body: invalid character ']' looking for beginning of object key string`,
+		`cannot parse JSON: cannot parse object: cannot find opening '"" for object key; unparsed tail: "]"`,
 		true,
 	)
 
 	// invalid body type json
 	f(
 		`[]`,
-		`cannot parse jwt body: json: cannot unmarshal array into Go value of type jwt.tbody`,
+		"missing `vm_access` claim",
 		true,
 	)
 
@@ -141,7 +185,7 @@ func TestParseJWTBody_Failure(t *testing.T) {
 	// vm_access claim invalid type
 	f(
 		`{"vm_access": 123}`,
-		"cannot parse jwt body vm_access: json: cannot unmarshal number into Go value of type string",
+		"unexpected type for `vm_access` field; got: \"number\", want object {}",
 		true,
 	)
 
@@ -155,14 +199,14 @@ func TestParseJWTBody_Failure(t *testing.T) {
 	// invalid vm_access: account_id type mismatch
 	f(
 		`{"vm_access": {"tenant_id": {"account_id": "1", "project_id": 5}}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-int32 value for key="account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid vm_access: project_id type mismatch
 	f(
 		`{"vm_access": {"tenant_id": {"account_id": 1, "project_id": "5"}}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-int32 value for key="project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
@@ -180,7 +224,7 @@ func TestParseJWTBody_Failure(t *testing.T) {
 		}
 	}
 }`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		"cannot parse `extra_labels` field: value doesn't contain object; it contains array",
 		true,
 	)
 
@@ -195,70 +239,70 @@ func TestParseJWTBody_Failure(t *testing.T) {
 		}
 	}
 }`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non string array[] type for key="extra_filters": value doesn't contain string; it contains object`,
 		true,
 	)
 
 	// invalid exp claim value type
 	f(
 		`{"exp": "1610976189", "vm_access": {}}`,
-		`cannot parse jwt body: json: cannot unmarshal string into Go struct field tbody.exp of type int64`,
+		"cannot parse `exp` field: value doesn't contain number; it contains string",
 		true,
 	)
 
 	// invalid metrics metrics_account_id claim value type
 	f(
 		`{"vm_access": {"metrics_account_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="metrics_account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics metrics_project_id claim value type
 	f(
 		`{"vm_access": {"metrics_project_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="metrics_project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics metrics_extra_labels claim value type
 	f(
 		`{"vm_access": {"metrics_extra_labels": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="metrics_extra_labels", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics metrics_extra_filters claim value type
 	f(
 		`{"vm_access": {"metrics_extra_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="metrics_extra_filters", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics logs_account_id claim value type
 	f(
 		`{"vm_access": {"logs_account_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="logs_account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics logs_project_id claim value type
 	f(
 		`{"vm_access": {"logs_project_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="logs_project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics logs_extra_filters claim value type
 	f(
 		`{"vm_access": {"logs_extra_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="logs_extra_filters", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics logs_extra_stream_filters claim value type
 	f(
 		`{"vm_access": {"logs_extra_stream_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="logs_extra_stream_filters", got: string, want: array string`,
 		true,
 	)
 }
@@ -271,7 +315,8 @@ func TestParseJWTBody_Success(t *testing.T) {
 		encoded := make([]byte, encodedLen)
 		base64.RawURLEncoding.Encode(encoded, []byte(data))
 
-		result, err := parseJWTBody(string(encoded))
+		var result body
+		err := result.parse(string(encoded))
 		if err != nil {
 			t.Fatalf("parseJWTBody() error: %s", err)
 		}
@@ -287,22 +332,22 @@ func TestParseJWTBody_Success(t *testing.T) {
 		if result.Jti != resultExpected.Jti {
 			t.Fatalf("unexpected jti; got %q; want %q", result.Jti, resultExpected.Jti)
 		}
-		if !reflect.DeepEqual(result.VMAccess.Tenant, resultExpected.VMAccess.Tenant) {
-			t.Fatalf("unexpected tenant; got %v; want %v", result.VMAccess.Tenant, resultExpected.VMAccess.Tenant)
+		if !reflect.DeepEqual(result.vmAccessClaim.Tenant, resultExpected.vmAccessClaim.Tenant) {
+			t.Fatalf("unexpected tenant; got %v; want %v", result.vmAccessClaim.Tenant, resultExpected.vmAccessClaim.Tenant)
 		}
-		if !reflect.DeepEqual(result.VMAccess.Labels, resultExpected.VMAccess.Labels) {
-			t.Fatalf("unexpected labels; got %v; want %v", result.VMAccess.Labels, resultExpected.VMAccess.Labels)
+		if !reflect.DeepEqual(result.vmAccessClaim.Labels, resultExpected.vmAccessClaim.Labels) {
+			t.Fatalf("unexpected labels; got %v; want %v", result.vmAccessClaim.Labels, resultExpected.vmAccessClaim.Labels)
 		}
-		if !reflect.DeepEqual(result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters) {
-			t.Fatalf("unexpected extra_filters; got %v; want %v", result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters)
+		if !reflect.DeepEqual(result.vmAccessClaim.ExtraFilters, resultExpected.vmAccessClaim.ExtraFilters) {
+			t.Fatalf("unexpected extra_filters; got %v; want %v", result.vmAccessClaim.ExtraFilters, resultExpected.vmAccessClaim.ExtraFilters)
 		}
 	}
 
 	f(`{"vm_access": {}}`, &body{
-		VMAccess: &VMAccessClaim{},
+		vmAccessClaim: VMAccessClaim{},
 	})
 	f(`{"vm_access": {"tenant_id": {}}}`, &body{
-		VMAccess: &VMAccessClaim{},
+		vmAccessClaim: VMAccessClaim{},
 	})
 
 	f(
@@ -316,7 +361,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
@@ -336,10 +381,10 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
-				Labels: Labels{
-					"project": "dev",
-					"team":    "mobile",
+			vmAccessClaim: VMAccessClaim{
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
@@ -356,7 +401,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				ExtraFilters: []string{
 					`{project="dev"}`,
 					`{team=~"mobile"}`,
@@ -384,14 +429,14 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: Labels{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{
 					`{project="dev"}`,
@@ -411,11 +456,27 @@ func TestParseJWTBody_Success(t *testing.T) {
     "vm_access": {}
 }`,
 		&body{
-			Exp:      1610976189,
-			Iat:      1610975889,
-			Jti:      "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
-			Scope:    "openid email profile",
-			VMAccess: &VMAccessClaim{},
+			Exp:   1610976189,
+			Iat:   1610975889,
+			Jti:   "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+			Scope: "openid email profile",
+		},
+	)
+	// scope as []string
+	f(
+		`
+{
+    "exp": 1610976189,
+    "iat": 1610975889,
+    "jti": "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+    "scope": ["openid","email","profile"],
+    "vm_access": {}
+}`,
+		&body{
+			Exp:   1610976189,
+			Iat:   1610975889,
+			Jti:   "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+			Scope: "openid email profile",
 		},
 	)
 
@@ -436,7 +497,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				MetricsAccountID: 1,
 				MetricsProjectID: 5,
 				MetricsExtraLabels: []string{
@@ -466,7 +527,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				LogsAccountID: 1,
 				LogsProjectID: 5,
 				LogsExtraFilters: []string{
@@ -521,8 +582,18 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewTokenFromRequest() error: %s", err)
 		}
-		if !reflect.DeepEqual(result.body.VMAccess, resultExpected.body.VMAccess) {
-			t.Fatalf("unexpected token body VMAccess;\ngot\n%v\nwant\n%v", result.body.VMAccess, resultExpected.body.VMAccess)
+		// assign nil values to simplify equal check below
+		result.header.buf = nil
+		result.header.p = nil
+		result.body.vmAccessClaim.labelsBuf = nil
+		if result.body.Iat != resultExpected.body.Iat {
+			t.Fatalf("unexpected iat: %d;%d", result.body.Iat, resultExpected.body.Iat)
+		}
+		if result.body.Exp != resultExpected.body.Exp {
+			t.Fatalf("unexpected exp: %d;%d", result.body.Exp, resultExpected.body.Exp)
+		}
+		if !reflect.DeepEqual(result.body.vmAccessClaim, resultExpected.body.vmAccessClaim) {
+			t.Fatalf("unexpected token body VMAccess;\ngot\n%v\nwant\n%v", result.body.vmAccessClaim, resultExpected.body.vmAccessClaim)
 		}
 		if !reflect.DeepEqual(result.header, resultExpected.header) {
 			t.Fatalf("unexpected token header\ngot\n%v\nwant\n%v", result.header, resultExpected.header)
@@ -538,23 +609,23 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected := &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -571,23 +642,23 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -604,8 +675,10 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			VMAccess: &VMAccessClaim{
+		body: body{
+			Iat: 1645536638,
+			Exp: 1645536758,
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 0,
 					AccountID: 1,
@@ -616,7 +689,7 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 				Mode: 1,
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Typ: "JWT",
 		},
@@ -632,8 +705,10 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			VMAccess: &VMAccessClaim{
+		body: body{
+			Iat: 1645606878,
+			Exp: 1645606998,
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 0,
 					AccountID: 1,
@@ -644,7 +719,7 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 				Mode: 1,
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Typ: "JWT",
 		},
@@ -660,24 +735,24 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -694,24 +769,24 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -729,17 +804,17 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 	}
 
 	resultExpected = &Token{
-		body: &body{
+		body: body{
 			Exp: 1725629232,
 			Iat: 1725625332,
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
 			Typ: "JWT",
@@ -757,17 +832,17 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 	}
 
 	resultExpected = &Token{
-		body: &body{
+		body: body{
 			Exp: 1725629232,
 			Iat: 1725625332,
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
 			Typ: "JWT",

lib/jwt/jwt_timing_test.go

@@ -0,0 +1,38 @@
+package jwt
+
+import "testing"
+
+func BenchmarkTokenParse(t *testing.B) {
+	f := func(name string, rawToken string) {
+		t.Helper()
+
+		t.Run(name, func(t *testing.B) {
+			t.ReportAllocs()
+			t.RunParallel(func(pb *testing.PB) {
+				var jt Token
+				for pb.Next() {
+					jt.Reset()
+					if err := jt.Parse(rawToken, true); err != nil {
+						t.Fatalf("unexpected parsing error: %s", err)
+					}
+				}
+			})
+
+		})
+	}
+
+	// simple token with only tenant_id
+	f("simple", `Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxOTAsImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9fSwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoidGcgdGciLCJwcm9qZWN0IjoibW9iaWxlIiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGciLCJ0ZWFtIjoiZGV2IiwiZ2l2ZW5fbmFtZSI6InRnIiwiZmFtaWx5X25hbWUiOiJ0ZyIsImVtYWlsIjoidGdAZmdodC5uZXQifQ.mpT7_kGOIZtoRv2Tn-_80YXmy7_3Qc4_xeaQr1Nhk4UyXSeWh6HB96wWkBS8Jhj3NksGj7bqxezOEbOBBaqlYn6cdGV2hVZ8GKT2zt6oRLCuuUORiRU1joBeIhVRMNtXvPXLFTs4e1VIKejncWbeKmXSneYCjJityixQza0mVyO7ldiXHc6J2f_wQJDPkwkFJJvfwwTbyu4maUzv5gNIvVSUfWnjPq3skFmnjwpsfD9KZnZg-pPTKUmri6kdK0YrFTGA5HT_DM77UkXzsDSMdHPP5tgiPD3LeK75djTZdMAidX53ai85BDn9d5vzi9nfoVezyN3dh0xqqaaQJGqjng`)
+
+	// gateway extra labels and extra filters
+	f("gateway labels and filters", `Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxOTAsImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9LCJleHRyYV9sYWJlbHMiOnsiZW52IjoicHJvZCIsInRlYW0iOiJvcHMifSwiZXh0cmFfZmlsdGVycyI6WyJtZXRyaWMiLCJ7c2VsZWN0b3I9XCJ2YWx1ZVwiIl19LCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsIm5hbWUiOiJ0ZyB0ZyIsInByb2plY3QiOiJtb2JpbGUiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJ0ZyIsInRlYW0iOiJkZXYiLCJnaXZlbl9uYW1lIjoidGciLCJmYW1pbHlfbmFtZSI6InRnIiwiZW1haWwiOiJ0Z0BmZ2h0Lm5ldCJ9.lUEn5nVQ6Trra-9YkbMKyhL0eiWmL2VSIKj2HDQSH43ZkeagLPQbPTnLYfkbuc1sI9tPcyFOPuwgdEAkckEgQ7szvw9g5bLrtT4etWoOnPJ1GaQpcn0z16w7bAgbMf8rpb0i4JMOXicRd7ARlkjyJZDjehaVUX726052qv2NG7npShafK0wei1QBpD3N34TJlqixbOnD1DCfsorwxzba8OuwgQI8lfTHWmgFO0611DGKZb1a-srTPZ5ziZ29NhtDAkbx6bZnYHMp_8CTLD6p0z34RM2wPWyI_2_AdKDbqkdDSZoapJneQDdoNsmMA0IUFETqBgfRTavnApkgeu12HA`)
+
+	// scope as []string
+	f("scope as slice string", `Bearer ewogICJ0eXAiOiJKV1QiLAogICJhbGciOiJSUzI1NiIsCiAgImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyIKfQ.ewogICJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLAogICJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLAogICJpYXQiOjE3MjU2MjUzMzIsCiAgIm5iZiI6MTcyNTYyNTMzMiwKICAiZXhwIjoxNzI1NjI5MjMyLAogICJuYW1lIjoiWmFraGFyIEJlc3NhcmFiIiwKICAib2lkIjoiOGI5ZWY2YjMtMWMwMS00YjczLTg0ODItMjRkNmI2NTE1Y2U0IiwKICAicHJlZmVycmVkX3VzZXJuYW1lIjoiei5iZXNzYXJhYkB2aWN0b3JpYW1ldHJpY3MuY29tIiwKICAicmgiOiIwLkFXTUJ6aDdhSlpGbWFFaW5leHNQbTc1ZlEtY1JjM3AtdWw1Sm1WNnZVMDV6d1JCakFaby4iLAogICJzdWIiOiJXRld3QTlYZjZpZXUxLUgwNDBuU0QxRVo3UWxOLTVHbWxob2p4czdMUFJRIiwKICAidGlkIjoiMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzIiwKICAidXRpIjoidlo1MjQySmhNVWFUUktaYVFCRjhBQSIsCiAgInZlciI6IjIuMCIsCiAgInZtX2FjY2VzcyI6IntcInRlbmFudF9pZFwiOntcInByb2plY3RfaWRcIjogNSwgXCJhY2NvdW50X2lkXCI6IDF9fSIsCiAgInNjb3BlIjogWyJvcGVuaWQiLCAidm0iXQp9.ZXdvZ0lDSjBlWEFpT2lKS1YxUWlMQW9nSUNKaGJHY2lPaUpTVXpJMU5pSXNDaUFnSW10cFpDSTZJa2c1Ym1vMVFVOVRjM2ROY0dobk1WTkdlRGRxWVZZdGJFSTVkeUlLZlEuLktrUG9qNWJoaDNWcnRyY3RVb0lHaE5vN2hNc2VGT3hESGVEQ2g3MFViV2l2LU5pb1Zia2duZk1CMkhacHN6WGU5WmNmX2FIaURJSVNTYkNTaDlvQnF1aS02OEJDcmplNFJWRkpGZFV6R3V1SmdOTS11YVpBcFJqSFNNZDUxb2RvbHFoUGFHS09URnJXVmlIWlpfVDdXaVNUcV84U3Y1a2x1Y2xMb0hEcU82MU5Na2w0TmRCVnQxM1hjRTBfM243U3VxTDdpaks2dGMwZ2NzcmJ5c3JNdl9jd2VRamZsLU5fV0N0SG40NnhadEhvX0RpZERabzc2TjV1NE52Uk1OZUxNcXZ0YTgzUzhPdzNyUUlhaUFjUUNHYjBqUU5hV2VEQlFzZUZ6SjRyR0h6RjAwZDlqVkNCSHVWRmI5eHNnSnJVUDZ0S05iT2hTeEY1RzBocElVYk5OUQ`)
+
+	// vm_access string
+	f("access claim string", `Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyJ9.eyJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLCJpc3MiOiJodHRwczovLzFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLCJpYXQiOjE3MjU2MjUzMzIsIm5iZiI6MTcyNTYyNTMzMiwiZXhwIjoxNzI1NjI5MjMyLCJvaWQiOiIwMDAwMC0xYzAxLTRiNzMtODQ4Mi0yNGQ2YjY1MTVjZTQiLCJ0aWQiOiIwMC02NjkxLTQ4NjgtYTc3Yi0xYjBmOWJiZTVmNDMiLCJ1dGkiOiJ2WjUyNDJKaE1VYVRSS1phUUJGOEFBIiwidmVyIjoiMi4wIiwidm1fYWNjZXNzIjoie1widGVuYW50X2lkXCI6e1wicHJvamVjdF9pZFwiOiA1LCBcImFjY291bnRfaWRcIjogMX19In0.RYYL-Ct-a3dlToRCemUCDbnY_HIFeJ1Feqzj6yXcchy_VtE0DjGu-qGspwPHsJe_JlgHSegN_wSlCLAuorO4vQxIVYansL-6AOQ8fiAh_HRA1dID6lvmxYIkCxNFIEyc7ufp7QJYZiyT_lKJkDOrXqWuJ5l_ajLVRSGK1kWRL0V_e6BsU8-2NF_f1gkPEpULooHmQfpdNszZwPpN_Hyd24gQmSbTZk1MA1jkuo6LLuMDyZK2UDnRQA3Xx480LYnl-VzlBLwv5fwEGFwOJC_E9olvAJxr8eYJEQA4lwsdpwmfJkWBlrdcOZNHzmNaTWMFxmDIBOirH-CUm9ndF2r-Og`)
+
+	// vmauth related claim fields
+	f("vmauth related fields", `Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyJ9.eyJhdWQiOiIwMDAwMC1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdC92Mi4wIiwiaWF0IjoxNzI1NjI1MzMyLCJuYmYiOjE3MjU2MjUzMzIsImV4cCI6MTcyNTYyOTIzMiwidmVyIjoiMi4wIiwidm1fYWNjZXNzIjp7Im1ldHJpY3NfYWNjb3VudF9pZCI6MTAwLCJtZXRyaWNzX3Byb2plY3RfaWQiOjEwMDA1LCJtZXRyaWNzX2V4dHJhX2ZpbHRlcnMiOlsie2ZpbHRlcj1cIjFcIn0iLCJ7ZmlsdGVyMj1cIjJcIn0iXSwibWV0cmljc19leHRyYV9sYWJlbHMiOlsia2V5PXZhbHVlIiwib3RoZXJfbGFiZWw9dmFsdWUiXSwibG9nc19hY2NvdW50X2lkIjo1MDAsImxvZ3NfcHJvamVjdF9pZCI6NTU1NSwibG9nc19leHRyYV9maWx0ZXJzIjpbImZpbHRlcj12YWx1ZSIsIm90aGVyX2ZpbGVyIl0sImxvZ3NfZXh0cmFfc3RyZWFtX2ZpbHRlcnMiOlsic3RyZWFtIGZpbHRlciIsIm90aGVyIHN0cmVhbSBmaWx0ZXIiLCJsYXN0IHN0cmVhbSBmaWx0ZXIiXX0sInNjb3BlIjoib3BlbmlkIn0.SVRbfypXpzJ1FL2ALu9_iO_J_UXTS0MiUX4SJ8ZqmN-JAsR8SudJAe1Lk8uubTsRtb234a8QYuzR1XhMLwM6SDkuioKC2VAGPV2YPb5Z7axv0juShJfZkaBaqf-zz_bx51-Bop6Xlpg5zySymYs9mLRwGKfIKMiIZVF5d0mDnG-BUawstQZX3RvVWODrLucIPiuJy9ry_tQz1uYbL8eadeqezfAPprB-bxGSScZ4SKeSW9j3wksB2zvAidlj5ZMnmkDRcXCkBgBxazQ0KeHXPly8kkC4yREtZiBCVz1HKsCncO-iWR2DFCf5jLwHiJVuwsTIjj7jdb9Hxgiu_CS3eA`)
+}

lib/promrelabel/debug.go

@@ -35,12 +35,6 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	labels, err := promutil.NewLabelsFromString(metric)
-	if err != nil {
-		err = fmt.Errorf("cannot parse metric: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
-		return
-	}
 	pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
 	if err != nil {
 		err = fmt.Errorf("cannot parse relabel configs: %w", err)
@@ -48,8 +42,33 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
-	WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
+	// metric input may contain multiple lines, and it's good to add support and print the debug steps for each of them.
+	// however, the target URL won't change
+	metrics := strings.Split(metric, "\n")
+	dsss := make([][]DebugStep, 0, len(metrics))
+	var lastErr error
+
+	for _, m := range metrics {
+		// try to trim all the special characters and space first
+		m = strings.TrimSpace(m)
+		labels, err := promutil.NewLabelsFromString(m)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		dss, tURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
+		targetURL = tURL
+		dsss = append(dsss, dss)
+	}
+
+	// break here if all failed
+	if len(dsss) == 0 && lastErr != nil {
+		err = fmt.Errorf("cannot parse metric: %w", err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, lastErr)
+		return
+	}
+
+	WriteRelabelDebugSteps(w, targetURL, targetID, format, dsss, metric, relabelConfigs, nil)
 }
 
 func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {

lib/promrelabel/debug.qtpl

@@ -6,15 +6,15 @@
 
 {% stripspace %}
 
-{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs  string, err error) %}
+{% func RelabelDebugSteps(targetURL, targetID, format string, dsss [][]DebugStep, metric string, relabelConfigs  string, err error) %}
   {% if format == "json" %}
-      {%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsJSON(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% else %}
-      {%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsHTML(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsHTML(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -60,11 +60,13 @@ function submitRelabelDebugForm(e) {
         </form>
         </div>
 
+        {% for _, dss := range dsss %}
         <div class="row">
             <main class="col-12">
               {%= relabelDebugSteps(dss, targetURL, targetID) %}
             </main>
         </div>
+        {% endfor %}
     </div>
 </body>
 </html>
@@ -77,7 +79,7 @@ function submitRelabelDebugForm(e) {
     </div>
 
     <div>
-        Labels:<br/>
+        Labels (One label set per line. Multiple lines will be treated as multiple label sets):<br/>
         <textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
     </div>
 {% endfunc %}
@@ -151,45 +153,52 @@ function submitRelabelDebugForm(e) {
 {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsJSON(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 {
     {% if err != nil %}
         "status": "error",
         "error": {%q= fmt.Sprintf("Error: %s", err) %}
     {% else %}
-        {% code var hasError bool %}
         "status": "success",
-        "steps": [
-            {% for i, ds := range dss %}
-                {% code
-                    inLabels, inErr := promutil.NewLabelsFromString(ds.In)
-                    outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
-                    changedLabels := getChangedLabelNames(inLabels, outLabels)
-                %}
+        "metrics": [
+            {% for idx, dss := range dsss %}
+                {% code var hasError bool %}
                 {
-                    "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
-                    "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
-                    "rule": {%q= ds.Rule %},
-                    "errors": {
-                        {% if inErr != nil %}
-                        "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
-                        {%code hasError = true %}
-                        {% else %}
-                        {% endif %}
-                        {% if outErr != nil %}
-                        "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
-                        {%code hasError = true %}
-                        {% endif %}
-                    }
+                    "steps": [
+                        {% for i, ds := range dss %}
+                            {% code
+                                inLabels, inErr := promutil.NewLabelsFromString(ds.In)
+                                outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
+                                changedLabels := getChangedLabelNames(inLabels, outLabels)
+                            %}
+                            {
+                                "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
+                                "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
+                                "rule": {%q= ds.Rule %},
+                                "errors": {
+                                    {% if inErr != nil %}
+                                    "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
+                                    {%code hasError = true %}
+                                    {% else %}
+                                    {% endif %}
+                                    {% if outErr != nil %}
+                                    "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
+                                    {%code hasError = true %}
+                                    {% endif %}
+                                }
+                            }
+                            {% if i != len(dss)-1 %},{% endif %}
+                        {% endfor %}
+                    ]
+                    {% if len(dss) > 0 && !hasError %}
+                    ,
+                      "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
+                      "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
+                    {% endif %}
                 }
-                {% if i != len(dss)-1 %},{% endif %}
+                {% if idx != len(dsss)-1 %},{% endif %}
             {% endfor %}
         ]
-        {% if len(dss) > 0 && !hasError %}
-        ,
-          "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
-          "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
-        {% endif %}
     {% endif %}
 }
 {% endfunc %}

lib/promscrape/relabel_debug.go

@@ -3,7 +3,9 @@ package promscrape
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -23,9 +25,39 @@ func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
 			targetID = ""
 		} else {
 			metric = labels.String()
-			relabelConfigs = pcs.String()
+			relabelConfigs += "# metrics_relabel_configs\n"
+			relabelConfigs += pcs.String()
+
+			rwRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+			rwURLRelabelConfigs := remotewrite.GetURLRelabelConfigData()
+
+			relabelConfigs += "\n# -remoteWrite.relabelConfig"
+			relabelConfigs += "\n" + rwRelabelConfigs
+
+			// we could have different relabel config for different remote write URL, but there's no way to know which one the user wants to debug.
+			// so we append the 1st one here, and comment out the rest. user can see them on the page and edit to activate them.
+			for i := range rwURLRelabelConfigs {
+				if i == 0 {
+					relabelConfigs += "\n# -remoteWrite.urlRelabelConfig"
+
+					// append the URL info
+					relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+
+					// append the relabeling config string
+					relabelConfigs += "\n" + rwURLRelabelConfigs[i].RelabelConfigStr
+					continue
+				}
+
+				// for the rest URLs add comment # before every line.
+				relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+				lines := strings.Split(rwURLRelabelConfigs[i].RelabelConfigStr, "\n")
+				for _, line := range lines {
+					relabelConfigs += "\n#" + line
+				}
+			}
 		}
 	}
+
 	if format == "json" {
 		httpserver.EnableCORS(w, r)
 		w.Header().Set("Content-Type", "application/json")