relabel debug: support multiple lines metrics input

Previously OPTIONS HTTP requests for CORS preflight checks would trigger
the original request handler. This pull request fixes that behavior to
align with https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5563

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

app/vmagent/datadogsketches/request_handler.go

@@ -49,6 +49,11 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 				Name:  "__name__",
 				Value: m.Name,
 			})
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			labels = append(labels, prompb.Label{
+				Name:  "host",
+				Value: sketch.Host,
+			})
 			for _, label := range m.Labels {
 				labels = append(labels, prompb.Label{
 					Name:  label.Name,
@@ -57,9 +62,6 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				labels = append(labels, prompb.Label{
 					Name:  name,
 					Value: value,

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,30 +83,58 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
+type UrlRelabelCfg struct {
+	Url           string `yaml:"url"`
+	RelabelConfig any    `yaml:"relabel_config"`
+
+	RelabelConfigStr string
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
-	p := remoteWriteURLRelabelConfigData.Load()
-	if p == nil {
+	cs := GetURLRelabelConfigData()
+	if cs == nil {
 		// Nothing to write to w
 		return
 	}
-	type urlRelabelCfg struct {
-		Url           string `yaml:"url"`
-		RelabelConfig any    `yaml:"relabel_config"`
+	d, _ := yaml.Marshal(cs)
+	_, _ = w.Write(d)
+}
+
+// GetURLRelabelConfigData is similar to WriteURLRelabelConfigData but returning data in []UrlRelabelCfg.
+func GetURLRelabelConfigData() []UrlRelabelCfg {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
 	}
-	var cs []urlRelabelCfg
+	var cs []UrlRelabelCfg
 	for i, url := range *remoteWriteURLs {
 		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
 		if !*showRemoteWriteURL {
 			url = fmt.Sprintf("%d:secret-url", i+1)
 		}
-		cs = append(cs, urlRelabelCfg{
+		cs = append(cs, UrlRelabelCfg{
 			Url:           url,
 			RelabelConfig: cfgData,
+
+			RelabelConfigStr: string(cfgDataBytes),
 		})
 	}
-	d, _ := yaml.Marshal(cs)
-	_, _ = w.Write(d)
+	return cs
 }
 
 func reloadRelabelConfigs() {

app/vmalert/config/config.go

@@ -81,12 +81,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.Interval.Duration() < 0 {
 		return fmt.Errorf("interval shouldn't be lower than 0")
 	}
-	if g.EvalOffset.Duration() < 0 {
-		return fmt.Errorf("eval_offset shouldn't be lower than 0")
-	}
-	// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
-	if g.EvalOffset.Duration() > g.Interval.Duration() {
-		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
+	// if `eval_offset` is set, the group interval must be specified explicitly(instead of inherited from global evaluationInterval flag) and must bigger than offset.
+	if g.EvalOffset.Duration().Abs() > g.Interval.Duration() {
+		return fmt.Errorf("the abs value of eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
 	}
 	if g.EvalOffset != nil && g.EvalDelay != nil {
 		return fmt.Errorf("eval_offset cannot be used with eval_delay")

app/vmalert/config/config_test.go

@@ -176,11 +176,17 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, false, "interval shouldn't be lower than 0")
 
 	f(&Group{
-		Name:       "wrong eval_offset",
+		Name:       "too big eval_offset",
 		Interval:   promutil.NewDuration(time.Minute),
 		EvalOffset: promutil.NewDuration(2 * time.Minute),
 	}, false, "eval_offset should be smaller than interval")
 
+	f(&Group{
+		Name:       "too big negative eval_offset",
+		Interval:   promutil.NewDuration(time.Minute),
+		EvalOffset: promutil.NewDuration(-2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
 	limit := -1
 	f(&Group{
 		Name:  "wrong limit",

app/vmalert/rule/group.go

@@ -484,8 +484,15 @@ func (g *Group) UpdateWith(newGroup *Group) {
 // delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
 func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
 	if g.EvalOffset != nil {
+		offset := *g.EvalOffset
+		// adjust the offset for negative evalOffset, the rule is:
+		// `eval_offset: -x` is equivalent to `eval_offset: y` for `interval: x+y`.
+		// For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
+		if offset < 0 {
+			offset += g.Interval
+		}
 		// if offset is specified, ignore the maxDelay and return a duration aligned with offset
-		currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
+		currentOffsetPoint := ts.Truncate(g.Interval).Add(offset)
 		if currentOffsetPoint.Before(ts) {
 			// wait until the next offset point
 			return currentOffsetPoint.Add(g.Interval).Sub(ts)

app/vmalert/rule/group_test.go

@@ -606,6 +606,15 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
 	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
 
+	// test group with negative offset -2min, which is equivalent to 3min offset for 5min interval
+	offset = -2 * time.Minute
+	g.EvalOffset = &offset
+
+	f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
+	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
+
 	maxDelay = time.Minute * 1
 	g.EvalOffset = nil
 

app/vminsert/datadogsketches/request_handler.go

@@ -45,15 +45,14 @@ func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompb.Label)
 		ms := sketch.ToSummary()
 		for _, m := range ms {
 			ctx.Labels = ctx.Labels[:0]
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			ctx.AddLabel("host", sketch.Host) // newly added
 			ctx.AddLabel("", m.Name)
 			for _, label := range m.Labels {
 				ctx.AddLabel(label.Name, label.Value)
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				ctx.AddLabel(name, value)
 			}
 			for j := range extraLabels {

app/vmselect/main.go

@@ -321,19 +321,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/tags/tagSeries":
 		graphiteTagsTagSeriesRequests.Inc()
-		if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags/tagMultiSeries":
 		graphiteTagsTagMultiSeriesRequests.Inc()
-		if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagMultiSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagMultiSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags":
 		graphiteTagsRequests.Inc()

apptest/model.go

@@ -33,6 +33,8 @@ type PrometheusQuerier interface {
 	// separate interface or rename this interface to allow for multiple querier
 	// types.
 	GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
+	GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
+	GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
 }
 
 // Writer contains methods for writing new data

apptest/tests/graphite_test.go

@@ -60,3 +60,60 @@ func TestClusterMetricsIndex(t *testing.T) {
 
 	testMetricsIndex(tc.T(), sut)
 }
+
+// testTagSeries tests the registration of new time series in index.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb.
+func testTagSeries(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, getStorageMetric func(string) int) {
+	t := tc.T()
+
+	assertNewTimeseriesCreatedTotal := func(want int) {
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected vm_new_timeseries_created_total",
+			Got: func() any {
+				return getStorageMetric("vm_new_timeseries_created_total")
+			},
+			Want: want,
+		})
+	}
+
+	rec := "disk.used;rack=a1;datacenter=dc1;server=web01"
+	sut.GraphiteTagsTagSeries(t, rec, apptest.QueryOpts{})
+	assertNewTimeseriesCreatedTotal(0)
+
+	recs := []string{
+		"metric.yyy;t2=a;t1=b;t3=c",
+		"metric.zzz;t5=d;t4=e;t6=f",
+		"metric.xxx;t8=g;t7=h;t9=i",
+	}
+	sut.GraphiteTagsTagMultiSeries(t, recs, apptest.QueryOpts{})
+	assertNewTimeseriesCreatedTotal(0)
+}
+
+func TestSingleTagSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultVmsingle()
+	getStorageMetric := func(name string) int {
+		return sut.GetIntMetric(t, name)
+	}
+
+	testTagSeries(tc, sut, getStorageMetric)
+}
+
+func TestClusterTagSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultCluster()
+	getStorageMetric := func(name string) int {
+		var v int
+		for _, s := range sut.Vmstorages {
+			v += s.GetIntMetric(t, name)
+		}
+		return v
+	}
+
+	testTagSeries(tc, sut, getStorageMetric)
+}

apptest/vmselect.go

@@ -307,6 +307,37 @@ func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) Graphite
 	return index
 }
 
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagMultiSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
 // APIV1AdminTenants sends a query to a /admin/tenants endpoint
 func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
 	t.Helper()

apptest/vmsingle.go

@@ -414,6 +414,37 @@ func (app *Vmsingle) GraphiteMetricsIndex(t *testing.T, _ QueryOpts) GraphiteMet
 	return index
 }
 
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmsingle) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmsingle) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagMultiSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
 // APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
 // and returns the statistics response for given params.
 //

dashboards/vm/vmauth.json

@@ -1612,7 +1612,7 @@
                 "type": "victoriametrics-metrics-datasource",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
@@ -1624,7 +1624,7 @@
                 "type": "victoriametrics-metrics-datasource",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

dashboards/vmauth.json

@@ -1611,7 +1611,7 @@
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
@@ -1623,7 +1623,7 @@
                 "type": "prometheus",
                 "uid": "$ds"
               },
-              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+              "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

docs/guides/getting-started-with-opentelemetry/README.md

@@ -316,4 +316,4 @@ using query `service.name: unknown_service:otel`.
 ## Limitations
 
 - VictoriaMetrics and VictoriaLogs do not support experimental JSON encoding [format](https://github.com/open-telemetry/opentelemetry-proto/blob/main/examples/metrics.json).
-- VictoriaMetrics supports only the `AggregationTemporalityCumulative` type for [histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#histogram) and [summary](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#summary-legacy). Either consider using cumulative temporality or use the [`delta-to-cumulative processor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) to convert to cumulative temporality in OpenTelemetry Collector.
+

docs/victoriametrics/FAQ.md

@@ -557,8 +557,8 @@ and proportionally to the total length of all the labels seen across all the reg
 
 Typical monitoring in Kubernetes generates moderate-to-high churn rate for time series because every restart of the `pod` creates a new set of time series
 for all the [metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#what-is-a-metric) exposed by that pod, with a new `pod` label.
-The number of labels and the summary length of `label=value` pairs per every time series in Kubernetes is quite large
-(~30-40 labels with ~1KB summary length of `label=value` pairs per time series). This contributes to quick growth of the `indexdb` over time,
+The number of labels and the total length of `label=value` pairs per every time series in Kubernetes is quite large
+(~30-40 labels with ~1KB total length of `label=value` pairs per time series). This contributes to quick growth of the `indexdb` over time,
 so its' size may exceed the size of the `data` folder by up to 2x in typical production cases.
 
 There are the following workarounds, which can reduce the growth rate of the `indexdb`:

docs/victoriametrics/README.md

@@ -708,7 +708,7 @@ Using the delete API is not recommended in the following cases, since it brings
   time series occupy disk space until the next merge operation, which can never occur when deleting too old data.
   [Forced merge](#forced-merge) may be used for freeing up disk space occupied by old data.
   Note that VictoriaMetrics doesn't delete entries from [IndexDB](#indexdb) for the deleted time series.
-  IndexDB is cleaned up once per the configured [retention](#retention).
+  IndexDB is cleaned up along with the corresponding data partition once it becomes outside the [-retentionPeriod](#retention).
 
 It's better to use the `-retentionPeriod` command-line flag for efficient pruning of old data.
 
@@ -1419,21 +1419,22 @@ See also [how to work with snapshots](#how-to-work-with-snapshots) and [IndexDB]
 ## IndexDB
 
 VictoriaMetrics identifies
-[time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) by
-`TSID` (time series ID) and stores
-[raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) sorted
-by TSID (see [Storage](#storage)). Thus, the TSID is a primary index and could
-be used for searching and retrieving raw samples. However, the TSID is never
-exposed to the clients, i.e. it is for internal use only.
+[time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
+by `TSID` (time series ID) and stores
+[raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+sorted by TSID (see [Storage](#storage)). Thus, the TSID is a primary index and
+could be used for searching and retrieving raw samples. However, the TSID is
+never exposed to the clients, i.e. it is for internal use only.
 
-Instead, VictoriaMetrics maintains an **inverted index** that enables searching
-the raw samples by metric name, label name, and label value by mapping these
-values to the corresponding TSIDs.
+Instead, VictoriaMetrics maintains an **inverted index** (known as `indexDB`)
+that enables searching the raw samples by metric name, label name, and label
+value by mapping these values to the corresponding TSIDs. Every data
+[partition](#storage) has its own indexDB.
 
 VictoriaMetrics uses two types of inverted indexes:
 
 * Global index. Searches using this index is performed across the entire
-  retention period.
+  partition time range.
 * Per-day index. This index stores mappings similar to ones in global index
   but also includes the date in each mapping. This speeds up data retrieval
   for queries within a shorter time range (which is often just the last day).
@@ -1441,19 +1442,18 @@ VictoriaMetrics uses two types of inverted indexes:
 When the search query is executed, VictoriaMetrics decides which index to use
 based on the time range of the query:
 
-* Per-day index is used if the search time range is 40 days or less.
-* Global index is used for search queries with a time range greater than 40
-  days.
+* Per-day index is used if the search time range is less than the partition time range.
+* Global index is used for search queries with a time range that matches exactly
+  or greater than the partition time range.
 
 Mappings are added to the indexes during the data ingestion:
 
-* In global index each mapping is created only once per retention period.
+* In global index each mapping is created only once per partition.
 * In the per-day index each mapping is created for each unique date that
   has been seen in the samples for the corresponding time series.
 
-IndexDB respects [retention period](#retention) and once it is over, the indexes
-are dropped. For the new retention period, the indexes are gradually populated
-again as the new samples arrive.
+Since indexDB is a part of a partition, it is dropped along with it as it
+becomes outside the [retention period](#retention).
 
 See also [Why IndexDB size is so large?](https://docs.victoriametrics.com/victoriametrics/faq/#why-indexdb-size-is-so-large).
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,9 +26,15 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: all VictoriaMetrics components: implement proper CORS preflight handling by responding 204 No Content to HTTP OPTIONS requests. See [#5563](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5563).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `access_log` configuration option for each user that will log requests to stdout, and support filtering by HTTP status codes. See more in [docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). See [#5936](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5936).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support negative values for the group `eval_offset` option, which allows starting group evaluation at `groupInterval-abs(eval_offset)` within `[0...groupInterval]`. See [#10424](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10424).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): enhance metrics relabel debug by adding remote write relabel configs to the relabel configs input. See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918). 
 
+* BUGFIX: [dashboards/vmauth](https://grafana.com/grafana/dashboards/21394): fix `requested from system` and `heap inuse` expressions in the memory usage panel. See [#10574](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10574).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not enable ACL when uploading backups to S3-compatible endpoints by default. ACL is not always supported by S3-compatible endpoints and it is not recommended to use ACLs to limit access to objects. See [#10539](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10539) for more details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly attach `host` label to the time series ingested via [/datadog/api/beta/sketches](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/#) API. See [#10557](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557).
 
 ## [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.137.0)
 

docs/victoriametrics/integrations/opentelemetry.md

@@ -13,10 +13,10 @@ It expects `protobuf`-encoded requests at `/opentelemetry/v1/metrics`. For gzip-
 
 See how to configure [OpenTelemetry Collector](https://docs.victoriametrics.com/victoriametrics/data-ingestion/opentelemetry-collector/) to push metrics to VictoriaMetrics.
 
-## Metric naming
+## Label sanitization
 
-By default, VictoriaMetrics stores the ingested OpenTelemetry [metric samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) as is **without any transformations**.
-The following label transformations can be enabled:
+By default, VictoriaMetrics stores the ingested OpenTelemetry [metric points](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#metric-points) as is **without any transformations**.
+The following label sanitization options can be enabled:
 * `-usePromCompatibleNaming` - replaces characters unsupported by Prometheus with `_` in metric names and labels **for all ingestion protocols**.
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time{service_name="foo"}`.
 * `-opentelemetry.usePrometheusNaming` - converts metric names and labels according to [OTLP Metric points to Prometheus specification](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.33.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus) for metrics ingested via OTLP.
@@ -26,10 +26,21 @@ The following label transformations can be enabled:
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 
+## Resource Attributes
+
+By default, VictoriaMetrics promotes all [OpenTelemetry resource](https://opentelemetry.io/docs/specs/otel/resource/data-model/) attributes to labels and attaches them to all ingested OTLP metrics.
+
 ## Exponential histograms
 
 OpenTelemetry [exponential histogram](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram) is automatically converted
-to [VictoriaMetrics histogram format](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350).
+to [VictoriaMetrics histogram format](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) during ingestion. Since VictoriaMetrics histogram doesn't support negative observations, all buckets in the negative range are dropped.
+
+## Delta Temporality
+
+In OpenTelemetry, some metric types(including sums, histograms, and exponential histograms) support delta and cumulative aggregation temporality. VictoriaMetrics works best with cumulative temporality, and it's recommended to export metrics with cumulative temporality or convert delta to cumulative temporality using [OpenTelemetry Collector deltatocumulative processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/deltatocumulativeprocessor) before sending to VictoriaMetrics.
+VictoriaMetrics stores delta temporality metric values as is {{% available_from "v1.132.0" %}}, they can be queried with [sum_over_time()](https://docs.victoriametrics.com/victoriametrics/metricsql/#sum_over_time) and [rate_over_sum()](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate_over_sum).
+
+> Do not apply [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling) to delta temporality metrics, since it might cause data loss.
 
 ## References
 

docs/victoriametrics/vmalert.md

@@ -144,8 +144,10 @@ name: <string>
 # Optional
 # Group will be evaluated at the exact offset in the range of [0...interval].
 # E.g. for Group with `interval: 1h` and `eval_offset: 5m` the evaluation will
-# start at 5th minute of the hour. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409
-# `interval` must be specified if `eval_offset` is used, and `eval_offset` cannot exceed `interval`.
+# start at 5th minute of the hour.
+# `eval_offset` also supports negative values, which means the evaluation will start at `interval-abs(eval_offset)` within [0...interval],
+# For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
+# `interval` must be specified if `eval_offset` is used, and the `abs(eval_offset)` cannot exceed `interval`.
 # `eval_offset` cannot be used with `eval_delay`, as group will be executed at the exact offset and `eval_delay` is ignored.
 [ eval_offset: <duration> ]
 

lib/httpserver/httpserver.go

@@ -357,6 +357,12 @@ func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
 		r.URL.Path = path
 	}
 
+	if r.Method == http.MethodOptions {
+		EnableCORS(w, r)
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+
 	w = &responseWriterWithAbort{
 		ResponseWriter: w,
 	}
@@ -511,6 +517,8 @@ func EnableCORS(w http.ResponseWriter, _ *http.Request) {
 		return
 	}
 	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "*")
+	w.Header().Set("Access-Control-Allow-Headers", "*")
 }
 
 func pprofHandler(profileName string, w http.ResponseWriter, r *http.Request) {

lib/httpserver/httpserver_test.go

@@ -144,6 +144,55 @@ func TestAuthKeyMetrics(t *testing.T) {
 	tstWithOutAuthKey("wrong", "wrong", 401)
 }
 
+func TestHandlerWrapperOptionsRequest(t *testing.T) {
+	handlerCalled := false
+	rh := func(_ http.ResponseWriter, _ *http.Request) bool {
+		handlerCalled = true
+		return true
+	}
+	headersToCheck := []string{"Access-Control-Allow-Origin", "Access-Control-Allow-Headers"}
+	f := func(t *testing.T, corsDisabled bool) {
+		t.Helper()
+		handlerCalled = false
+
+		origDisableCORS := *disableCORS
+		*disableCORS = corsDisabled
+		defer func() {
+			*disableCORS = origDisableCORS
+		}()
+
+		wantCORSHeaderValue := "*"
+		if corsDisabled {
+			wantCORSHeaderValue = ""
+		}
+		req := httptest.NewRequest(http.MethodOptions, "/api/v1/query_range", nil)
+		w := httptest.NewRecorder()
+
+		handlerWrapper(w, req, rh)
+
+		res := w.Result()
+		_ = res.Body.Close()
+
+		if res.StatusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code; (-%d;+%d)", http.StatusNoContent, res.StatusCode)
+		}
+		if handlerCalled {
+			t.Fatalf("request handler must not be called for OPTIONS requests")
+		}
+		for _, h := range headersToCheck {
+			got := res.Header.Get(h)
+			if wantCORSHeaderValue != got {
+				t.Fatalf("unexpected header: %s value: (-%s;+%s)", h, wantCORSHeaderValue, got)
+			}
+		}
+	}
+
+	// CORS disabled
+	f(t, false)
+	// CORS enabled
+	f(t, true)
+}
+
 func TestHandlerWrapper(t *testing.T) {
 	const hstsHeader = "foo"
 	const frameOptionsHeader = "bar"

lib/jwt/jwt.go

@@ -3,12 +3,14 @@ package jwt
 import (
 	"bytes"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
+	"math"
 	"net/http"
-	"slices"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/valyala/fastjson"
 )
 
 const (
@@ -32,8 +34,8 @@ var (
 // Token represents jwt token
 // https://auth0.com/docs/tokens/json-web-tokens
 type Token struct {
-	header             *header
-	body               *body
+	header             header
+	body               body
 	payload, signature []byte
 }
 
@@ -41,55 +43,380 @@ type header struct {
 	Alg string `json:"alg"`
 	Typ string `json:"typ"`
 	Kid string `json:"kid"`
+
+	buf []byte
+	p   *fastjson.Parser
+}
+
+func (h *header) parse(src string) error {
+	var err error
+	h.buf, err = decodeB64(h.buf[:0], src)
+	if err != nil {
+		return err
+	}
+
+	h.p = parserPool.Get()
+	jv, err := h.p.ParseBytes(h.buf)
+	if err != nil {
+		return err
+	}
+	if jv == nil {
+		return fmt.Errorf("unexpected empty json")
+	}
+	if jv.Type() != fastjson.TypeObject {
+		return fmt.Errorf("unexpected non json object {} type: %q", jv.Type())
+	}
+	h.Alg, err = stringFromJSONValue(jv, "alg")
+	if err != nil {
+		return err
+	}
+	h.Typ, err = stringFromJSONValue(jv, "typ")
+	if err != nil {
+		return err
+	}
+	h.Kid, err = stringFromJSONValue(jv, "kid")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *header) reset() {
+	h.Alg = ""
+	h.Typ = ""
+	h.Kid = ""
+
+	h.buf = h.buf[:0]
+	if h.p != nil {
+		parserPool.Put(h.p)
+		h.p = nil
+	}
 }
 
 type body struct {
 	// expired at time unix_ts
 	Exp int64 `json:"exp"`
 	// issued at time unix_ts
-	Iat      int64          `json:"iat"`
-	Jti      string         `json:"jti,omitempty"`
-	Scope    string         `json:"scope,omitempty"`
-	VMAccess *VMAccessClaim `json:"vm_access"`
+	Iat           int64  `json:"iat"`
+	Jti           string `json:"jti,omitempty"`
+	Scope         string `json:"scope,omitempty"`
+	vmAccessClaim VMAccessClaim
+
+	buf []byte
+	p   *fastjson.Parser
+
+	// allClaims holds entire json body
+	// for the HasClaims() method
+	allClaims *fastjson.Value
+
+	// claimsParser holds optional parser for `vm_access` string representation
+	claimsParser *fastjson.Parser
 }
 
-// Labels defines labels added to filters or incoming time series.
-type Labels map[string]string
+func (b *body) parse(src string) error {
 
-// AsExtraLabels - converts labels to label=value pairs.
-func (l Labels) AsExtraLabels() []string {
-	if len(l) == 0 {
-		return nil
+	var err error
+	b.buf, err = decodeB64(b.buf[:0], src)
+	if err != nil {
+		return err
 	}
-	res := make([]string, 0, len(l))
-	for k, v := range l {
-		res = append(res, k+"="+v)
+	b.p = parserPool.Get()
+	jv, err := b.p.ParseBytes(b.buf)
+	if err != nil {
+		return err
 	}
-	// sort for consistent uri.
-	slices.Sort(res)
-	return res
+	if expObject := jv.Get("exp"); expObject != nil {
+		b.Exp, err = expObject.Int64()
+		if err != nil {
+			return fmt.Errorf("cannot parse `exp` field: %w", err)
+		}
+	}
+	if iatObject := jv.Get("iat"); iatObject != nil {
+		b.Iat, err = iatObject.Int64()
+		if err != nil {
+			return fmt.Errorf("cannot parse `iat` field: %w", err)
+		}
+	}
+	vaObject := jv.Get("vm_access")
+	if vaObject == nil {
+		return ErrVMAccessFieldMissing
+	}
+	// some IDPs encode custom claims as a string
+	// try parsing as an object and fallback to a string
+	switch vaObject.Type() {
+	case fastjson.TypeObject:
+		if err := b.vmAccessClaim.parseFrom(vaObject); err != nil {
+			return err
+		}
+	case fastjson.TypeString:
+		b.claimsParser = parserPool.Get()
+		va, err := b.claimsParser.ParseBytes(vaObject.GetStringBytes())
+		if err != nil {
+			return fmt.Errorf("cannot parse `vm_access` string json: %w", err)
+		}
+		if err := b.vmAccessClaim.parseFrom(va); err != nil {
+			return fmt.Errorf("cannot parse `vm_access` values from string json: %w", err)
+		}
+	case fastjson.TypeNull:
+		return ErrVMAccessFieldMissing
+	default:
+		return fmt.Errorf("unexpected type for `vm_access` field; got: %q, want object {}", vaObject.Type())
+	}
+	b.Jti = bytesutil.ToUnsafeString(jv.GetStringBytes("jti"))
+
+	if scopeObject := jv.Get("scope"); scopeObject != nil {
+		// some IDPs encode scope as a string and some as an array
+		switch scopeObject.Type() {
+		case fastjson.TypeString:
+			sb := scopeObject.GetStringBytes()
+			b.Scope = bytesutil.ToUnsafeString(sb)
+		case fastjson.TypeArray:
+			var sizeNeeded int
+			ss := scopeObject.GetArray()
+			for _, v := range ss {
+				sizeNeeded += len(v.GetStringBytes()) + 1
+			}
+			dst := make([]byte, 0, sizeNeeded)
+			for idx, v := range ss {
+				dst = append(dst, v.GetStringBytes()...)
+				if idx < len(ss)-1 {
+					dst = append(dst, ' ')
+				}
+			}
+			b.Scope = bytesutil.ToUnsafeString(dst)
+		default:
+			return fmt.Errorf("unexpected type for `scope` field; got %q, want String or []String", scopeObject.Type())
+		}
+	}
+	b.allClaims = jv
+
+	return nil
 }
 
+func (b *body) reset() {
+	b.Exp = 0
+	b.Iat = 0
+	b.Jti = ""
+	b.Scope = ""
+	b.buf = b.buf[:0]
+	b.allClaims = nil
+	b.vmAccessClaim.reset()
+	if b.p != nil {
+		parserPool.Put(b.p)
+		b.p = nil
+	}
+	if b.claimsParser != nil {
+		parserPool.Put(b.claimsParser)
+		b.claimsParser = nil
+	}
+
+}
+
+// Parse parses JWT token from given source string
+//
+// Token field is valid until src is reachable
+func (t *Token) Parse(src string, enforceAuthPrefix bool) error {
+	if enforceAuthPrefix && (len(src) < len(prefix) || !strings.EqualFold(src[:len(prefix)], prefix)) {
+		return fmt.Errorf("wrong format, prefix: %s is missing", prefix)
+	}
+	// While https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 states that only Bearer prefix is allowed,
+	// it claims to be conformant to the generic syntax defined in https://datatracker.ietf.org/doc/html/rfc2617#section-1.2
+	// which permits case-insensitive auth scheme.
+	// So we should be tolerant to different cases of "Bearer" prefix.
+	if len(src) >= len(prefix) && strings.EqualFold(src[:len(prefix)], prefix) {
+		src = src[len(prefix):]
+	}
+
+	// assume jwt token has the following structure:
+	// header.body.signature
+	var header, body, signature string
+	idx := strings.IndexByte(src, '.')
+	if idx <= 0 {
+		return ErrBadTokenFormat
+	}
+	header = src[:idx]
+	src = src[idx+1:]
+	idx = strings.IndexByte(src, '.')
+	if idx <= 0 {
+		return ErrBadTokenFormat
+	}
+	body = src[:idx]
+	signature = src[idx+1:]
+	if err := t.parse(header, body, signature); err != nil {
+		return err
+	}
+	return nil
+}
+
+// HasClaims checks if Token has all given claim key value pairs
+func (t *Token) HasClaims(claims map[string]string) bool {
+	for k, v := range claims {
+		gotV := t.body.allClaims.Get(k)
+		if gotV == nil || gotV.Type() != fastjson.TypeString {
+			return false
+		}
+		tcv := bytesutil.ToUnsafeString(gotV.GetStringBytes())
+		if tcv != v {
+			return false
+		}
+	}
+
+	return true
+}
+
+// VMAccess return a reference to the VMAccessClaim
+// all data are valid until Token is reachable
+func (t *Token) VMAccess() *VMAccessClaim {
+	return &t.body.vmAccessClaim
+}
+
+// Reset release memory used by token
+// Token cannot be used after this call
+func (t *Token) Reset() {
+	t.header.reset()
+	t.body.reset()
+	t.payload = t.payload[:0]
+	t.signature = t.signature[:0]
+}
+
+// VMAccessClaim represent JWT claim object
 type VMAccessClaim struct {
-	Tenant TenantID `json:"tenant_id"`
-	Labels Labels   `json:"extra_labels,omitempty"`
 	// promql filters applied to each select query
 	ExtraFilters []string `json:"extra_filters,omitempty"`
+
+	MetricsExtraFilters    []string `json:"metrics_extra_filters,omitempty"`
+	MetricsExtraLabels     []string `json:"metrics_extra_labels,omitempty"`
+	LogsExtraFilters       []string `json:"logs_extra_filters,omitempty"`
+	LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty"`
+
+	Labels []string `json:"extra_labels,omitempty"`
+	// labelsBuf holds allocated memory for Labels
+	labelsBuf []byte
+	Tenant    TenantID `json:"tenant_id"`
 	// role can be denied as 1 = read, 2 = write, 3 = read and write
 	// 0 = unconfigured - read and write
 	Mode int `json:"mode,omitempty"`
 
-	// TODO: use different claim struct for vmauth and vmgateway
-	// parsing must be dynamic based on provided hint
-	MetricsAccountID    uint32   `json:"metrics_account_id,omitempty"`
-	MetricsProjectID    uint32   `json:"metrics_project_id,omitempty"`
-	MetricsExtraFilters []string `json:"metrics_extra_filters,omitempty"`
-	MetricsExtraLabels  []string `json:"metrics_extra_labels,omitempty"`
+	MetricsAccountID uint32 `json:"metrics_account_id,omitempty"`
+	MetricsProjectID uint32 `json:"metrics_project_id,omitempty"`
 
-	LogsAccountID          uint32   `json:"logs_account_id,omitempty"`
-	LogsProjectID          uint32   `json:"logs_project_id,omitempty"`
-	LogsExtraFilters       []string `json:"logs_extra_filters,omitempty"`
-	LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty"`
+	LogsAccountID uint32 `json:"logs_account_id,omitempty"`
+	LogsProjectID uint32 `json:"logs_project_id,omitempty"`
+}
+
+func (vac *VMAccessClaim) reset() {
+	vac.Tenant.AccountID = 0
+	vac.Tenant.ProjectID = 0
+	clear(vac.Labels)
+	vac.Labels = vac.Labels[:0]
+	vac.labelsBuf = vac.labelsBuf[:0]
+	clear(vac.ExtraFilters)
+	vac.ExtraFilters = vac.ExtraFilters[:0]
+	vac.Mode = 0
+
+	vac.MetricsAccountID = 0
+	vac.MetricsProjectID = 0
+	clear(vac.MetricsExtraFilters)
+	vac.MetricsExtraFilters = vac.MetricsExtraFilters[:0]
+	clear(vac.MetricsExtraLabels)
+	vac.MetricsExtraLabels = vac.MetricsExtraLabels[:0]
+	vac.LogsAccountID = 0
+	vac.LogsProjectID = 0
+	clear(vac.LogsExtraFilters)
+	vac.LogsExtraFilters = vac.LogsExtraFilters[:0]
+	clear(vac.LogsExtraStreamFilters)
+	vac.LogsExtraStreamFilters = vac.LogsExtraStreamFilters[:0]
+}
+
+func (vac *VMAccessClaim) parseFrom(jv *fastjson.Value) error {
+
+	if err := vac.Tenant.parseFrom(jv); err != nil {
+		return err
+	}
+
+	var err error
+	vac.ExtraFilters, err = stringSliceFromJSONValue(vac.ExtraFilters, jv, "extra_filters")
+	if err != nil {
+		return err
+	}
+	efs := jv.Get("extra_labels")
+	if efs != nil {
+		efsO, err := efs.Object()
+		if err != nil {
+			return fmt.Errorf("cannot parse `extra_labels` field: %w", err)
+		}
+		buf := vac.labelsBuf[:0]
+		var visitErr error
+		efsO.Visit(func(key []byte, v *fastjson.Value) {
+			if visitErr != nil {
+				return
+			}
+			vs, err := v.StringBytes()
+			if err != nil {
+				visitErr = fmt.Errorf("unexpected value for key=%q: %w", string(key), err)
+			}
+			start := len(buf)
+			sizeNeeded := len(key) + 1 + len(vs)
+			if len(buf)+sizeNeeded >= cap(buf) {
+				// allocate new slice without memory fragmentation
+				// old slice will be referenced by vac.Labels
+				start = 0
+				buf = make([]byte, 0, len(buf)+sizeNeeded)
+			}
+			buf = append(buf, key...)
+			buf = append(buf, '=')
+			buf = append(buf, vs...)
+			ef := bytesutil.ToUnsafeString(buf[start:])
+			vac.Labels = append(vac.Labels, ef)
+		})
+		vac.labelsBuf = buf
+		if visitErr != nil {
+			return fmt.Errorf("cannot parse `extra_labels` field: %w", visitErr)
+		}
+	}
+	mode := jv.Get("mode")
+	if mode != nil {
+		vac.Mode, err = mode.Int()
+		if err != nil {
+			return fmt.Errorf("unexpected `mode` value: %w", err)
+		}
+	}
+	vac.MetricsAccountID, err = uint32FromJSONValue(jv, "metrics_account_id")
+	if err != nil {
+		return err
+	}
+	vac.MetricsProjectID, err = uint32FromJSONValue(jv, "metrics_project_id")
+	if err != nil {
+		return err
+	}
+
+	vac.MetricsExtraFilters, err = stringSliceFromJSONValue(vac.MetricsExtraFilters, jv, "metrics_extra_filters")
+	if err != nil {
+		return err
+	}
+	vac.MetricsExtraLabels, err = stringSliceFromJSONValue(vac.MetricsExtraLabels, jv, "metrics_extra_labels")
+	if err != nil {
+		return err
+	}
+	vac.LogsAccountID, err = uint32FromJSONValue(jv, "logs_account_id")
+	if err != nil {
+		return err
+	}
+	vac.LogsProjectID, err = uint32FromJSONValue(jv, "logs_project_id")
+	if err != nil {
+		return err
+	}
+	vac.LogsExtraFilters, err = stringSliceFromJSONValue(vac.LogsExtraFilters, jv, "logs_extra_filters")
+	if err != nil {
+		return err
+	}
+	vac.LogsExtraStreamFilters, err = stringSliceFromJSONValue(vac.LogsExtraStreamFilters, jv, "logs_extra_stream_filters")
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // TenantID represents tenantID.
@@ -98,12 +425,33 @@ type TenantID struct {
 	AccountID int32 `json:"account_id"`
 }
 
+func (tid *TenantID) parseFrom(jv *fastjson.Value) error {
+	tidObject := jv.Get("tenant_id")
+	if tidObject == nil {
+		return nil
+	}
+	var err error
+	tid.AccountID, err = int32FromJSONValue(tidObject, "account_id")
+	if err != nil {
+		return err
+	}
+	tid.ProjectID, err = int32FromJSONValue(tidObject, "project_id")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // String implements interface.
 func (tid TenantID) String() string {
 	return fmt.Sprintf("%d:%d", tid.AccountID, tid.ProjectID)
 }
 
 // NewToken creates token from raw string.
+//
+// Deprecated: allocates a new Token on every call.
+// Prefer acquiring a Token from a sync.Pool, calling t.Parse(), and returning it after use.
 func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
 	if enforceAuthPrefix && (len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix)) {
 		return nil, fmt.Errorf("wrong format, prefix: %s is missing", prefix)
@@ -122,10 +470,16 @@ func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
 		return nil, ErrBadTokenFormat
 	}
 	var t Token
-	return t.parse(jwt[0], jwt[1], jwt[2])
+	if err := t.parse(jwt[0], jwt[1], jwt[2]); err != nil {
+		return nil, err
+	}
+	return &t, nil
 }
 
 // NewTokenFromRequestWithCustomHeader return new jwt token from request by provided header
+//
+// Deprecated: allocates a new Token on every call.
+// Prefer acquiring a Token from a sync.Pool, calling t.Parse(), and returning it after use.
 func NewTokenFromRequestWithCustomHeader(r *http.Request, headerName string, enforceAuthPrefix bool) (*Token, error) {
 	auth := r.Header.Get(headerName)
 	if len(auth) == 0 {
@@ -134,28 +488,25 @@ func NewTokenFromRequestWithCustomHeader(r *http.Request, headerName string, enf
 	return NewToken(auth, enforceAuthPrefix)
 }
 
-func (t *Token) parse(header, body, signature string) (*Token, error) {
-	b, err := parseJWTBody(body)
-	if err != nil {
-		return nil, err
+func (t *Token) parse(header, body, signature string) error {
+	if err := t.body.parse(body); err != nil {
+		return fmt.Errorf("cannot parse token body: %w", err)
 	}
-	if b.VMAccess == nil {
-		return nil, ErrVMAccessFieldMissing
-	}
-	t.body = b
-	h, err := parseJWTHeader(header)
-	if err != nil {
-		return nil, err
-	}
-	t.header = h
-
-	t.payload = []byte(header + "." + body)
-	t.signature, err = decodeB64([]byte(signature))
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode signature as b64: %w", err)
+	if err := t.header.parse(header); err != nil {
+		return fmt.Errorf("cannot parse token header: %w", err)
 	}
 
-	return t, nil
+	t.payload = bytesutil.ResizeNoCopyNoOverallocate(t.payload, len(header)+len(body)+1)
+	t.payload = append(t.payload[:0], header...)
+	t.payload = append(t.payload, '.')
+	t.payload = append(t.payload, body...)
+	var err error
+	t.signature, err = decodeB64(t.signature[:0], signature)
+	if err != nil {
+		return fmt.Errorf("cannot decode token signature: %w", err)
+	}
+
+	return nil
 }
 
 // IsExpired checks if jwt token is expired.
@@ -166,10 +517,10 @@ func (t *Token) IsExpired(currentTime time.Time) bool {
 // CanWrite checks if token has write permissions.
 func (t *Token) CanWrite() bool {
 	// unconfigured
-	if t.body.VMAccess.Mode == 0 {
+	if t.body.vmAccessClaim.Mode == 0 {
 		return true
 	}
-	if write&t.body.VMAccess.Mode > 0 {
+	if write&t.body.vmAccessClaim.Mode > 0 {
 		return true
 	}
 	return false
@@ -178,10 +529,10 @@ func (t *Token) CanWrite() bool {
 // CanRead check if token has read permissions.
 func (t *Token) CanRead() bool {
 	// unconfigured
-	if t.body.VMAccess.Mode == 0 {
+	if t.body.vmAccessClaim.Mode == 0 {
 		return true
 	}
-	if read&t.body.VMAccess.Mode > 0 {
+	if read&t.body.vmAccessClaim.Mode > 0 {
 		return true
 	}
 	return false
@@ -189,101 +540,36 @@ func (t *Token) CanRead() bool {
 
 // AccessLabels returns vm_access labels for given JWT token,
 // in key=value format.
+//
+// Returned value is only valid until Token is reachable
 func (t *Token) AccessLabels() []string {
-	return t.body.VMAccess.Labels.AsExtraLabels()
+	return t.body.vmAccessClaim.Labels
 }
 
 // Tenant returns tenantID for token.
 func (t *Token) Tenant() TenantID {
-	return t.body.VMAccess.Tenant
+	return t.body.vmAccessClaim.Tenant
 }
 
 // ExtraFilters metricsql filters for select queries
+//
+// Returned value is only valid until Token is reachable
 func (t *Token) ExtraFilters() []string {
-	return t.body.VMAccess.ExtraFilters
+	return t.body.vmAccessClaim.ExtraFilters
 }
 
-func (t *Token) VMAccess() *VMAccessClaim {
-	return t.body.VMAccess
-}
-
-func parseJWTHeader(data string) (*header, error) {
-	var jh header
-	decoded, err := decodeB64([]byte(data))
-	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt header as b64: %w", err)
-	}
-	if err := json.Unmarshal(decoded, &jh); err != nil {
-		return nil, fmt.Errorf("cannot parse jwt header: %w", err)
-	}
-	return &jh, nil
-}
-
-func parseJWTBody(data string) (*body, error) {
-	type tbody struct {
-		// expired at time unix_ts
-		Exp int64 `json:"exp"`
-		// issued at time unix_ts
-		Iat   int64           `json:"iat"`
-		Jti   string          `json:"jti,omitempty"`
-		Scope json.RawMessage `json:"scope,omitempty"`
-		// store as raw message to support different types
-		VMAccess *json.RawMessage `json:"vm_access"`
-	}
-	var tb tbody
-
-	decoded, err := decodeB64([]byte(data))
-	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
-	}
-	if err := json.Unmarshal(decoded, &tb); err != nil {
-		return nil, fmt.Errorf("cannot parse jwt body: %w", err)
-	}
-
-	if tb.VMAccess == nil {
-		return nil, ErrVMAccessFieldMissing
-	}
-
-	// some IDPs encode custom claims as a string
-	// try parsing as an object and fallback to a string
-	var a VMAccessClaim
-	if err := json.Unmarshal(*tb.VMAccess, &a); err != nil {
-		var s string
-		if err := json.Unmarshal(*tb.VMAccess, &s); err != nil {
-			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
-		}
-
-		if err := json.Unmarshal([]byte(s), &a); err != nil {
-			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
-		}
-	}
-
-	// some IDPs encode scope as a string and some as an array
-	var scope string
-	if tb.Scope != nil {
-		if err := json.Unmarshal(tb.Scope, &scope); err != nil {
-			var scopeSlice []string
-			if err := json.Unmarshal(tb.Scope, &scopeSlice); err != nil {
-				return nil, fmt.Errorf("cannot parse jwt body scope: %w", err)
-			}
-			scope = strings.Join(scopeSlice, " ")
-		}
-	}
-
-	parsedBody := &body{
-		Exp:      tb.Exp,
-		Iat:      tb.Iat,
-		Jti:      tb.Jti,
-		Scope:    scope,
-		VMAccess: &a,
-	}
-	return parsedBody, nil
-}
-
-func decodeB64(data []byte) ([]byte, error) {
+func decodeB64(dst []byte, src string) ([]byte, error) {
+	data := bytesutil.ToUnsafeBytes(src)
 	idx := bytes.IndexAny(data, "+/")
 	// slow path, std base64 encoding convert it to url encoding
+	// it could be encoded with standard Base64 (+/) instead of Base64URL (-_).
 	if idx >= 0 {
+		// make a copy of provided input, src cannot be modified by parser
+		bb := decodeb64BufferPool.Get()
+		defer decodeb64BufferPool.Put(bb)
+		b := bb.B[:0]
+		b = append(b, data...)
+		data = b
 		for idx, c := range data {
 			switch c {
 			case '+':
@@ -292,12 +578,94 @@ func decodeB64(data []byte) ([]byte, error) {
 				data[idx] = '_'
 			}
 		}
-
 	}
-	dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(data)))
+	dst = bytesutil.ResizeNoCopyNoOverallocate(dst, base64.RawURLEncoding.DecodedLen(len(data)))
 	_, err := base64.RawURLEncoding.Decode(dst, data)
 	if err != nil {
-		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
+		return nil, err
 	}
 	return dst, nil
 }
+
+// stringFromJSONValue is a helper with missing String parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func stringFromJSONValue(jv *fastjson.Value, key string) (string, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return "", nil
+	}
+	b, err := jvInner.StringBytes()
+	if err != nil {
+		return "", fmt.Errorf("unexpected non-string value for key=%q: %w", key, err)
+	}
+
+	return bytesutil.ToUnsafeString(b), nil
+}
+
+// uint32FromJSONValue is a helper for missing Uint32 parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func uint32FromJSONValue(jv *fastjson.Value, key string) (uint32, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return 0, nil
+	}
+	u64, err := jvInner.Uint64()
+	if err != nil {
+		return 0, fmt.Errorf("unexpected non-uint32 value for key=%q: %w", key, err)
+	}
+	if u64 > math.MaxUint32 {
+		return 0, fmt.Errorf("value cannot exceed uint32 for key=%q", key)
+	}
+
+	return uint32(u64), nil
+}
+
+// int32FromJSONValue is a helper for missing Int32 parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func int32FromJSONValue(jv *fastjson.Value, key string) (int32, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return 0, nil
+	}
+	i64, err := jvInner.Int64()
+	if err != nil {
+		return 0, fmt.Errorf("unexpected non-int32 value for key=%q: %w", key, err)
+	}
+	if i64 > math.MaxInt32 || i64 < math.MinInt32 {
+		return 0, fmt.Errorf("value cannot exceed int32 for key=%q", key)
+	}
+
+	return int32(i64), nil
+}
+
+// stringSliceFromJSONValue is a helper for missing StringArray parse method from fastjson package
+//
+// If key is required, perform check with Exists() call
+func stringSliceFromJSONValue(dst []string, jv *fastjson.Value, key string) ([]string, error) {
+	jvInner := jv.Get(key)
+	if jvInner == nil {
+		return dst, nil
+	}
+	if jvInner.Type() != fastjson.TypeArray {
+		return nil, fmt.Errorf("unexpected type for key=%q, got: %s, want: array string", key, jvInner.Type())
+	}
+	for _, ef := range jvInner.GetArray() {
+		if ef == nil {
+			continue
+		}
+		efs, err := ef.StringBytes()
+		if err != nil {
+			return nil, fmt.Errorf("unexpected non string array[] type for key=%q: %w", key, err)
+		}
+		dst = append(dst, bytesutil.ToUnsafeString(efs))
+
+	}
+	return dst, nil
+}
+
+var parserPool fastjson.ParserPool
+
+var decodeb64BufferPool bytesutil.ByteBufferPool

lib/jwt/jwt_test.go

@@ -17,54 +17,97 @@ func TestParseJWTHeader_Failure(t *testing.T) {
 			base64.RawURLEncoding.Encode(encoded, []byte(data))
 			data = string(encoded)
 		}
-		if _, err := parseJWTHeader(data); err != nil {
+		var h header
+		if err := h.parse(data); err != nil {
 			if err.Error() != expectedErr {
-				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+				t.Fatalf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
 			}
 		} else {
-			t.Errorf("expecting non-nil error")
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid input
 	f(
 		`bad input`,
-		`cannot decode jwt header as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		`illegal base64 data at input byte 3`,
 		false,
 	)
 
 	// invalid b644
 	f(
 		`YmFk`,
-		`cannot parse jwt header: invalid character 'b' looking for beginning of value`,
+		`cannot parse JSON: cannot parse number: unexpected char: "b"; unparsed tail: "bad"`,
 		false,
 	)
 
 	// invalid header json
 	f(`{]`,
-		`cannot parse jwt header: invalid character ']' looking for beginning of object key string`,
+		`cannot parse JSON: cannot parse object: cannot find opening '"" for object key; unparsed tail: "]"`,
 		true,
 	)
 
 	// invalid header type json
 	f(`[]`,
-		`cannot parse jwt header: json: cannot unmarshal array into Go value of type jwt.header`,
+		`unexpected non json object {} type: "array"`,
+		true,
+	)
+
+	// alg field is not a string
+	f(
+		`{"alg": 123, "typ": "JWT", "kid": "key-1"}`,
+		`unexpected non-string value for key="alg": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// typ field is not a string
+	f(
+		`{"alg": "RS256", "typ": 123, "kid": "key-1"}`,
+		`unexpected non-string value for key="typ": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// kid field is not a string
+	f(
+		`{"alg": "RS256", "typ": "JWT", "kid": 123}`,
+		`unexpected non-string value for key="kid": value doesn't contain string; it contains number`,
+		true,
+	)
+
+	// standard Base64 with + character (slow path in decodeB64)
+	f(
+		`{"alg": "RS256", "typ": "JWT/"}`,
+		`illegal base64 data at input byte 0`,
+		false,
+	)
+
+	// invalid header type json
+	f(`[]`,
+		`unexpected non json object {} type: "array"`,
 		true,
 	)
 }
 
 func TestParseJWTHeader_Success(t *testing.T) {
-	f := func(data string, expected *header) {
+	f := func(data string, expected header) {
 		t.Helper()
 		encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
 		encoded := make([]byte, encodedLen)
 		base64.RawURLEncoding.Encode(encoded, []byte(data))
-		header, err := parseJWTHeader(string(encoded))
+		var h header
+		err := h.parse(string(encoded))
 		if err != nil {
 			t.Fatalf("parseJWTHeader() error: %s", err)
 		}
-		if !reflect.DeepEqual(header, expected) {
-			t.Fatalf("unexpected token header;\ngot\n%v\nwant\n%v", header, expected)
+
+		if h.Alg != expected.Alg {
+			t.Fatalf("unexpected Alg:\ngot\n%s\nwant\n%s", h.Alg, expected.Alg)
+		}
+		if h.Typ != expected.Typ {
+			t.Fatalf("unexpected Typ:\ngot\n%s\nwant\n%s", h.Typ, expected.Typ)
+		}
+		if h.Kid != expected.Kid {
+			t.Fatalf("unexpected Kid:\ngot\n%s\nwant\n%s", h.Kid, expected.Kid)
 		}
 	}
 
@@ -77,7 +120,7 @@ func TestParseJWTHeader_Success(t *testing.T) {
 			"alg": %q,
 			"kid": "test"
 		}`, supportedAlgorithms[i]),
-			&header{
+			header{
 				Alg: supportedAlgorithms[i],
 				Kid: "test",
 			},
@@ -94,40 +137,41 @@ func TestParseJWTBody_Failure(t *testing.T) {
 			base64.RawURLEncoding.Encode(encoded, []byte(data))
 			data = string(encoded)
 		}
-		if _, err := parseJWTBody(data); err != nil {
+		var b body
+		if err := b.parse(data); err != nil {
 			if err.Error() != expectedErr {
-				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+				t.Fatalf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
 			}
 		} else {
-			t.Errorf("expecting non-nil error")
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid input
 	f(
 		`bad input`,
-		`cannot decode jwt body as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		`illegal base64 data at input byte 3`,
 		false,
 	)
 
 	// invalid b644
 	f(
 		`YmFk`,
-		`cannot parse jwt body: invalid character 'b' looking for beginning of value`,
+		`cannot parse JSON: cannot parse number: unexpected char: "b"; unparsed tail: "bad"`,
 		false,
 	)
 
 	// invalid body json
 	f(
 		`{]`,
-		`cannot parse jwt body: invalid character ']' looking for beginning of object key string`,
+		`cannot parse JSON: cannot parse object: cannot find opening '"" for object key; unparsed tail: "]"`,
 		true,
 	)
 
 	// invalid body type json
 	f(
 		`[]`,
-		`cannot parse jwt body: json: cannot unmarshal array into Go value of type jwt.tbody`,
+		"missing `vm_access` claim",
 		true,
 	)
 
@@ -141,7 +185,7 @@ func TestParseJWTBody_Failure(t *testing.T) {
 	// vm_access claim invalid type
 	f(
 		`{"vm_access": 123}`,
-		"cannot parse jwt body vm_access: json: cannot unmarshal number into Go value of type string",
+		"unexpected type for `vm_access` field; got: \"number\", want object {}",
 		true,
 	)
 
@@ -155,14 +199,14 @@ func TestParseJWTBody_Failure(t *testing.T) {
 	// invalid vm_access: account_id type mismatch
 	f(
 		`{"vm_access": {"tenant_id": {"account_id": "1", "project_id": 5}}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-int32 value for key="account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid vm_access: project_id type mismatch
 	f(
 		`{"vm_access": {"tenant_id": {"account_id": 1, "project_id": "5"}}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-int32 value for key="project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
@@ -180,7 +224,7 @@ func TestParseJWTBody_Failure(t *testing.T) {
 		}
 	}
 }`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		"cannot parse `extra_labels` field: value doesn't contain object; it contains array",
 		true,
 	)
 
@@ -195,70 +239,70 @@ func TestParseJWTBody_Failure(t *testing.T) {
 		}
 	}
 }`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non string array[] type for key="extra_filters": value doesn't contain string; it contains object`,
 		true,
 	)
 
 	// invalid exp claim value type
 	f(
 		`{"exp": "1610976189", "vm_access": {}}`,
-		`cannot parse jwt body: json: cannot unmarshal string into Go struct field tbody.exp of type int64`,
+		"cannot parse `exp` field: value doesn't contain number; it contains string",
 		true,
 	)
 
 	// invalid metrics metrics_account_id claim value type
 	f(
 		`{"vm_access": {"metrics_account_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="metrics_account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics metrics_project_id claim value type
 	f(
 		`{"vm_access": {"metrics_project_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="metrics_project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics metrics_extra_labels claim value type
 	f(
 		`{"vm_access": {"metrics_extra_labels": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="metrics_extra_labels", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics metrics_extra_filters claim value type
 	f(
 		`{"vm_access": {"metrics_extra_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="metrics_extra_filters", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics logs_account_id claim value type
 	f(
 		`{"vm_access": {"logs_account_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="logs_account_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics logs_project_id claim value type
 	f(
 		`{"vm_access": {"logs_project_id": "1"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected non-uint32 value for key="logs_project_id": value doesn't contain number; it contains string`,
 		true,
 	)
 
 	// invalid metrics logs_extra_filters claim value type
 	f(
 		`{"vm_access": {"logs_extra_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="logs_extra_filters", got: string, want: array string`,
 		true,
 	)
 
 	// invalid metrics logs_extra_stream_filters claim value type
 	f(
 		`{"vm_access": {"logs_extra_stream_filters": "aString"}}`,
-		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		`unexpected type for key="logs_extra_stream_filters", got: string, want: array string`,
 		true,
 	)
 }
@@ -271,7 +315,8 @@ func TestParseJWTBody_Success(t *testing.T) {
 		encoded := make([]byte, encodedLen)
 		base64.RawURLEncoding.Encode(encoded, []byte(data))
 
-		result, err := parseJWTBody(string(encoded))
+		var result body
+		err := result.parse(string(encoded))
 		if err != nil {
 			t.Fatalf("parseJWTBody() error: %s", err)
 		}
@@ -287,22 +332,22 @@ func TestParseJWTBody_Success(t *testing.T) {
 		if result.Jti != resultExpected.Jti {
 			t.Fatalf("unexpected jti; got %q; want %q", result.Jti, resultExpected.Jti)
 		}
-		if !reflect.DeepEqual(result.VMAccess.Tenant, resultExpected.VMAccess.Tenant) {
-			t.Fatalf("unexpected tenant; got %v; want %v", result.VMAccess.Tenant, resultExpected.VMAccess.Tenant)
+		if !reflect.DeepEqual(result.vmAccessClaim.Tenant, resultExpected.vmAccessClaim.Tenant) {
+			t.Fatalf("unexpected tenant; got %v; want %v", result.vmAccessClaim.Tenant, resultExpected.vmAccessClaim.Tenant)
 		}
-		if !reflect.DeepEqual(result.VMAccess.Labels, resultExpected.VMAccess.Labels) {
-			t.Fatalf("unexpected labels; got %v; want %v", result.VMAccess.Labels, resultExpected.VMAccess.Labels)
+		if !reflect.DeepEqual(result.vmAccessClaim.Labels, resultExpected.vmAccessClaim.Labels) {
+			t.Fatalf("unexpected labels; got %v; want %v", result.vmAccessClaim.Labels, resultExpected.vmAccessClaim.Labels)
 		}
-		if !reflect.DeepEqual(result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters) {
-			t.Fatalf("unexpected extra_filters; got %v; want %v", result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters)
+		if !reflect.DeepEqual(result.vmAccessClaim.ExtraFilters, resultExpected.vmAccessClaim.ExtraFilters) {
+			t.Fatalf("unexpected extra_filters; got %v; want %v", result.vmAccessClaim.ExtraFilters, resultExpected.vmAccessClaim.ExtraFilters)
 		}
 	}
 
 	f(`{"vm_access": {}}`, &body{
-		VMAccess: &VMAccessClaim{},
+		vmAccessClaim: VMAccessClaim{},
 	})
 	f(`{"vm_access": {"tenant_id": {}}}`, &body{
-		VMAccess: &VMAccessClaim{},
+		vmAccessClaim: VMAccessClaim{},
 	})
 
 	f(
@@ -316,7 +361,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
@@ -336,10 +381,10 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
-				Labels: Labels{
-					"project": "dev",
-					"team":    "mobile",
+			vmAccessClaim: VMAccessClaim{
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
@@ -356,7 +401,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				ExtraFilters: []string{
 					`{project="dev"}`,
 					`{team=~"mobile"}`,
@@ -384,14 +429,14 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: Labels{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{
 					`{project="dev"}`,
@@ -411,11 +456,27 @@ func TestParseJWTBody_Success(t *testing.T) {
     "vm_access": {}
 }`,
 		&body{
-			Exp:      1610976189,
-			Iat:      1610975889,
-			Jti:      "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
-			Scope:    "openid email profile",
-			VMAccess: &VMAccessClaim{},
+			Exp:   1610976189,
+			Iat:   1610975889,
+			Jti:   "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+			Scope: "openid email profile",
+		},
+	)
+	// scope as []string
+	f(
+		`
+{
+    "exp": 1610976189,
+    "iat": 1610975889,
+    "jti": "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+    "scope": ["openid","email","profile"],
+    "vm_access": {}
+}`,
+		&body{
+			Exp:   1610976189,
+			Iat:   1610975889,
+			Jti:   "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+			Scope: "openid email profile",
 		},
 	)
 
@@ -436,7 +497,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				MetricsAccountID: 1,
 				MetricsProjectID: 5,
 				MetricsExtraLabels: []string{
@@ -466,7 +527,7 @@ func TestParseJWTBody_Success(t *testing.T) {
     }
 }`,
 		&body{
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				LogsAccountID: 1,
 				LogsProjectID: 5,
 				LogsExtraFilters: []string{
@@ -521,8 +582,18 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewTokenFromRequest() error: %s", err)
 		}
-		if !reflect.DeepEqual(result.body.VMAccess, resultExpected.body.VMAccess) {
-			t.Fatalf("unexpected token body VMAccess;\ngot\n%v\nwant\n%v", result.body.VMAccess, resultExpected.body.VMAccess)
+		// assign nil values to simplify equal check below
+		result.header.buf = nil
+		result.header.p = nil
+		result.body.vmAccessClaim.labelsBuf = nil
+		if result.body.Iat != resultExpected.body.Iat {
+			t.Fatalf("unexpected iat: %d;%d", result.body.Iat, resultExpected.body.Iat)
+		}
+		if result.body.Exp != resultExpected.body.Exp {
+			t.Fatalf("unexpected exp: %d;%d", result.body.Exp, resultExpected.body.Exp)
+		}
+		if !reflect.DeepEqual(result.body.vmAccessClaim, resultExpected.body.vmAccessClaim) {
+			t.Fatalf("unexpected token body VMAccess;\ngot\n%v\nwant\n%v", result.body.vmAccessClaim, resultExpected.body.vmAccessClaim)
 		}
 		if !reflect.DeepEqual(result.header, resultExpected.header) {
 			t.Fatalf("unexpected token header\ngot\n%v\nwant\n%v", result.header, resultExpected.header)
@@ -538,23 +609,23 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected := &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -571,23 +642,23 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -604,8 +675,10 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			VMAccess: &VMAccessClaim{
+		body: body{
+			Iat: 1645536638,
+			Exp: 1645536758,
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 0,
 					AccountID: 1,
@@ -616,7 +689,7 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 				Mode: 1,
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Typ: "JWT",
 		},
@@ -632,8 +705,10 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			VMAccess: &VMAccessClaim{
+		body: body{
+			Iat: 1645606878,
+			Exp: 1645606998,
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 0,
 					AccountID: 1,
@@ -644,7 +719,7 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 				Mode: 1,
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Typ: "JWT",
 		},
@@ -660,24 +735,24 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -694,24 +769,24 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 		},
 	}
 	resultExpected = &Token{
-		body: &body{
-			Exp:   1610889266,
-			Iat:   1610888966,
+		body: body{
+			Exp:   1610976189,
+			Iat:   1610975889,
 			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
 			Scope: "openid email profile",
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
-				Labels: map[string]string{
-					"project": "dev",
-					"team":    "mobile",
+				Labels: []string{
+					"project=dev",
+					"team=mobile",
 				},
 				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "HS256",
 			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
 			Typ: "JWT",
@@ -729,17 +804,17 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 	}
 
 	resultExpected = &Token{
-		body: &body{
+		body: body{
 			Exp: 1725629232,
 			Iat: 1725625332,
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
 			Typ: "JWT",
@@ -757,17 +832,17 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
 	}
 
 	resultExpected = &Token{
-		body: &body{
+		body: body{
 			Exp: 1725629232,
 			Iat: 1725625332,
-			VMAccess: &VMAccessClaim{
+			vmAccessClaim: VMAccessClaim{
 				Tenant: TenantID{
 					ProjectID: 5,
 					AccountID: 1,
 				},
 			},
 		},
-		header: &header{
+		header: header{
 			Alg: "RS256",
 			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
 			Typ: "JWT",

lib/jwt/jwt_timing_test.go

@@ -0,0 +1,38 @@
+package jwt
+
+import "testing"
+
+func BenchmarkTokenParse(t *testing.B) {
+	f := func(name string, rawToken string) {
+		t.Helper()
+
+		t.Run(name, func(t *testing.B) {
+			t.ReportAllocs()
+			t.RunParallel(func(pb *testing.PB) {
+				var jt Token
+				for pb.Next() {
+					jt.Reset()
+					if err := jt.Parse(rawToken, true); err != nil {
+						t.Fatalf("unexpected parsing error: %s", err)
+					}
+				}
+			})
+
+		})
+	}
+
+	// simple token with only tenant_id
+	f("simple", `Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxOTAsImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9fSwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoidGcgdGciLCJwcm9qZWN0IjoibW9iaWxlIiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGciLCJ0ZWFtIjoiZGV2IiwiZ2l2ZW5fbmFtZSI6InRnIiwiZmFtaWx5X25hbWUiOiJ0ZyIsImVtYWlsIjoidGdAZmdodC5uZXQifQ.mpT7_kGOIZtoRv2Tn-_80YXmy7_3Qc4_xeaQr1Nhk4UyXSeWh6HB96wWkBS8Jhj3NksGj7bqxezOEbOBBaqlYn6cdGV2hVZ8GKT2zt6oRLCuuUORiRU1joBeIhVRMNtXvPXLFTs4e1VIKejncWbeKmXSneYCjJityixQza0mVyO7ldiXHc6J2f_wQJDPkwkFJJvfwwTbyu4maUzv5gNIvVSUfWnjPq3skFmnjwpsfD9KZnZg-pPTKUmri6kdK0YrFTGA5HT_DM77UkXzsDSMdHPP5tgiPD3LeK75djTZdMAidX53ai85BDn9d5vzi9nfoVezyN3dh0xqqaaQJGqjng`)
+
+	// gateway extra labels and extra filters
+	f("gateway labels and filters", `Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxOTAsImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9LCJleHRyYV9sYWJlbHMiOnsiZW52IjoicHJvZCIsInRlYW0iOiJvcHMifSwiZXh0cmFfZmlsdGVycyI6WyJtZXRyaWMiLCJ7c2VsZWN0b3I9XCJ2YWx1ZVwiIl19LCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsIm5hbWUiOiJ0ZyB0ZyIsInByb2plY3QiOiJtb2JpbGUiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJ0ZyIsInRlYW0iOiJkZXYiLCJnaXZlbl9uYW1lIjoidGciLCJmYW1pbHlfbmFtZSI6InRnIiwiZW1haWwiOiJ0Z0BmZ2h0Lm5ldCJ9.lUEn5nVQ6Trra-9YkbMKyhL0eiWmL2VSIKj2HDQSH43ZkeagLPQbPTnLYfkbuc1sI9tPcyFOPuwgdEAkckEgQ7szvw9g5bLrtT4etWoOnPJ1GaQpcn0z16w7bAgbMf8rpb0i4JMOXicRd7ARlkjyJZDjehaVUX726052qv2NG7npShafK0wei1QBpD3N34TJlqixbOnD1DCfsorwxzba8OuwgQI8lfTHWmgFO0611DGKZb1a-srTPZ5ziZ29NhtDAkbx6bZnYHMp_8CTLD6p0z34RM2wPWyI_2_AdKDbqkdDSZoapJneQDdoNsmMA0IUFETqBgfRTavnApkgeu12HA`)
+
+	// scope as []string
+	f("scope as slice string", `Bearer ewogICJ0eXAiOiJKV1QiLAogICJhbGciOiJSUzI1NiIsCiAgImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyIKfQ.ewogICJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLAogICJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLAogICJpYXQiOjE3MjU2MjUzMzIsCiAgIm5iZiI6MTcyNTYyNTMzMiwKICAiZXhwIjoxNzI1NjI5MjMyLAogICJuYW1lIjoiWmFraGFyIEJlc3NhcmFiIiwKICAib2lkIjoiOGI5ZWY2YjMtMWMwMS00YjczLTg0ODItMjRkNmI2NTE1Y2U0IiwKICAicHJlZmVycmVkX3VzZXJuYW1lIjoiei5iZXNzYXJhYkB2aWN0b3JpYW1ldHJpY3MuY29tIiwKICAicmgiOiIwLkFXTUJ6aDdhSlpGbWFFaW5leHNQbTc1ZlEtY1JjM3AtdWw1Sm1WNnZVMDV6d1JCakFaby4iLAogICJzdWIiOiJXRld3QTlYZjZpZXUxLUgwNDBuU0QxRVo3UWxOLTVHbWxob2p4czdMUFJRIiwKICAidGlkIjoiMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzIiwKICAidXRpIjoidlo1MjQySmhNVWFUUktaYVFCRjhBQSIsCiAgInZlciI6IjIuMCIsCiAgInZtX2FjY2VzcyI6IntcInRlbmFudF9pZFwiOntcInByb2plY3RfaWRcIjogNSwgXCJhY2NvdW50X2lkXCI6IDF9fSIsCiAgInNjb3BlIjogWyJvcGVuaWQiLCAidm0iXQp9.ZXdvZ0lDSjBlWEFpT2lKS1YxUWlMQW9nSUNKaGJHY2lPaUpTVXpJMU5pSXNDaUFnSW10cFpDSTZJa2c1Ym1vMVFVOVRjM2ROY0dobk1WTkdlRGRxWVZZdGJFSTVkeUlLZlEuLktrUG9qNWJoaDNWcnRyY3RVb0lHaE5vN2hNc2VGT3hESGVEQ2g3MFViV2l2LU5pb1Zia2duZk1CMkhacHN6WGU5WmNmX2FIaURJSVNTYkNTaDlvQnF1aS02OEJDcmplNFJWRkpGZFV6R3V1SmdOTS11YVpBcFJqSFNNZDUxb2RvbHFoUGFHS09URnJXVmlIWlpfVDdXaVNUcV84U3Y1a2x1Y2xMb0hEcU82MU5Na2w0TmRCVnQxM1hjRTBfM243U3VxTDdpaks2dGMwZ2NzcmJ5c3JNdl9jd2VRamZsLU5fV0N0SG40NnhadEhvX0RpZERabzc2TjV1NE52Uk1OZUxNcXZ0YTgzUzhPdzNyUUlhaUFjUUNHYjBqUU5hV2VEQlFzZUZ6SjRyR0h6RjAwZDlqVkNCSHVWRmI5eHNnSnJVUDZ0S05iT2hTeEY1RzBocElVYk5OUQ`)
+
+	// vm_access string
+	f("access claim string", `Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyJ9.eyJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLCJpc3MiOiJodHRwczovLzFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLCJpYXQiOjE3MjU2MjUzMzIsIm5iZiI6MTcyNTYyNTMzMiwiZXhwIjoxNzI1NjI5MjMyLCJvaWQiOiIwMDAwMC0xYzAxLTRiNzMtODQ4Mi0yNGQ2YjY1MTVjZTQiLCJ0aWQiOiIwMC02NjkxLTQ4NjgtYTc3Yi0xYjBmOWJiZTVmNDMiLCJ1dGkiOiJ2WjUyNDJKaE1VYVRSS1phUUJGOEFBIiwidmVyIjoiMi4wIiwidm1fYWNjZXNzIjoie1widGVuYW50X2lkXCI6e1wicHJvamVjdF9pZFwiOiA1LCBcImFjY291bnRfaWRcIjogMX19In0.RYYL-Ct-a3dlToRCemUCDbnY_HIFeJ1Feqzj6yXcchy_VtE0DjGu-qGspwPHsJe_JlgHSegN_wSlCLAuorO4vQxIVYansL-6AOQ8fiAh_HRA1dID6lvmxYIkCxNFIEyc7ufp7QJYZiyT_lKJkDOrXqWuJ5l_ajLVRSGK1kWRL0V_e6BsU8-2NF_f1gkPEpULooHmQfpdNszZwPpN_Hyd24gQmSbTZk1MA1jkuo6LLuMDyZK2UDnRQA3Xx480LYnl-VzlBLwv5fwEGFwOJC_E9olvAJxr8eYJEQA4lwsdpwmfJkWBlrdcOZNHzmNaTWMFxmDIBOirH-CUm9ndF2r-Og`)
+
+	// vmauth related claim fields
+	f("vmauth related fields", `Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyJ9.eyJhdWQiOiIwMDAwMC1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdC92Mi4wIiwiaWF0IjoxNzI1NjI1MzMyLCJuYmYiOjE3MjU2MjUzMzIsImV4cCI6MTcyNTYyOTIzMiwidmVyIjoiMi4wIiwidm1fYWNjZXNzIjp7Im1ldHJpY3NfYWNjb3VudF9pZCI6MTAwLCJtZXRyaWNzX3Byb2plY3RfaWQiOjEwMDA1LCJtZXRyaWNzX2V4dHJhX2ZpbHRlcnMiOlsie2ZpbHRlcj1cIjFcIn0iLCJ7ZmlsdGVyMj1cIjJcIn0iXSwibWV0cmljc19leHRyYV9sYWJlbHMiOlsia2V5PXZhbHVlIiwib3RoZXJfbGFiZWw9dmFsdWUiXSwibG9nc19hY2NvdW50X2lkIjo1MDAsImxvZ3NfcHJvamVjdF9pZCI6NTU1NSwibG9nc19leHRyYV9maWx0ZXJzIjpbImZpbHRlcj12YWx1ZSIsIm90aGVyX2ZpbGVyIl0sImxvZ3NfZXh0cmFfc3RyZWFtX2ZpbHRlcnMiOlsic3RyZWFtIGZpbHRlciIsIm90aGVyIHN0cmVhbSBmaWx0ZXIiLCJsYXN0IHN0cmVhbSBmaWx0ZXIiXX0sInNjb3BlIjoib3BlbmlkIn0.SVRbfypXpzJ1FL2ALu9_iO_J_UXTS0MiUX4SJ8ZqmN-JAsR8SudJAe1Lk8uubTsRtb234a8QYuzR1XhMLwM6SDkuioKC2VAGPV2YPb5Z7axv0juShJfZkaBaqf-zz_bx51-Bop6Xlpg5zySymYs9mLRwGKfIKMiIZVF5d0mDnG-BUawstQZX3RvVWODrLucIPiuJy9ry_tQz1uYbL8eadeqezfAPprB-bxGSScZ4SKeSW9j3wksB2zvAidlj5ZMnmkDRcXCkBgBxazQ0KeHXPly8kkC4yREtZiBCVz1HKsCncO-iWR2DFCf5jLwHiJVuwsTIjj7jdb9Hxgiu_CS3eA`)
+}

lib/promrelabel/debug.go

@@ -35,12 +35,6 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	labels, err := promutil.NewLabelsFromString(metric)
-	if err != nil {
-		err = fmt.Errorf("cannot parse metric: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
-		return
-	}
 	pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
 	if err != nil {
 		err = fmt.Errorf("cannot parse relabel configs: %w", err)
@@ -48,8 +42,33 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
-	WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
+	// metric input may contain multiple lines, and it's good to add support and print the debug steps for each of them.
+	// however, the target URL won't change
+	metrics := strings.Split(metric, "\n")
+	dsss := make([][]DebugStep, 0, len(metrics))
+	var lastErr error
+
+	for _, m := range metrics {
+		// try to trim all the special characters and space first
+		m = strings.TrimSpace(m)
+		labels, err := promutil.NewLabelsFromString(m)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		dss, tURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
+		targetURL = tURL
+		dsss = append(dsss, dss)
+	}
+
+	// break here if all failed
+	if len(dsss) == 0 && lastErr != nil {
+		err = fmt.Errorf("cannot parse metric: %w", err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, lastErr)
+		return
+	}
+
+	WriteRelabelDebugSteps(w, targetURL, targetID, format, dsss, metric, relabelConfigs, nil)
 }
 
 func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {

lib/promrelabel/debug.qtpl

@@ -6,15 +6,15 @@
 
 {% stripspace %}
 
-{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs  string, err error) %}
+{% func RelabelDebugSteps(targetURL, targetID, format string, dsss [][]DebugStep, metric string, relabelConfigs  string, err error) %}
   {% if format == "json" %}
-      {%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsJSON(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% else %}
-      {%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsHTML(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsHTML(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -60,11 +60,13 @@ function submitRelabelDebugForm(e) {
         </form>
         </div>
 
+        {% for _, dss := range dsss %}
         <div class="row">
             <main class="col-12">
               {%= relabelDebugSteps(dss, targetURL, targetID) %}
             </main>
         </div>
+        {% endfor %}
     </div>
 </body>
 </html>
@@ -77,7 +79,7 @@ function submitRelabelDebugForm(e) {
     </div>
 
     <div>
-        Labels:<br/>
+        Labels (One label set per line. Multiple lines will be treated as multiple label sets):<br/>
         <textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
     </div>
 {% endfunc %}
@@ -151,45 +153,52 @@ function submitRelabelDebugForm(e) {
 {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsJSON(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 {
     {% if err != nil %}
         "status": "error",
         "error": {%q= fmt.Sprintf("Error: %s", err) %}
     {% else %}
-        {% code var hasError bool %}
         "status": "success",
-        "steps": [
-            {% for i, ds := range dss %}
-                {% code
-                    inLabels, inErr := promutil.NewLabelsFromString(ds.In)
-                    outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
-                    changedLabels := getChangedLabelNames(inLabels, outLabels)
-                %}
+        "metrics": [
+            {% for idx, dss := range dsss %}
+                {% code var hasError bool %}
                 {
-                    "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
-                    "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
-                    "rule": {%q= ds.Rule %},
-                    "errors": {
-                        {% if inErr != nil %}
-                        "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
-                        {%code hasError = true %}
-                        {% else %}
-                        {% endif %}
-                        {% if outErr != nil %}
-                        "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
-                        {%code hasError = true %}
-                        {% endif %}
-                    }
+                    "steps": [
+                        {% for i, ds := range dss %}
+                            {% code
+                                inLabels, inErr := promutil.NewLabelsFromString(ds.In)
+                                outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
+                                changedLabels := getChangedLabelNames(inLabels, outLabels)
+                            %}
+                            {
+                                "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
+                                "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
+                                "rule": {%q= ds.Rule %},
+                                "errors": {
+                                    {% if inErr != nil %}
+                                    "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
+                                    {%code hasError = true %}
+                                    {% else %}
+                                    {% endif %}
+                                    {% if outErr != nil %}
+                                    "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
+                                    {%code hasError = true %}
+                                    {% endif %}
+                                }
+                            }
+                            {% if i != len(dss)-1 %},{% endif %}
+                        {% endfor %}
+                    ]
+                    {% if len(dss) > 0 && !hasError %}
+                    ,
+                      "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
+                      "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
+                    {% endif %}
                 }
-                {% if i != len(dss)-1 %},{% endif %}
+                {% if idx != len(dsss)-1 %},{% endif %}
             {% endfor %}
         ]
-        {% if len(dss) > 0 && !hasError %}
-        ,
-          "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
-          "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
-        {% endif %}
     {% endif %}
 }
 {% endfunc %}

lib/promscrape/relabel_debug.go

@@ -3,7 +3,9 @@ package promscrape
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -23,9 +25,39 @@ func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
 			targetID = ""
 		} else {
 			metric = labels.String()
-			relabelConfigs = pcs.String()
+			relabelConfigs += "# metrics_relabel_configs\n"
+			relabelConfigs += pcs.String()
+
+			rwRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+			rwURLRelabelConfigs := remotewrite.GetURLRelabelConfigData()
+
+			relabelConfigs += "\n# -remoteWrite.relabelConfig"
+			relabelConfigs += "\n" + rwRelabelConfigs
+
+			// we could have different relabel config for different remote write URL, but there's no way to know which one the user wants to debug.
+			// so we append the 1st one here, and comment out the rest. user can see them on the page and edit to activate them.
+			for i := range rwURLRelabelConfigs {
+				if i == 0 {
+					relabelConfigs += "\n# -remoteWrite.urlRelabelConfig"
+
+					// append the URL info
+					relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+
+					// append the relabeling config string
+					relabelConfigs += "\n" + rwURLRelabelConfigs[i].RelabelConfigStr
+					continue
+				}
+
+				// for the rest URLs add comment # before every line.
+				relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+				lines := strings.Split(rwURLRelabelConfigs[i].RelabelConfigStr, "\n")
+				for _, line := range lines {
+					relabelConfigs += "\n#" + line
+				}
+			}
 		}
 	}
+
 	if format == "json" {
 		httpserver.EnableCORS(w, r)
 		w.Header().Set("Content-Type", "application/json")