fix ai code review remarks

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -65,7 +65,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/changelog-linter.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           fetch-depth: 0  # we need full history for commit verification
           persist-credentials: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/codeql-analysis-go.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           path: __vm
           persist-credentials: false
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: 'vm-runner'
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -79,7 +79,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -106,7 +106,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/vmui.yml

@@ -26,13 +26,11 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

.golangci.yml

@@ -23,3 +23,4 @@ linters:
         text: 'SA(4003|1019|5011):'
     paths:
       - ^app/vmui/
+      - app/vmui/packages/vmui/node_modules/

Makefile

@@ -447,7 +447,7 @@ vet:
 	go vet ./app/...
 	go vet ./apptest/...
 
-check-all: fmt vet golangci-lint govulncheck
+check-all: fmt vet golangci-lint
 
 clean-checkers: remove-golangci-lint remove-govulncheck
 
@@ -471,8 +471,9 @@ test-full-386:
 
 apptest:
 	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+	go test ./apptest/... -skip="^Test(Cluster|Mixed|Legacy).*"
 
+# App tests for legacy indexDB
 apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
@@ -489,6 +490,22 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
+# App tests for mixed setups where vmsingle and vmcluster coexist.
+apptest-mixed: victoria-metrics-race
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VMSELECT_PATH=$${DIR}/vmselect-prod \
+	go test ./apptest/tests -run="^TestMixed.*"
+
 benchmark:
 	go test -run=NO_TESTS -bench=. ./lib/...
 	go test -run=NO_TESTS -bench=. ./app/...

README.md

@@ -2,7 +2,6 @@
 
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
-[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)

app/victoria-metrics/main.go

@@ -89,7 +89,7 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
-	vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()

app/vmagent/influx/request_handler.go

@@ -51,7 +51,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vmagent/main.go

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmagent/opentelemetry/request_handler.go

@@ -63,6 +63,7 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range tss {
@@ -82,7 +83,19 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	var metadataTotal int
 	if prommetadata.IsEnabled() {
-		ctx.WriteRequest.Metadata = mms
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: mm.AccountID,
+				ProjectID: mm.ProjectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
 		metadataTotal = len(mms)
 	}
 

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 }
 
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+	if len(extraLabels) == 0 && !prommetadata.IsEnabled() && at == nil {
+		return insertRowsFast(at, timeseries)
+	}
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -102,3 +105,17 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }
+
+func insertRowsFast(at *auth.Token, timeseries []prompb.TimeSeries) error {
+	rowsTotal := 0
+	for i := range timeseries {
+		rowsTotal += len(timeseries[i].Samples)
+	}
+	wr := &prompb.WriteRequest{Timeseries: timeseries}
+	if !remotewrite.TryPush(at, wr) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,

app/vmagent/remotewrite/obfuscate.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+type obfuscateLabelsCtx struct {
+	labels []prompb.Label
+
+	// buf holds allocations for cached results below
+	buf []byte
+
+	// cacheResults maps original label values to their SHA-256 hex digests,
+	// avoiding redundant hashing of repeated values within a single batch.
+	cacheResults map[string]string
+}
+
+func (olctx *obfuscateLabelsCtx) reset() {
+	promrelabel.CleanLabels(olctx.labels)
+	olctx.labels = olctx.labels[:0]
+	clear(olctx.cacheResults)
+	olctx.buf = olctx.buf[:0]
+}
+
+var obfuscateLabelsCtxPool = &sync.Pool{
+	New: func() any {
+		return &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+	},
+}
+
+func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
+	return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
+}
+
+func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
+	ctx.reset()
+	obfuscateLabelsCtxPool.Put(ctx)
+}
+
+func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
+	if len(obfuscateLabels) == 0 || len(tss) == 0 {
+		return tss
+	}
+	labels := olctx.labels
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		found := false
+		for _, labelName := range obfuscateLabels {
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
+			if tmp == nil {
+				continue
+			}
+			found = true
+			if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
+				// fast path: the obfuscated result was calculated before
+				tmp.Value = obfuscatedValue
+				continue
+			}
+
+			digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
+			buf := olctx.buf
+			bufLen := len(buf)
+			buf = hex.AppendEncode(buf, digest[:])
+			obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			olctx.buf = buf
+			olctx.cacheResults[tmp.Value] = obfuscatedValue
+			tmp.Value = obfuscatedValue
+		}
+		if found {
+			ts.Labels = labels[labelsLen:]
+		} else {
+			labels = labels[:labelsLen]
+		}
+	}
+	olctx.labels = labels
+	return tss
+}
+
+func (rwctx *remoteWriteCtx) initObfuscateLabels() {
+	if len(*obfuscateLabels) == 0 {
+		return
+	}
+	idx := rwctx.idx
+	rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
+	rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
+
+	for _, label := range rwObfuscateLabelsList {
+		if label == "" {
+			continue
+		}
+		if !slices.Contains(rwctx.obfuscateLabels, label) {
+			rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
+		}
+	}
+}

app/vmagent/remotewrite/obfuscate_test.go

@@ -0,0 +1,186 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func TestRemoteWriteObfuscateLabels(t *testing.T) {
+	f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
+		t.Helper()
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		defer metrics.UnregisterAllMetrics()
+		originValue := *obfuscateLabels
+		defer func() {
+			*obfuscateLabels = originValue
+		}()
+		*obfuscateLabels = []string{obfuscateLabelList}
+		rwctx.initObfuscateLabels()
+
+		outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
+
+		if !reflect.DeepEqual(expectedTss, outputTss) {
+			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
+		}
+	}
+
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+
+	// 1. obfuscation is not set.
+	f("",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 1. obfuscation is set for another rwctx.
+	f(",ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 2. obfuscate the value of "ip" label
+	f("ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 3. obfuscate the values of "ip" and "instance"
+	f("ip^^instance",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: sha256Result("1234")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 4. duplicate label names in config must produce single SHA-256, not double
+	f("ip^^ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+}

app/vmagent/remotewrite/obfuscate_timing_test.go

@@ -0,0 +1,94 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
+	originValue := *obfuscateLabels
+	defer func() {
+		*obfuscateLabels = originValue
+	}()
+	*obfuscateLabels = []string{"ip^^instance"}
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+	expected := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("123")},
+				{Name: "instance", Value: sha256Result("12345")},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("1236")},
+				{Name: "instance", Value: sha256Result("some-long-instante-string")},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	defer metrics.UnregisterAllMetrics()
+
+	inputTss := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "123"},
+				{Name: "instance", Value: "12345"},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "1236"},
+				{Name: "instance", Value: "some-long-instante-string"},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		rwctx.initObfuscateLabels()
+		var localTss []prompb.TimeSeries
+		for pb.Next() {
+			// always make a shallow copy because obfuscate changes input
+			localTss = localTss[:0]
+			localTss = append(localTss, inputTss...)
+			olctx.reset()
+			outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
+			if !reflect.DeepEqual(expected, outputTss) {
+				b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
+			}
+		}
+	})
+
+}

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmagent/remotewrite/remotewrite.go

@@ -12,19 +12,18 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -106,6 +105,14 @@ var (
 		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
 	disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
+
+	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
+	obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
+		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
+		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
 )
 
 var (
@@ -162,8 +169,8 @@ func InitSecretFlags() {
 }
 
 var (
-	shardByURLLabelsMap       map[string]struct{}
-	shardByURLIgnoreLabelsMap map[string]struct{}
+	shardByURLLabelsFilter       []string
+	shardByURLIgnoreLabelsFilter []string
 )
 
 // Init initializes remotewrite.
@@ -210,8 +217,8 @@ func Init() {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
 	}
-	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
-	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
+	shardByURLLabelsFilter = slices.Clone(*shardByURLLabels)
+	shardByURLIgnoreLabelsFilter = slices.Clone(*shardByURLIgnoreLabels)
 
 	initLabelsGlobal()
 
@@ -307,6 +314,10 @@ func initRemoteWriteCtxs(urls []string) {
 	}
 	fs.RegisterPathFsMetrics(*tmpDataPath)
 
+	if slices.Contains(*enableMdx, true) && *shardByURL {
+		logger.Fatalf("-remoteWrite.mdx.enable and -remoteWrite.shardByURL cannot be set to true simultaneously.")
+	}
+
 	if *shardByURL {
 		consistentHashNodes := make([]string, 0, len(urls))
 		for i, url := range urls {
@@ -565,6 +576,14 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
 			mm.ProjectID = at.ProjectID
 		}
 	}
+	tmp := mms[:0]
+	for _, mm := range mms {
+		if timeserieslimits.IsMetricMetadataExceeding(&mm) {
+			continue
+		}
+		tmp = append(tmp, mm)
+	}
+	mms = tmp
 	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
 	// Since metadata is usually small and there is no guarantee that metadata can be sent to
 	// the same remote storage with the corresponding metrics.
@@ -698,18 +717,18 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 
 	for _, ts := range tssBlock {
 		hashLabels := ts.Labels
-		if len(shardByURLLabelsMap) > 0 {
+		if len(shardByURLLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLLabelsMap[label.Name]; ok {
+				if slices.Contains(shardByURLLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
 			tmpLabels.Labels = hashLabels
-		} else if len(shardByURLIgnoreLabelsMap) > 0 {
+		} else if len(shardByURLIgnoreLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
+				if !slices.Contains(shardByURLIgnoreLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
@@ -810,34 +829,26 @@ var (
 // it omits the '=' separator between label name and value for backward compatibility.
 // Changing it would re-shard all series across remoteWrite targets.
 func getLabelsHashForShard(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
 func getLabelsHash(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, '=')
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString("=")
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
-var labelsHashBufPool bytesutil.ByteBufferPool
-
 func logSkippedSeries(labels []prompb.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
@@ -862,6 +873,7 @@ type remoteWriteCtx struct {
 
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
+	mdxFilter    *mdx.Filter
 
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
@@ -874,8 +886,11 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
+	obfuscateLabels []string
+
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
+	mdxRowsPreserved       *metrics.Counter
 
 	pushFailures                 *metrics.Counter
 	metadataDroppedOnPushFailure *metrics.Counter
@@ -972,7 +987,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
-
 	rwctx := &remoteWriteCtx{
 		idx:            argIdx,
 		fq:             fq,
@@ -988,6 +1002,17 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
+	rwctx.initObfuscateLabels()
+
+	if enableMdx.GetOptionalArg(argIdx) {
+		mdxFilter := mdx.NewFilter()
+		rwctx.mdxFilter = mdxFilter
+		rwctx.mdxRowsPreserved = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_preserved_total{path=%q,url=%q}`, queuePath, sanitizedURL))
+		_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_mdx_tracked_instances{path=%q,url=%q}`, queuePath, sanitizedURL), func() float64 {
+			return float64(mdxFilter.VMInstancesCount())
+		})
+
+	}
 
 	return rwctx
 }
@@ -1002,6 +1027,11 @@ func (rwctx *remoteWriteCtx) MustStop() {
 		rwctx.deduplicator.MustStop()
 		rwctx.deduplicator = nil
 	}
+	if rwctx.mdxFilter != nil {
+		rwctx.mdxFilter.MustStop()
+		rwctx.mdxFilter = nil
+		rwctx.mdxRowsPreserved = nil
+	}
 
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
@@ -1017,6 +1047,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 	rwctx.rowsPushedAfterRelabel = nil
 	rwctx.rowsDroppedByRelabel = nil
+
 }
 
 // TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1024,16 +1055,41 @@ func (rwctx *remoteWriteCtx) MustStop() {
 // TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
 func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	var rctx *relabelCtx
+	var mctx *mdx.Ctx
 	var v *[]prompb.TimeSeries
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if mctx != nil {
+			mdx.PutContext(mctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
+	if rwctx.mdxFilter != nil {
+		mctx = mdx.GetContext()
+		// Make a copy of tss before applying relabeling in order to prevent
+		// from affecting time series for other remoteWrite.mdx configs.
+		copyTimeSeriesIfNeeded()
+		tss = rwctx.mdxFilter.Filter(mctx, tss)
+		if len(tss) == 0 {
+			return true
+		}
+		rowsCount := getRowsCount(tss)
+		rwctx.mdxRowsPreserved.Add(rowsCount)
+	}
+
 	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
@@ -1043,8 +1099,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
@@ -1062,8 +1117,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		} else if rwctx.streamAggrDropInput {
@@ -1071,8 +1125,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropUnaggregatedSeries(tss, matchIdxs.B)
 		}
@@ -1153,24 +1206,41 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
 func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
+	var olctx *obfuscateLabelsCtx
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if olctx != nil {
+			putObfuscateLabelsCtx(olctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
+	if len(rwctx.obfuscateLabels) != 0 {
+		copyTimeSeriesIfNeeded()
+		olctx = getObfuscateLabelsCtx()
+		tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
+	}
+
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
@@ -1191,15 +1261,6 @@ func getRowsCount(tss []prompb.TimeSeries) int {
 	}
 	return rowsCount
 }
-
-func newMapFromStrings(a []string) map[string]struct{} {
-	m := make(map[string]struct{}, len(a))
-	for _, s := range a {
-		m[s] = struct{}{}
-	}
-	return m
-}
-
 func getMaxHourlySeries() int {
 	limit := *maxHourlySeries
 	if limit == -1 || limit > math.MaxInt32 {

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )
@@ -282,7 +289,8 @@ func processFlags() {
 
 func setUp() {
 	const maxConcurrentRequests = 4
-	vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	maxQueueDuration := 5 * time.Second
+	vmstorage.Init(maxConcurrentRequests, maxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {

app/vmalert/config/config_test.go

@@ -145,10 +145,10 @@ func TestRuleValidate(t *testing.T) {
 }
 
 func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+	f := func(data []byte, validateExpressions bool, errStrExpected string) {
 		t.Helper()
 
-		err := group.Validate(nil, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, nil, validateExpressions)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -158,275 +158,238 @@ func TestGroupValidate_Failure(t *testing.T) {
 		}
 	}
 
-	f(&Group{}, false, "group name must be set")
+	f([]byte(`
+groups:
+- name: ""
+`), false, "group name must be set")
 
-	f(&Group{
-		Name: "both record and alert are not set",
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
+	f([]byte(`
+groups:
+- name: both record and alert are not set
+  rules:
+  - expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+  - expr: "sumSeries(time('foo.bar',10))"
+`), false, "invalid rule")
 
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutil.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
+	f([]byte(`
+groups:
+- name: negative interval
+  interval: -1ms
+`), false, "interval shouldn't be lower than 0")
 
-	f(&Group{
-		Name:       "too big eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big eval_offset
+  interval: 1m
+  eval_offset: 2m
+`), false, "eval_offset should be smaller than interval")
 
-	f(&Group{
-		Name:       "too big negative eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(-2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big negative eval_offset
+  interval: 1m
+  eval_offset: -2m
+`), false, "eval_offset should be smaller than interval")
 
-	limit := -1
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: &limit,
-	}, false, "invalid limit")
+	f([]byte(`
+groups:
+- name: wrong limit
+  limit: -1
+`), false, "invalid limit")
 
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
+	f([]byte(`
+groups:
+- name: wrong concurrency
+  concurrency: -1
+`), false, "invalid concurrency")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+  - alert: alert
+    expr: up == 1
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
+	f([]byte(`
+groups:
+- name: test thanos
+  type: thanos
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), true, "unknown datasource type")
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus expr",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus expr
+  type: prometheus
+  rules:
+  - record: record
+    expr: "up | 0"
+`), true, "bad MetricsQL expr")
 
-	f(&Group{
-		Name: "test graphite expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite expr
+  type: graphite
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: some-description
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "stats count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr multipart
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test graphite with prometheus expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				ID:     1,
-				Expr:   "sumSeries(time('foo.bar',10))",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Record: "r2",
-				ID:     2,
-				Expr:   "sum(up == 0 ) by (host)",
-			},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite with prometheus expr
+  type: graphite
+  rules:
+  - record: r1
+    expr: "sumSeries(time('foo.bar',10))"
+    for: 10ms
+  - record: r2
+    expr: "sum(up == 0 ) by (host)"
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs with prometheus exp",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "sum(up == 0 ) by (host)",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs with prometheus expr
+  type: vlogs
+  rules:
+  - record: r1
+    expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test prometheus with vlogs exp",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "* | stats by (path) count()",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus with vlogs expr
+  type: prometheus
+  rules:
+  - record: r1
+    expr: "* | stats by (path) count()"
+    for: 10ms
+`), true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+	f := func(data []byte, validateAnnotations, validateExpressions bool) {
 		t.Helper()
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
 			validateTplFn = notifier.ValidateTemplates
 		}
-		err := group.Validate(validateTplFn, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, validateTplFn, validateExpressions)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	}
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: "up | 0"
+`), false, false)
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
-				},
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, false)
 
 	// validate annotations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
-{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
-  {{ . | first | value | humanize1024 }}B
-{{ end }}`,
-				},
-			},
-		},
-	}, true, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "\n{{ with printf \"node_memory_MemTotal{job='node',instance='%s'}\" \"localhost\" | query }}\n  {{ . | first | value | humanize1024 }}B\n{{ end }}"
+`), true, false)
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
-	f(&Group{
-		Name: "test victorialogs",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
+	f([]byte(`
+groups:
+- name: test prometheus
+  type: prometheus
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
+
+	f([]byte(`
+groups:
+- name: test victorialogs
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: " _time: 1m | stats count(*) as requests"
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
 }
 
 func TestHashRule_NotEqual(t *testing.T) {

app/vmalert/datasource/client.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -267,7 +268,11 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
+		err = &httpserver.ErrorWithStatusCode{
+			StatusCode: resp.StatusCode,
+			Err:        fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
+		}
+		return nil, err
 	}
 	return resp, nil
 }

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/notifier/alert.go

@@ -31,6 +31,8 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
+	// Interval contains the evaluation interval of the Alert's group
+	Interval time.Duration
 	// ActiveAt defines the moment of time when Alert has become active
 	ActiveAt time.Time
 	// Start defines the moment of time when Alert has become firing
@@ -84,6 +86,7 @@ type AlertTplData struct {
 	Labels    map[string]string
 	Value     float64
 	Expr      string
+	Interval  time.Duration
 	AlertID   uint64
 	GroupID   uint64
 	ActiveAt  time.Time
@@ -96,6 +99,7 @@ var tplHeaders = []string{
 	"{{ $type := .Type }}",
 	"{{ $labels := .Labels }}",
 	"{{ $expr := .Expr }}",
+	"{{ $interval := .Interval }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
 	"{{ $alertID := .AlertID }}",
@@ -115,6 +119,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		Type:     a.Type,
 		Labels:   labels,
 		Expr:     a.Expr,
+		Interval: a.Interval,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,

app/vmalert/notifier/alert_test.go

@@ -129,6 +129,17 @@ func TestAlertExecTemplate(t *testing.T) {
 		"exprEscapedHTML":   "vm_rows{"label"="bar"}<0",
 	})
 
+	// interval-template
+	f(&Alert{
+		Interval: 10 * time.Second,
+	}, map[string]string{
+		"interval":         "{{ .Interval }}",
+		"intervalVariable": "{{ $interval }}",
+	}, map[string]string{
+		"interval":         "10s",
+		"intervalVariable": "10s",
+	})
+
 	// query
 	f(&Alert{
 		Expr: `vm_rows{"label"="bar"}>0`,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmalert/replay.go

@@ -25,11 +25,12 @@ var (
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
 		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
-		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+		"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
@@ -73,7 +74,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 
 	for _, cfg := range groupsCfg {
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
-		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
+		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
 	}
 	logger.Infof("replay evaluation finished, generated %d samples", totalRows)
 	if err := rw.Close(); err != nil {

app/vmalert/replay_test.go

@@ -8,8 +8,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 type fakeReplayQuerier struct {
@@ -32,6 +34,14 @@ func (fc *fakeRWClient) Close() error {
 }
 
 func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
+	_, err = metricsql.Parse(q)
+	if err != nil {
+		return res, &httpserver.ErrorWithStatusCode{
+			StatusCode: 422,
+			Err:        err,
+		}
+	}
+
 	key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
 	dps, ok := fr.registry[q]
 	if !ok {
@@ -275,4 +285,28 @@ func TestReplay(t *testing.T) {
 			},
 		},
 	}, 10)
+
+	// rule with wrong expression won't break the other rule with continueWithExecutionErr
+	continueWithExecutionErrOld := *continueWithExecutionErr
+	defer func() {
+		*continueWithExecutionErr = continueWithExecutionErrOld
+	}()
+	*continueWithExecutionErr = true
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+		{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string][]datasource.Metric{
+			"sum(up)": {
+				"12:00:00+12:01:00": {
+					{
+						Timestamps: []int64{1, 2},
+						Values:     []float64{1, 2},
+					},
+				},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	}, 2)
 }

app/vmalert/rule/alerting.go

@@ -530,6 +530,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
 			a.Value = m.Values[0]
+			a.Interval = ar.EvalInterval
 			a.Annotations = annotations
 			a.KeepFiringSince = time.Time{}
 			continue
@@ -612,6 +613,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 		Type:      ar.Type.String(),
 		Labels:    ls.origin,
 		Expr:      ar.Expr,
+		Interval:  ar.EvalInterval,
 		AlertID:   hash(ls.processed),
 		GroupID:   ar.GroupID,
 		ActiveAt:  activeAt,
@@ -673,6 +675,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
 		Name:        ar.Name,
 		Type:        ar.Type.String(),
 		Expr:        ar.Expr,
+		Interval:    ar.EvalInterval,
 		For:         ar.For,
 		ActiveAt:    start,
 		Value:       m.Values[0],

app/vmalert/rule/alerting_test.go

@@ -662,6 +662,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -682,6 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "for-firing"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -703,6 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -759,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -771,6 +775,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -1134,7 +1139,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		fq.Add(metrics...)
 		fq.SetPartialResponse(isResponsePartial)
 
-		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
+		ts := time.Unix(3600, 0)
+		if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		for hash, expAlert := range alertsExpected {
@@ -1152,12 +1158,14 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 
 	f(&AlertingRule{
-		Name: "common",
+		Name:         "common",
+		EvalInterval: time.Hour,
 		Labels: map[string]string{
 			"region": "east",
 		},
 		Annotations: map[string]string{
-			"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":   `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1166,7 +1174,8 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "foo"`,
+				"summary":   `common: Too high connection number for "foo"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1176,7 +1185,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		},
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "bar"`,
+				"summary":   `common: Too high connection number for "bar"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1388,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
 		"pod":           "vmalert-0",
-		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1398,7 +1408,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/group.go

@@ -97,6 +97,7 @@ type groupMetrics struct {
 	iterationMissed   *metrics.Counter
 	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
+	iterationLimit    *metrics.Gauge
 }
 
 // merges group rule labels into result map
@@ -336,6 +337,12 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
+	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_rule_group_results_limit{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		limit := g.Limit
+		g.mu.RUnlock()
+		return float64(limit)
+	})
 	for i := range g.Rules {
 		g.Rules[i].registerMetrics(g.metrics.set)
 	}
@@ -541,7 +548,7 @@ func (g *Group) infof(format string, args ...any) {
 }
 
 // Replay performs group replay
-func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
+func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	var total int
 	step := g.Interval * time.Duration(maxDataPoint)
 	ri := rangeIterator{start: start, end: end, step: step}
@@ -569,7 +576,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 			if !disableProgressBar {
 				bar = pb.StartNew(iterations)
 			}
-			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			if bar != nil {
 				bar.Finish()
 			}
@@ -591,7 +598,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 		rule := g.Rules[i]
 		sem <- struct{}{}
 		wg.Go(func() {
-			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			<-sem
 		})
 	}
@@ -611,7 +618,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	return total
 }
 
-func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
+func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
 	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
@@ -626,7 +633,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 		start := ri.s
 		end := ri.e
 		wg.Go(func() {
-			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
+			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}

app/vmalert/rule/rule.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -118,7 +120,7 @@ func (s *ruleState) add(e StateEntry) {
 	s.entries[s.cur] = e
 }
 
-func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
+func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
 	var err error
 	var tss []prompb.TimeSeries
 	for i := range replayRuleRetryAttempts {
@@ -126,6 +128,21 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		if err == nil {
 			break
 		}
+		// retry request if possible to tolerate temporary network or datasource unavailability issues
+		var esc *httpserver.ErrorWithStatusCode
+		if errors.As(err, &esc) {
+			statusCode := esc.StatusCode
+			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
+			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity {
+				if continueWithExecutionErr {
+					logger.Errorf("rule %q: %s", r, err)
+					return 0, nil
+				} else {
+					return 0, err
+				}
+			}
+		}
 		logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
 		time.Sleep(time.Second)
 	}

app/vmalert/rule/web.go

@@ -75,10 +75,13 @@ type GroupAlerts struct {
 // ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type ApiRule struct {
-	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
+	// Rule state must be one of these under following scenarios:
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state. (only for alerting rules)
+	//  "firing": at least 1 alert in the rule in firing state. (only for alerting rules)
+	//  "inactive": rule's last evaluation was successful but no alert in the rule in firing or pending state. (only for alerting rules)
+	//  "unhealthy": rule's last evaluation was failed with error. (for both alerting and recording rules)
+	// 	"nomatch": rule's last evaluation was successful but no time series matched the rule's expression. (for both alerting and recording rules)
+	//  "ok": the recording rule's last evaluation was successful. (only for recording rules)
 	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
@@ -237,6 +240,7 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 }
 
 func (r *ApiRule) ExtendState() {
+	// if alerting rule already has alerts, then state is already set to either "pending" or "firing" and we don't need to change it
 	if len(r.Alerts) > 0 {
 		return
 	}

app/vmalert/web.go

@@ -115,7 +115,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		WriteRule(w, r, rule)
 		return true
-	// current used by old vmalert UI and Grafana Alerts
+	// used by old vmalert UI
 	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
@@ -128,6 +128,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			state = rf.states[0]
 			rf.states = rf.states[:1]
 		}
+		// enable extendedStates by default for vmalert UI
+		rf.extendedStates = true
 		lr := rh.groups(rf)
 		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
@@ -543,10 +545,10 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
-			ruleWithExtendedState := rule
-			ruleWithExtendedState.ExtendState()
+			// extendedStates is used by the vmalert UI to extend the rule state with values such as "nomatch" and "unhealthy".
+			// those states are not supported by Grafana and not part of the Prometheus API spec yet
 			if rf.extendedStates {
-				rule = ruleWithExtendedState
+				rule.ExtendState()
 			}
 			if !rf.matchesRule(&rule) {
 				continue
@@ -554,18 +556,7 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if rf.excludeAlerts {
 				rule.Alerts = nil
 			}
-			ruleState := ruleWithExtendedState.State
-			if ruleState == "inactive" || ruleState == "pending" || ruleState == "firing" {
-				g.States["ok"]++
-			} else {
-				// 1. alerting rule with unhealthy or nomatch state
-				// 2. recording rule
-				g.States[ruleState]++
-
-				if ruleState == "nomatch" {
-					g.States["ok"]++
-				}
-			}
+			g.States[rule.State]++
 			filteredRules = append(filteredRules, rule)
 		}
 		if len(g.Rules) == 0 || len(filteredRules) > 0 {

app/vmalert/web.qtpl

@@ -130,9 +130,12 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
-                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
+                                {% if g.States["inactive"] > 0 %}<span class="badge bg-light text-success border border-success" title="None of the alert instances is in a pending or firing state">{%d g.States["inactive"] %} inactive</span> {% endif %}
+                                {% if g.States["pending"] > 0 %}<span class="badge bg-warning text-dark" title="At least one alert instance is pending">{%d g.States["pending"] %} pending</span> {% endif %}
+                                {% if g.States["firing"] > 0 %}<span class="badge bg-danger" title="At least one alert instance is firing">{%d g.States["firing"] %} firing</span> {% endif %}
+                                {% if g.States["ok"] > 0 %}<span class="badge bg-success" title="Recording rule last evaluation succeeded">{%d g.States["ok"] %} ok</span>{% endif %}
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Last evaluation failed with an error">{%d g.States["unhealthy"] %} unhealthy</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Rule expression matched no time series">{%d g.States["nomatch"] %} nomatch</span> {% endif %}
                             </span>
                         </span>
                     </span>
@@ -188,8 +191,21 @@
                                                     {% else %}
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
-                                                    |
+                                                    {% if r.State == "inactive" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "pending" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "firing" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "ok" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "unhealthy" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "nomatch" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% endif %}
                                                     {%= seriesFetchedWarn(prefix, &r) %}
+                                                    |
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">

app/vmauth/auth_config.go

@@ -118,9 +118,10 @@ type AccessLogFilters struct {
 }
 
 func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
-	if ui.AccessLog == nil {
+	if ui == nil || ui.AccessLog == nil {
 		return
 	}
+
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -134,6 +135,17 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
 		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
+// hasAnyURLs reports whether ui has at least one backend URL route configured.
+// It is used only for unauthorized_user config section, since other users
+// must always have either URLPrefix or URLMaps set.
+func (ui *UserInfo) hasAnyURLs() bool {
+	if ui == nil {
+		return false
+	}
+
+	return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`
@@ -983,8 +995,11 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-		if err := ui.initURLs(); err != nil {
-			return nil, err
+
+		if ui.hasAnyURLs() {
+			if err := ui.initURLs(); err != nil {
+				return nil, err
+			}
 		}
 
 		metricLabels, err := ui.getMetricLabels()

app/vmauth/example_config.yml

@@ -140,6 +140,18 @@ users:
   - "ProjectID: {{.MetricsProjectID}}"
   url_prefix: "http://vminsert:8480/insert/prometheus"
 
+# JWT-based routing that relies solely on custom claims.
+# The `vm_access` claim is missing, default value will be used.
+# e.g. {"role": "admin"}.
+- name: jwt-custom-claims
+  jwt:
+    skip_verify: true
+    vm_default_access_claim:
+        metrics_account_id: 1
+    match_claims:
+      role: admin
+  url_prefix: "http://vmselect-admin:8481/select/0/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -65,6 +65,8 @@ type JWTConfig struct {
 	MatchClaims       map[string]string `yaml:"match_claims,omitempty"`
 	parsedMatchClaims []*jwt.Claim
 
+	DefaultVMAccessClaim *jwt.VMAccessClaim `yaml:"default_vm_access_claim,omitempty"`
+
 	// verifierPool is used to verify JWT tokens.
 	// It is initialized from PublicKeys and/or PublicKeyFiles.
 	// In this case, it is initialized once at config reload and never updated until next reload
@@ -432,7 +434,6 @@ func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
 			}
 			if strings.Contains(p, placeholderPrefix) {
 				return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
-
 			}
 		}
 		for param, values := range bu.Query() {
@@ -487,7 +488,6 @@ func hasAnyPlaceholders(u *url.URL) bool {
 				return true
 			}
 		}
-
 	}
 	return false
 }

app/vmauth/main.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/textproto"
@@ -31,6 +32,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
 var (
@@ -173,11 +175,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
-		if ui != nil {
+		if ui.hasAnyURLs() {
 			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
+		ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 		handleMissingAuthorizationError(w)
 		return true
 	}
@@ -191,17 +194,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
 		defer putToken(tkn)
-		processUserRequest(w, r, ui, tkn)
-		return true
+		// Call processUserRequest only if the token contains the vm_access claim
+		// or a default claim is configured; otherwise fall through to unauthorized_user.
+		if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
+			processUserRequest(w, r, ui, tkn)
+			return true
+		}
 	}
 
 	uu := authConfig.Load().UnauthorizedUser
-	if uu != nil {
+	if uu.hasAnyURLs() {
 		processUserRequest(w, r, uu, nil)
 		return true
 	}
 
 	invalidAuthTokenRequests.Inc()
+	slowdownUnauthorizedResponse(r)
+	uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 	if *logInvalidAuthTokens {
 		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 		err = &httpserver.ErrorWithStatusCode{
@@ -377,7 +386,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 			return nil, &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
 					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
+				StatusCode: http.StatusRequestTimeout,
 			}
 		}
 
@@ -424,8 +433,12 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		}
 		targetURL := bu.url
 		if tkn != nil {
+			vmac := tkn.VMAccess()
+			if !tkn.HasVMAccessClaim() {
+				vmac = ui.JWT.DefaultVMAccessClaim
+			}
 			// for security reasons allow templating only for configured url values and headers
-			targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
+			targetURL, hc = replaceJWTPlaceholders(bu, hc, vmac)
 		}
 		if isDefault {
 			// Don't change path and add request_path query param for default route.
@@ -881,3 +894,20 @@ func debugInfo(u *url.URL, r *http.Request) string {
 	fmt.Fprint(s, ")")
 	return s.String()
 }
+
+// slowdownUnauthorizedResponse adds a random delay in the [2..3] seconds range before returning an unauthorized response.
+// This reduces the effectiveness of brute-force.
+//
+// Recommended by OWASP Top10:
+// https://owasp.org/Top10/2025/A07_2025-Authentication_Failures
+func slowdownUnauthorizedResponse(r *http.Request) {
+
+	d := 2*time.Second + time.Duration(rand.IntN(1000))*time.Millisecond
+	t := timerpool.Get(d)
+
+	select {
+	case <-t.C:
+	case <-r.Context().Done():
+	}
+	timerpool.Put(t)
+}

app/vmauth/main_synctest_test.go

@@ -0,0 +1,54 @@
+//go:build synctest
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+func TestBufferRequestBody_Timeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+
+		defaultMaxQueueDuration := *maxQueueDuration
+		defer func() {
+			*maxQueueDuration = defaultMaxQueueDuration
+		}()
+
+		*maxQueueDuration = 100 * time.Millisecond
+
+		ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
+		defer cancel()
+
+		_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		esc, ok := err.(*httpserver.ErrorWithStatusCode)
+		if !ok {
+			t.Fatalf("unexpected error type: %s", err)
+		}
+		if esc.StatusCode != http.StatusRequestTimeout {
+			t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
+		}
+	})
+}
+
+type timeoutBody struct {
+	delay time.Duration
+}
+
+func (r *timeoutBody) Read(_ []byte) (int, error) {
+	time.Sleep(r.delay)
+	return 0, context.DeadlineExceeded
+}
+
+func (r *timeoutBody) Close() error {
+	return nil
+}

app/vmauth/main_test.go

@@ -739,6 +739,12 @@ users:
 		"vm_access": map[string]any{},
 	}, false)
 
+	// token without vm_access claim, but with a custom claim usable for routing
+	roleToken := genToken(t, map[string]any{
+		"exp":  time.Now().Add(10 * time.Minute).Unix(),
+		"role": "admin",
+	}, true)
+
 	fullToken := genToken(t, map[string]any{
 		"exp": time.Now().Add(10 * time.Minute).Unix(),
 		"vm_access": map[string]any{
@@ -779,6 +785,45 @@ statusCode=401
 Unauthorized`
 	f(simpleCfgStr, request, responseExpected)
 
+	// token without vm_access claim should fall through to unauthorized_user
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	responseExpected = `
+statusCode=200
+path: /bar/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+unauthorized_user:
+  url_prefix: {BACKEND}/bar
+users:
+- jwt:
+    public_keys:
+    - %q
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
+	// token without vm_access claim is accepted when default_vm_access_claim configured
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+roleToken)
+	responseExpected = `
+statusCode=200
+path: /foo/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+    default_vm_access_claim:
+      metrics_account_id: 10
+      metrics_project_id: 10
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
 	// expired token
 	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
 	request.Header.Set(`Authorization`, `Bearer `+expiredToken)

app/vmctl/remoteread/remoteread.go

@@ -8,12 +8,14 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -234,9 +236,29 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
 	// shouldn't be accounted as an error.
 	for _, res := range readResp.Results {
 		for _, ts := range res.Timeseries {
-			vmTs := convertSamples(ts.Samples, ts.Labels)
-			if err := callback(vmTs); err != nil {
-				return err
+			// A series contains either float samples or native histogram samples.
+			// Both fields are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain samples of both types.
+			if len(ts.Samples) > 0 {
+				vmTs := convertSamples(ts.Samples, ts.Labels)
+				if err := callback(vmTs); err != nil {
+					return err
+				}
+			}
+			if len(ts.Histograms) > 0 {
+				hSamples := make([]histogramSample, 0, len(ts.Histograms))
+				for _, h := range ts.Histograms {
+					hSamples = append(hSamples, histogramSample{
+						timestamp: h.Timestamp,
+						fh:        h.ToFloatHistogram(),
+					})
+				}
+				for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
+					if err := callback(vmTs); err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -263,17 +285,45 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
 
 		for _, series := range res.ChunkedSeries {
 			samples := make([]prompb.Sample, 0)
+			var hSamples []histogramSample
 			for _, chunk := range series.Chunks {
-				s, err := parseSamples(chunk.Data)
-				if err != nil {
-					return err
+				switch chunk.Type {
+				case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
+					// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
+					// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
+					// MUST be set when using anything else than XOR". Senders before native
+					// histograms support (Prometheus < 2.40) do not set this field at all,
+					// so UNKNOWN chunks must be parsed as XOR ones.
+					s, err := parseSamples(chunk.Data)
+					if err != nil {
+						return err
+					}
+					samples = append(samples, s...)
+				case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
+					hs, err := parseHistograms(chunk.Type, chunk.Data)
+					if err != nil {
+						return err
+					}
+					hSamples = append(hSamples, hs...)
+				default:
+					return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
 				}
-				samples = append(samples, s...)
 			}
 
-			ts := convertSamples(samples, series.Labels)
-			if err := callback(ts); err != nil {
-				return err
+			// A series contains either XOR chunks or native histogram chunks.
+			// Both are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain chunks of both types.
+			if len(samples) > 0 {
+				ts := convertSamples(samples, series.Labels)
+				if err := callback(ts); err != nil {
+					return err
+				}
+			}
+			for _, ts := range convertHistograms(hSamples, series.Labels) {
+				if err := callback(ts); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -312,6 +362,151 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
 	return samples, it.Err()
 }
 
+// histogramSample represents a single native histogram sample.
+type histogramSample struct {
+	timestamp int64
+	fh        *histogram.FloatHistogram
+}
+
+func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
+	var enc chunkenc.Encoding
+	switch encoding {
+	case prompb.Chunk_HISTOGRAM:
+		enc = chunkenc.EncHistogram
+	case prompb.Chunk_FLOAT_HISTOGRAM:
+		enc = chunkenc.EncFloatHistogram
+	default:
+		return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
+	}
+	c, err := chunkenc.FromData(enc, chunk)
+	if err != nil {
+		return nil, fmt.Errorf("error read chunk: %w", err)
+	}
+
+	var hSamples []histogramSample
+	it := c.Iterator(nil)
+	for {
+		typ := it.Next()
+		if typ == chunkenc.ValNone {
+			break
+		}
+		switch typ {
+		case chunkenc.ValHistogram:
+			ts, h := it.AtHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        h.ToFloat(nil),
+			})
+		case chunkenc.ValFloatHistogram:
+			ts, fh := it.AtFloatHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        fh,
+			})
+		default:
+			// Skip unsupported values
+			continue
+		}
+	}
+	if err := it.Err(); err != nil {
+		return nil, fmt.Errorf("error iterate over chunks: %w", err)
+	}
+
+	return hSamples, nil
+}
+
+// convertHistograms converts native histogram samples into VictoriaMetrics histogram
+// time series in the same way as VictoriaMetrics converts native histograms
+// received via Prometheus remote write protocol: every native histogram sample
+// is converted into `<name>_count` and `<name>_sum` series plus a set of
+// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
+// The only difference is that for native histograms with custom buckets (NHCB)
+// bucket bounds are taken from the custom values, while the remote write protocol
+// parser ignores custom values and estimates the bounds with the exponential formula.
+// See https://prometheus.io/docs/specs/native_histograms/#data-model
+func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
+	if len(hSamples) == 0 {
+		return nil
+	}
+
+	labelPairs := make([]vm.LabelPair, 0, len(labels))
+	nameValue := ""
+	for _, label := range labels {
+		if label.Name == "__name__" {
+			nameValue = label.Value
+			continue
+		}
+		labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
+	}
+	// the metric has no name, skip it in the same way as VictoriaMetrics does
+	// when it receives a native histogram without the metric name via remote write protocol.
+	if nameValue == "" {
+		return nil
+	}
+
+	countSeries := &vm.TimeSeries{
+		Name:       nameValue + "_count",
+		LabelPairs: labelPairs,
+	}
+	sumSeries := &vm.TimeSeries{
+		Name:       nameValue + "_sum",
+		LabelPairs: labelPairs,
+	}
+	bucketSeries := make(map[string]*vm.TimeSeries)
+	// vmranges preserves the order of bucketSeries creation
+	// in order to get deterministic results.
+	var vmranges []string
+
+	for _, hs := range hSamples {
+		fh := hs.fh
+		countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
+		countSeries.Values = append(countSeries.Values, fh.Count)
+		sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
+		sumSeries.Values = append(sumSeries.Values, fh.Sum)
+
+		it := fh.AllBucketIterator()
+		for it.Next() {
+			b := it.At()
+			if b.Count <= 0 {
+				continue
+			}
+			vmrange := formatVmrange(b.Lower, b.Upper)
+			s := bucketSeries[vmrange]
+			if s == nil {
+				bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
+				copy(bucketLabelPairs, labelPairs)
+				bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
+				s = &vm.TimeSeries{
+					Name:       nameValue + "_bucket",
+					LabelPairs: bucketLabelPairs,
+				}
+				bucketSeries[vmrange] = s
+				vmranges = append(vmranges, vmrange)
+			}
+			s.Timestamps = append(s.Timestamps, hs.timestamp)
+			s.Values = append(s.Values, b.Count)
+		}
+	}
+
+	tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
+	tss = append(tss, countSeries, sumSeries)
+	for _, vmrange := range vmranges {
+		tss = append(tss, bucketSeries[vmrange])
+	}
+	return tss
+}
+
+// formatVmrange formats the given bucket bounds into `vmrange` label value
+// in the same way as VictoriaMetrics does for native histograms
+// received via Prometheus remote write protocol.
+func formatVmrange(lower, upper float64) string {
+	b := make([]byte, 0, 24)
+	b = strconv.AppendFloat(b, lower, 'e', 3, 64)
+	b = append(b, "..."...)
+	b = strconv.AppendFloat(b, upper, 'e', 3, 64)
+	return string(b)
+}
+
 type keyValue struct {
 	key   string
 	value string

app/vmctl/remoteread/remoteread_test.go

@@ -0,0 +1,334 @@
+package remoteread
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage/remote"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+)
+
+func testHistogram(mul int64) *histogram.Histogram {
+	return &histogram.Histogram{
+		Schema:          0,
+		Count:           uint64(10 * mul),
+		Sum:             25.5 * float64(mul),
+		ZeroThreshold:   0.001,
+		ZeroCount:       uint64(2 * mul),
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+		PositiveBuckets: []int64{1 * mul, 2 * mul},
+		NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		NegativeBuckets: []int64{4 * mul},
+	}
+}
+
+func TestConvertHistograms(t *testing.T) {
+	f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
+		t.Helper()
+
+		tss := convertHistograms(hSamples, labels)
+		if !reflect.DeepEqual(tss, expected) {
+			t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
+		}
+	}
+
+	// series without samples
+	f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
+
+	// series without the metric name must be skipped
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+	}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
+
+	// native histogram must be converted to _count, _sum and _bucket series
+	// in the same way as VictoriaMetrics does for Prometheus remote write protocol
+	labels := []prompb.Label{
+		{Name: "__name__", Value: "request_duration_seconds"},
+		{Name: "job", Value: "bar"},
+	}
+	jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
+	bucketLabels := func(vmrange string) []vm.LabelPair {
+		return []vm.LabelPair{
+			{Name: "job", Value: "bar"},
+			{Name: "vmrange", Value: vmrange},
+		}
+	}
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+		{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
+	}, labels, []*vm.TimeSeries{
+		{
+			Name:       "request_duration_seconds_count",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{10, 20},
+		},
+		{
+			Name:       "request_duration_seconds_sum",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{25.5, 51},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{4, 8},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{2, 4},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{1, 2},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{3, 6},
+		},
+	})
+}
+
+func TestParseHistograms(t *testing.T) {
+	c := chunkenc.NewHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse histogram chunk: %s", err)
+	}
+	if len(hSamples) != 2 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
+	}
+	for i, expected := range []struct {
+		timestamp int64
+		count     float64
+		sum       float64
+	}{
+		{timestamp: 1000, count: 10, sum: 25.5},
+		{timestamp: 2000, count: 20, sum: 51},
+	} {
+		if hSamples[i].timestamp != expected.timestamp {
+			t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
+		}
+		if hSamples[i].fh.Count != expected.count {
+			t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
+		}
+		if hSamples[i].fh.Sum != expected.sum {
+			t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
+		}
+	}
+
+	// unsupported chunk encoding must return error
+	if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
+		t.Fatalf("expecting non-nil error for unsupported chunk encoding")
+	}
+}
+
+func TestProcessResponse(t *testing.T) {
+	readResp := &prompb.ReadResponse{
+		Results: []*prompb.QueryResult{
+			{
+				Timeseries: []*prompb.TimeSeries{
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "cpu_usage"},
+							{Name: "job", Value: "bar"},
+						},
+						Samples: []prompb.Sample{
+							{Timestamp: 1000, Value: 1.5},
+						},
+					},
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "request_duration_seconds"},
+							{Name: "job", Value: "bar"},
+						},
+						Histograms: []prompb.Histogram{
+							prompb.FromIntHistogram(1000, testHistogram(1)),
+						},
+					},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(readResp)
+	if err != nil {
+		t.Fatalf("cannot marshal ReadResponse: %s", err)
+	}
+	compressed := snappy.Encode(nil, data)
+
+	var tss []*vm.TimeSeries
+	err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process response: %s", err)
+	}
+
+	// 1 float series + _count + _sum + 4 buckets
+	if len(tss) != 7 {
+		t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
+	}
+	if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[1])
+	}
+	if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[2])
+	}
+	for _, ts := range tss[3:] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+}
+
+type nopFlusher struct{}
+
+func (nopFlusher) Flush() {}
+
+func TestProcessStreamResponse(t *testing.T) {
+	// build a histogram chunk
+	hc := chunkenc.NewHistogramChunk()
+	hApp, err := hc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create histogram chunk appender: %s", err)
+	}
+	if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	// build a float chunk
+	xc := chunkenc.NewXORChunk()
+	xApp, err := xc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create xor chunk appender: %s", err)
+	}
+	xApp.Append(0, 1000, 1.5)
+
+	res := &prompb.ChunkedReadResponse{
+		ChunkedSeries: []*prompb.ChunkedSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "request_duration_seconds"},
+					{Name: "job", Value: "bar"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "cpu_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "memory_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					// the `type` field may be unset for XOR chunks,
+					// such chunks must be parsed as XOR ones
+					{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(res)
+	if err != nil {
+		t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
+	}
+	var buf bytes.Buffer
+	cw := remote.NewChunkedWriter(&buf, nopFlusher{})
+	if _, err := cw.Write(data); err != nil {
+		t.Fatalf("cannot write chunked response: %s", err)
+	}
+
+	var tss []*vm.TimeSeries
+	err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process stream response: %s", err)
+	}
+
+	// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
+	if len(tss) != 8 {
+		t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
+	}
+	if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[1])
+	}
+	for _, ts := range tss[2:6] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+	if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[6])
+	}
+	if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
+	}
+}
+
+func TestParseFloatHistograms(t *testing.T) {
+	c := chunkenc.NewFloatHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	fh := testHistogram(1).ToFloat(nil)
+	if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
+		t.Fatalf("cannot append float histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse float histogram chunk: %s", err)
+	}
+	if len(hSamples) != 1 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
+	}
+	if hSamples[0].timestamp != 1000 {
+		t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
+	}
+	if hSamples[0].fh.Count != 10 {
+		t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
+	}
+}

app/vmctl/vm/vm.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -111,7 +112,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 		if strings.Contains(dst, "?") {
 			separator = "&"
 		}
-		dst += fmt.Sprintf("%sextra_label=%s", separator, extraLabel)
+		dst += fmt.Sprintf("%sextra_label=%s", separator, url.QueryEscape(extraLabel))
 	}
 	return dst, nil
 }

app/vmctl/vm/vm_test.go

@@ -33,11 +33,14 @@ func TestAddExtraLabelsToImportPath_Success(t *testing.T) {
 	f("/api/v1/import", nil, "/api/v1/import")
 
 	// ok one extra label
-	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance=host-1")
+	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance%3Dhost-1")
 
 	// ok two extra labels
-	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
 
 	// ok two extra with exist param
-	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
+
+	// ok special chars in label value
+	f("/api/v1/import", []string{"team=a&b"}, "/api/v1/import?extra_label=team%3Da%26b")
 }

app/vminsert/common/insert_ctx.go

@@ -175,13 +175,19 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmpbs))
-	for idx, mmpb := range mmpbs {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmpbs {
+		if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
+			continue
+		}
+		mm := &mms[cnt]
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
 
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
@@ -201,14 +207,19 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmps))
-	for idx, mmpb := range mmps {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmps {
+		mm := &mms[cnt]
+		if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
+			continue
+		}
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
-
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{

app/vminsert/influx/request_handler.go

@@ -49,7 +49,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vminsert/main.go

@@ -68,9 +68,11 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented.")
+	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
+	maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
 )
 
 var (
@@ -106,7 +108,7 @@ func Init() {
 	promscrape.Init(func(_ *auth.Token, wr *prompb.WriteRequest) {
 		prompush.Push(wr)
 	})
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	timeserieslimits.MustInit(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 }
 
 // Stop stops vminsert.

app/vmselect/main.go

@@ -369,6 +369,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -388,6 +392,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -541,7 +549,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmselect/prometheus/prometheus.go

@@ -956,6 +956,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	mayCache := !httputil.GetBool(r, "nocache")
+	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
@@ -977,18 +978,19 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	}
 
 	ec := &promql.EvalConfig{
-		Start:               start,
-		End:                 end,
-		Step:                step,
-		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           0, // let vmstorage use maxUniqueTimeseries by default
-		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
-		Deadline:            deadline,
-		MayCache:            mayCache,
-		LookbackDelta:       lookbackDelta,
-		RoundDigits:         getRoundDigits(r),
-		EnforcedTagFilterss: etfs,
-		CacheTagFilters:     etfs,
+		Start:                            start,
+		End:                              end,
+		Step:                             step,
+		MaxPointsPerSeries:               *maxPointsPerTimeseries,
+		MaxSeries:                        0, // let vmstorage use maxUniqueTimeseries by default
+		QuotedRemoteAddr:                 httpserver.GetQuotedRemoteAddr(r),
+		Deadline:                         deadline,
+		MayCache:                         mayCache,
+		OptimizeRepeatedBinaryOpSubexprs: optimizeRepeatedBinaryOpSubexprs,
+		LookbackDelta:                    lookbackDelta,
+		RoundDigits:                      getRoundDigits(r),
+		EnforcedTagFilterss:              etfs,
+		CacheTagFilters:                  etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},

app/vmselect/promql/binary_op.go

@@ -164,15 +164,21 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		switch true {
-		case metricsql.IsBinaryOpCmp(op):
-			// Do not remove empty series for comparison operations,
-			// since this may lead to missing result.
-		default:
+		isCmpOp := metricsql.IsBinaryOpCmp(op)
+		if !isCmpOp {
+			// Do not remove empty series for comparison operations, since NaN can be an
+			// explicitly present sample value. In particular, `value != NaN` is true.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -183,6 +189,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			logger.Panicf("BUG: len(left) must match len(right) and len(dst); got %d vs %d vs %d", len(left), len(right), len(dst))
 		}
 		isBool := bfa.be.Bool
+		fillLeft := bfa.be.FillLeft
+		fillRight := bfa.be.FillRight
+		// A NaN produced by a vector comparison denotes a filtered-out sample rather
+		// than an explicitly present NaN value. Drop it when this result is used as
+		// the right-hand side of another comparison.
+		// Note that filtered-out samples surviving as NaN inside other wrappers such as
+		// transform functions or arithmetic operations aren't detected, since such NaN
+		// is indistinguishable from an explicitly present NaN value there.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
+		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -193,6 +209,24 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			}
 			for j, a := range leftValues {
 				b := rightValues[j]
+				leftIsNaN := math.IsNaN(a)
+				rightIsNaN := math.IsNaN(b)
+				// Both sides are NaN.
+				if leftIsNaN && rightIsNaN {
+					dstValues[j] = bf(a, b, isBool)
+					continue
+				}
+				if dropNaNRight && rightIsNaN && fillRight == nil {
+					dstValues[j] = nan
+					continue
+				}
+				// Apply the fill value when either the left or right side is NaN, but not both.
+				if leftIsNaN && fillLeft != nil {
+					a = fillLeft.N
+				}
+				if rightIsNaN && fillRight != nil {
+					b = fillRight.N
+				}
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
@@ -202,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
+// isVectorComparisonExpr returns whether e is a comparison operation
+// with at least one non-scalar operand.
+func isVectorComparisonExpr(e metricsql.Expr) bool {
+	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
+		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
+		// the shape of the result. Subqueries are left as is, since rollup
+		// functions skip NaN values on their own.
+		e = re.Expr
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
+		return false
+	}
+	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
+}
+
+// isScalarLikeExpr returns whether e always evaluates to a scalar value.
+func isScalarLikeExpr(e metricsql.Expr) bool {
+	switch e := e.(type) {
+	case *metricsql.NumberExpr, *metricsql.DurationExpr:
+		return true
+	case *metricsql.BinaryOpExpr:
+		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
+	case *metricsql.FuncExpr:
+		switch strings.ToLower(e.Name) {
+		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
+			return true
+		}
+	}
+	return false
+}
+
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {
@@ -226,7 +292,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +305,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0])}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0])}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +370,28 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with NaN values for the fill_left()/fill_right()/fill() modifiers.
+// These NaN values will be replaced later with the fill value if needed.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = math.NaN()
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -132,6 +132,9 @@ type EvalConfig struct {
 	// Whether the response can be cached.
 	MayCache bool
 
+	// Whether repeated cacheable binary op subexpressions can be optimized.
+	OptimizeRepeatedBinaryOpSubexprs bool
+
 	// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
 	LookbackDelta int64
 
@@ -171,6 +174,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.MaxPointsPerSeries = src.MaxPointsPerSeries
 	ec.Deadline = src.Deadline
 	ec.MayCache = src.MayCache
+	ec.OptimizeRepeatedBinaryOpSubexprs = src.OptimizeRepeatedBinaryOpSubexprs
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
@@ -420,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -447,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -455,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -466,82 +486,185 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
-	if !canPushdownCommonFilters(be) {
-		// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
-		// from exprFirst to exprSecond.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
-		qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
-		defer qt.Done()
-		var wg sync.WaitGroup
-
-		var tssFirst []*timeseries
-		var errFirst error
-		qtFirst := qt.NewChild("expr1")
-		wg.Go(func() {
-			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
-			qtFirst.Done()
-		})
-
-		var tssSecond []*timeseries
-		var errSecond error
-		qtSecond := qt.NewChild("expr2")
-		wg.Go(func() {
-			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
-			qtSecond.Done()
-		})
-
-		wg.Wait()
-		if errFirst != nil {
-			return nil, nil, errFirst
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
 		}
-		if errSecond != nil {
-			return nil, nil, errSecond
+		return tssSecond, tssFirst, nil
+	}
+	if canPushdownCommonFilters(be) {
+		// Execute binary operation in the following way:
+		//
+		// 1) execute the exprFirst
+		// 2) get common label filters for series returned at step 1
+		// 3) push down the found common label filters to exprSecond. This filters out unneeded series
+		//    during exprSecond execution instead of spending compute resources on extracting and processing these series
+		//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+		// 4) execute the exprSecond with possible additional filters found at step 3
+		//
+		// Typical use cases:
+		// - Kubernetes-related: show pod creation time with the node name:
+		//
+		//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
+		//
+		//   Without the optimization `kube_pod_info` would select and spend compute resources
+		//   for more time series than needed. The selected time series would be dropped later
+		//   when matching time series on the right and left sides of binary operand.
+		//
+		// - Generic alerting queries, which rely on `info` metrics.
+		//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
+		//
+		// - Queries, which get additional labels from `info` metrics.
+		//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
+		tssFirst, err := evalExpr(qt, ec, exprFirst)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
+			// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
+			// since the "exprFirst op exprSecond" would return an empty result in any case.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
+			return nil, nil, nil
+		}
+		lfs := getCommonLabelFilters(tssFirst)
+		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
+		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
+		tssSecond, err := evalExpr(qt, ec, exprSecond)
+		if err != nil {
+			return nil, nil, err
+		}
+		return sortResult(tssFirst, tssSecond)
+	}
+
+	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
+	// in exprFirst and exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
+	if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
+		qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
+		defer qt.Done()
+
+		qtFirst := qt.NewChild("expr1")
+		tssFirst, err := evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		qtSecond := qt.NewChild("expr2")
+		tssSecond, err := evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+		if err != nil {
+			return nil, nil, err
 		}
 		return tssFirst, tssSecond, nil
 	}
 
-	// Execute binary operation in the following way:
-	//
-	// 1) execute the exprFirst
-	// 2) get common label filters for series returned at step 1
-	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
-	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-	// 4) execute the exprSecond with possible additional filters found at step 3
-	//
-	// Typical use cases:
-	// - Kubernetes-related: show pod creation time with the node name:
-	//
-	//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
-	//
-	//   Without the optimization `kube_pod_info` would select and spend compute resources
-	//   for more time series than needed. The selected time series would be dropped later
-	//   when matching time series on the right and left sides of binary operand.
-	//
-	// - Generic alerting queries, which rely on `info` metrics.
-	//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
-	//
-	// - Queries, which get additional labels from `info` metrics.
-	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-	tssFirst, err := evalExpr(qt, ec, exprFirst)
-	if err != nil {
-		return nil, nil, err
+	// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
+	// from exprFirst to exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
+	qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
+	defer qt.Done()
+	var wg sync.WaitGroup
+
+	var tssFirst []*timeseries
+	var errFirst error
+	qtFirst := qt.NewChild("expr1")
+	wg.Go(func() {
+		tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+	})
+
+	var tssSecond []*timeseries
+	var errSecond error
+	qtSecond := qt.NewChild("expr2")
+	wg.Go(func() {
+		tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+	})
+
+	wg.Wait()
+	if errFirst != nil {
+		return nil, nil, errFirst
 	}
-	if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
-		// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
-		// since the "exprFirst op exprSecond" would return an empty result in any case.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
-		return nil, nil, nil
+	if errSecond != nil {
+		return nil, nil, errSecond
 	}
-	lfs := getCommonLabelFilters(tssFirst)
-	lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
-	exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
-	tssSecond, err := evalExpr(qt, ec, exprSecond)
-	if err != nil {
-		return nil, nil, err
+	return sortResult(tssFirst, tssSecond)
+}
+
+func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {
+	if !ec.OptimizeRepeatedBinaryOpSubexprs {
+		return false
 	}
-	return tssFirst, tssSecond, nil
+	if ec.Start == ec.End {
+		return false
+	}
+	if !ec.mayCache() {
+		return false
+	}
+
+	candidatesFirst := make(map[string]struct{}, 1)
+	var b []byte
+	visitOptimizedAggrs(exprFirst, func(ae *metricsql.AggrFuncExpr) {
+		if hasUnseededVolatileFunc(ae) {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		candidatesFirst[string(b)] = struct{}{}
+	})
+	if len(candidatesFirst) == 0 {
+		return false
+	}
+
+	repeated := false
+	visitOptimizedAggrs(exprSecond, func(ae *metricsql.AggrFuncExpr) {
+		if repeated {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		_, repeated = candidatesFirst[string(b)]
+	})
+	return repeated
+}
+
+func visitOptimizedAggrs(e metricsql.Expr, f func(ae *metricsql.AggrFuncExpr)) {
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		ae, ok := expr.(*metricsql.AggrFuncExpr)
+		if !ok {
+			return
+		}
+		if getIncrementalAggrFuncCallbacks(ae.Name) == nil {
+			return
+		}
+		fe, _ := tryGetArgRollupFuncWithMetricExpr(ae)
+		if fe == nil {
+			return
+		}
+		f(ae)
+	})
+}
+
+func hasUnseededVolatileFunc(e metricsql.Expr) bool {
+	found := false
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		if found {
+			return
+		}
+		fe, ok := expr.(*metricsql.FuncExpr)
+		if !ok {
+			return
+		}
+		switch strings.ToLower(fe.Name) {
+		case "now":
+			found = true
+		case "rand", "rand_normal", "rand_exponential":
+			found = len(fe.Args) == 0
+		}
+	})
+	return found
 }
 
 func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {

app/vmselect/promql/eval_test.go

@@ -170,3 +170,87 @@ func TestGetSumInstantValues(t *testing.T) {
 		[]*timeseries{ts("foo", 100, 1)},
 	)
 }
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsGate(t *testing.T) {
+	e, err := metricsql.Parse(`count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`)
+	if err != nil {
+		t.Fatalf("unexpected error in metricsql.Parse(): %s", err)
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok {
+		t.Fatalf("unexpected expr type; got %T; want *metricsql.BinaryOpExpr", e)
+	}
+
+	f := func(name string, ec *EvalConfig, resultExpected bool) {
+		t.Helper()
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v", name, result, resultExpected)
+		}
+	}
+
+	f("disabled optimization", &EvalConfig{
+		Start: 1000,
+		End:   2000,
+		Step:  1000,
+	}, false)
+	f("disabled cache", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("instant query", &EvalConfig{
+		Start:                            1000,
+		End:                              1000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("repeated cacheable aggregate subexpression", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, true)
+	f("unaligned range query", &EvalConfig{
+		Start:                            1001,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+}
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsExpressions(t *testing.T) {
+	f := func(name, q string, resultExpected bool) {
+		t.Helper()
+		e, err := metricsql.Parse(q)
+		if err != nil {
+			t.Fatalf("unexpected error in metricsql.Parse(%q) for %q: %s", q, name, err)
+		}
+		be, ok := e.(*metricsql.BinaryOpExpr)
+		if !ok {
+			t.Fatalf("unexpected expr type for %q; got %T; want *metricsql.BinaryOpExpr", name, e)
+		}
+		ec := &EvalConfig{Start: 1000, End: 2000, Step: 1000, MayCache: true, OptimizeRepeatedBinaryOpSubexprs: true}
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v; query: %q", name, result, resultExpected, q)
+		}
+	}
+
+	f("original issue query", `count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`, true)
+	f("right side contains repeated count aggregate", `count(foo) by (job) / (count(foo) by (job) + 1)`, true)
+	f("same sum aggregate", `sum(rate(foo[5m])) by (job) / sum(rate(foo[5m])) by (job)`, true)
+	f("same inner rollup but different aggregates", `sum(rate(foo[5m])) by (job) / count(rate(foo[5m])) by (job)`, false)
+	f("different count aggregates", `count(foo) by (job) / count(bar) by (job)`, false)
+	f("bare metric selector", `foo / foo`, false)
+	f("bare rollup function", `rate(a[5m]) / rate(a[5m])`, false)
+	f("now at modifier", `sum(rate(foo[5m] @ now())) by (job) / sum(rate(foo[5m] @ now())) by (job)`, false)
+	f("unseeded rand at modifier", `sum(rate(foo[5m] @ rand())) by (job) / sum(rate(foo[5m] @ rand())) by (job)`, false)
+	f("unseeded rand_normal at modifier", `sum(rate(foo[5m] @ rand_normal())) by (job) / sum(rate(foo[5m] @ rand_normal())) by (job)`, false)
+	f("unseeded rand_exponential at modifier", `sum(rate(foo[5m] @ rand_exponential())) by (job) / sum(rate(foo[5m] @ rand_exponential())) by (job)`, false)
+	f("seeded rand at modifier", `sum(rate(foo[5m] @ rand(1))) by (job) / sum(rate(foo[5m] @ rand(1))) by (job)`, true)
+}

app/vmselect/promql/exec_test.go

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != NaN`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != 1200`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (1 > 2)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
+		// A missing sample on the left side results in NaN for every comparison
+		// operation, so no special handling is needed there.
+		t.Parallel()
+		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
+		// Prometheus drops the timestamps filtered out by the comparison on the right.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`
@@ -4006,6 +4136,275 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill() both sides NaN case`, func(t *testing.T) {
+		t.Parallel()
+		q := `(
+			label_set(time() <= 1200, "foo", "common")
+		) + fill(10) (
+			label_set(time() >= 1600, "foo", "common")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1010, 1210, nan, 1610, 1810, 2010},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`
@@ -4833,13 +5232,137 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`buckets_limit(zero)`, func(t *testing.T) {
+	// buckets that are consecutively empty at left and right ends will not be preserved.
+	t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
 		t.Parallel()
-		q := `buckets_limit(0, (
-			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
-			alias(label_set(50, "le", "120", "x", "y"), "metric"),
-		))`
-		resultExpected := []netstorage.Result{}
+		q := `sort(buckets_limit(3, (
+			alias(label_set(36, "le", "+Inf"), "metric"),
+			alias(label_set(36, "le", "25"), "metric"),
+			alias(label_set(36, "le", "21"), "metric"),
+			alias(label_set(36, "le", "19"), "metric"),
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+
+	// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
+	t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(buckets_limit(5, (
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("6"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("metric")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.MetricGroup = []byte("metric")
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
 		f(q, resultExpected)
 	})
 	t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6228,50 +6751,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
 		f(q, resultExpected)
 	})
-	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
-		t.Parallel()
-		q := `sort_by_label(
-			buckets_limit(
-				3,
-				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
-			), "le"
-		)`
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("+Inf"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("1.000e+00"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("2.448e+00"),
-			},
-		}
-		resultExpected := []netstorage.Result{r1, r2, r3}
-		f(q, resultExpected)
-	})
 	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

app/vmselect/promql/rollup.go

@@ -1055,7 +1055,7 @@ func newRollupHoltWinters(args []any) (rollupFunc, error) {
 			return nan
 		}
 
-		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing .
+		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29 .
 		// TODO: determine whether this shit really works.
 		s0 := rfa.prevValue
 		if math.IsNaN(s0) {

app/vmselect/promql/timeseries.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"strconv"
 	"sync"
@@ -288,6 +289,9 @@ func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
 }
 
 func marshalBytesFast(dst []byte, s []byte) []byte {
+	if len(s) > math.MaxUint16 {
+		logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
+	}
 	dst = encoding.MarshalUint16(dst, uint16(len(s)))
 	dst = append(dst, s...)
 	return dst

app/vmselect/promql/transform.go

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	if limit <= 0 {
-		return nil, nil
+		return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
 	}
 	if limit < 3 {
 		// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,6 +461,23 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 				prevValue = value
 			}
 		}
+
+		// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
+		epsilon := 1e-9
+		isEmptyBucket := func(hits float64) bool {
+			return !math.IsNaN(hits) && math.Abs(hits) < epsilon
+		}
+		l := 0
+		r := len(leGroup) - 1
+		for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
+			r--
+		}
+		for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
+			l++
+		}
+		leGroup = leGroup[l : r+1]
+
 		for len(leGroup) > limit {
 			// Preserve the first and the last bucket for better accuracy for min and max values
 			xxMinIdx := 1
@@ -1121,29 +1138,29 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 
 func fixBrokenBuckets(i int, xss []leTimeseries) {
 	// Buckets are already sorted by le, so their values must be in ascending order,
-	// since the next bucket includes all the previous buckets.
-	// If the next bucket has lower value than the current bucket,
-	// then the next bucket must be substituted with the current bucket value.
+	// since the upper bucket includes all the lower buckets.
+	// If the upper bucket has lower value than the current bucket,
+	// then the upper bucket must be substituted with the current bucket value.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102
 	if len(xss) < 2 {
 		return
 	}
 
-	vNext := xss[0].ts.Values[i]
+	vPrev := xss[0].ts.Values[i]
 	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
 	// compared with upper buckets in the loop below.
-	if math.IsNaN(vNext) {
-		vNext = 0
-		xss[0].ts.Values[i] = vNext
+	if math.IsNaN(vPrev) {
+		vPrev = 0
+		xss[0].ts.Values[i] = vPrev
 	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
-	// or are bigger than the lower bucket values.
+	// or are smaller than the lower bucket values.
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
-		if math.IsNaN(v) || vNext > v {
-			xss[j].ts.Values[i] = vNext
+		if math.IsNaN(v) || vPrev > v {
+			xss[j].ts.Values[i] = vPrev
 		} else {
-			vNext = v
+			vPrev = v
 		}
 	}
 }

app/vmselect/vmui/assets/MetricsQL-CQAtdgI6.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmselect/vmui/assets/rolldown-runtime-CNC7AqOf.js

@@ -0,0 +1 @@
+var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/assets/rolldown-runtime-Cyuzqnbw.js

@@ -1 +0,0 @@
-var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-CusQvJzs.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
+  <script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -30,6 +30,17 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	maxBackfillAge = flagutil.NewRetentionDuration("maxBackfillAge", "0", "The maximum allowed age for the ingested samples with historical timestamps. "+
+		"Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. "+
+		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
+		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
+	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
+		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -108,7 +119,7 @@ func DataPath() string {
 }
 
 // Init initializes vmstorage.
-func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
@@ -143,6 +154,7 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	opts := storage.OpenOptions{
 		Retention:                   retentionPeriod.Duration(),
 		FutureRetention:             futureRetention.Duration(),
+		MaxBackfillAge:              maxBackfillAge.Duration(),
 		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
 		MaxHourlySeries:             getMaxHourlySeries(),
 		MaxDailySeries:              getMaxDailySeries(),
@@ -169,6 +181,21 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
 
+	if *vmselectAddr != "" {
+		var err error
+		limits := vmselectapi.Limits{
+			MaxConcurrentRequests:         vmselectMaxConcurrentRequests,
+			MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
+			MaxQueueDuration:              vmselectMaxQueueDuration,
+			MaxQueueDurationFlagName:      "search.maxQueueDuration",
+		}
+		api := newVMStorageWithTenantID(vmStorage)
+		vmselectSrv, err = vmselectapi.NewServer(*vmselectAddr, api, limits, *vmselectDisableRPCCompression)
+		if err != nil {
+			logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
+		}
+	}
+
 	VMInsertAPI = vmStorage
 	VMSelectAPI = vmStorage
 	GetSearch = vmStorage.GetSearch
@@ -191,6 +218,8 @@ var (
 
 	// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
 	DebugFlush func()
+
+	vmselectSrv *vmselectapi.Server
 )
 
 // Stop stops the vmstorage
@@ -201,6 +230,10 @@ func Stop() {
 
 	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime := time.Now()
+
+	if vmselectSrv != nil {
+		vmselectSrv.MustStop()
+	}
 	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -443,8 +476,10 @@ func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/inmemory"}`, tm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/small"}`, tm.SmallSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/big"}`, tm.BigSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/metaindex"}`, tm.MetaindexSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/metaindex"}`, idbm.MetaindexSizeBytes)
 
 	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)

app/vmstorage/vmstorage.go

@@ -164,6 +164,10 @@ func (vms *VMStorage) IsReadOnly() bool {
 }
 
 func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	return vms.initSearch(qt, sq, marshalDefault, deadline)
+}
+
+func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, marshal marshalFunc, deadline uint64) (vmselectapi.BlockIterator, error) {
 	vms.wg.Add(1)
 
 	tr := sq.GetTimeRange()
@@ -178,6 +182,7 @@ func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery
 		return nil, fmt.Errorf("missing tag filters")
 	}
 	bi := getBlockIterator()
+	bi.marshal = marshal
 	bi.wgDone = vms.wg.Done
 	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
 	if err := bi.sr.Error(); err != nil {
@@ -198,11 +203,19 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
 	return searchQueryLimit
 }
 
+type marshalFunc func(dst []byte, src *storage.MetricBlock) []byte
+
+// marshalDefault is the default implementation of the MetricBlock marshaling.
+func marshalDefault(dst []byte, src *storage.MetricBlock) []byte {
+	return src.Marshal(dst)
+}
+
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
-	sr     storage.Search
-	mb     storage.MetricBlock
-	wgDone func()
+	sr      storage.Search
+	mb      storage.MetricBlock
+	marshal marshalFunc
+	wgDone  func()
 }
 
 var blockIteratorsPool sync.Pool
@@ -228,10 +241,10 @@ func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	if !bi.sr.NextMetricBlock() {
 		return dst, false
 	}
-	mb := bi.mb
+	mb := &bi.mb
 	mb.MetricName = bi.sr.MetricBlockRef.MetricName
 	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	dst = mb.Marshal(dst[:0])
+	dst = bi.marshal(dst[:0], mb)
 	return dst, true
 }
 

app/vmstorage/vmstorage_with_tenant_id.go

@@ -0,0 +1,282 @@
+package vmstorage
+
+import (
+	"flag"
+	"fmt"
+	"math"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	accountID = flag.Uint64("accountID", 0, "The accountID of the stored data")
+	projectID = flag.Uint64("projectID", 0, "The projectID of the stored data")
+)
+
+func newVMStorageWithTenantID(vms *VMStorage) *VMStorageWithTenantID {
+	if *accountID > math.MaxUint32 {
+		logger.Fatalf("-accountID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *accountID)
+	}
+	if *projectID > math.MaxUint32 {
+		logger.Fatalf("-projectID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *projectID)
+	}
+	return &VMStorageWithTenantID{
+		vms:       vms,
+		accountID: uint32(*accountID),
+		projectID: uint32(*projectID),
+	}
+}
+
+// VMStorageWithTenantID is a thin wrapper around VMStorage type that overrides
+// its methods to properly serve requests coming from a vmselect (require
+// tenantID).
+//
+// A new instance of this type should be created using
+// newVMStorageWithTenantID(). The created instance does not require closing.
+// The instance also does not take ownership of vms and it is the responsibility
+// of the caller to close vms.
+type VMStorageWithTenantID struct {
+	vms *VMStorage
+
+	accountID uint32
+	projectID uint32
+}
+
+// InitSearch initializes a storage search for a request initiated by a
+// vmselect.
+//
+// The search is initialized only if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+// Otherwise, the method returns an interator that will return no data.
+//
+// The method also overrides the data format of the data returned by the
+// iterator by prepending accountID and projectID bytes to the metric name and
+// the data block (a format used in vmcluster).
+func (vmst *VMStorageWithTenantID) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return emptyBI, nil
+	}
+	return vmst.vms.initSearch(qt, sq, vmst.marshalMetricBlock, deadline)
+}
+
+var emptyBI = &emptyBlockIterator{}
+
+// emptyBlockIterator is an implementation of vmselectapi.BlockIterator that
+// always returns no data.
+type emptyBlockIterator struct{}
+
+func (*emptyBlockIterator) MustClose() {}
+
+func (*emptyBlockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	return dst, false
+}
+
+func (*emptyBlockIterator) Error() error {
+	return nil
+}
+
+// marshalMetricBlock serializes a metric block in the format expected by
+// vmselect.
+//
+// vmselect expects metric names and data blocks to have the tenantID but
+// vmsingle does not have it. Therefore the tenantID needs to be included to
+// every metric name and block.
+func (vmst *VMStorageWithTenantID) marshalMetricBlock(dst []byte, src *storage.MetricBlock) []byte {
+	// Marshal metric name:
+	// 1. Marshal metric name length + accountID length + projectID length (in
+	//    bytes).
+	// 2. append accountID and projectID bytes
+	// 3. Finally append metric name bytes
+	dst = encoding.MarshalVarUint64(dst, uint64(len(src.MetricName))+8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = append(dst, src.MetricName...)
+
+	// Marshal data block.
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = storage.MarshalBlock(dst, &src.Block)
+
+	return dst
+}
+
+// SearchMetricNames searches the storage for metric names that match the query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+//
+// Found metric names are prepended with accountID and projectID bytes (a format
+// used in vmcluster).
+func (vmst *VMStorageWithTenantID) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+
+	metricNames, err := vmst.vms.SearchMetricNames(qt, sq, deadline)
+	if err != nil {
+		return nil, err
+	}
+
+	// vmselect expects metric names to have the tenantID but vmsingle does not
+	// have it. Therefore the tenantID needs to be prepended to every metric
+	// name.
+	dst := make([]byte, 0, 8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	tenantID := string(dst)
+
+	for i, metricName := range metricNames {
+		metricNames[i] = tenantID + metricName
+	}
+	return metricNames, nil
+}
+
+// LabelValues searches the storage for label values that match the query and
+// correspond to a label whose name is `labelName`. The returned result
+// will contain not more than `maxLabelValues`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
+}
+
+// TagValueSuffixes searches the storage for Graphite tag value suffixes. The
+// returned result will contain not more than `maxSuffixes`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return nil, nil
+	}
+	return vmst.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+}
+
+// LabelNames searches the storage for label names that match the query.
+// The returned result will contain not more than `maxLabelNames`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelNames(qt, sq, maxLabelNames, deadline)
+}
+
+// SeriesCount returns the total number of metrics stored in the database.
+//
+// The method may return inflated numbers. How inflated the count depends
+// on the churn rate and the retention period. For example, if a metric lasts
+// for 2 months, it will be counted twice.
+//
+// The method also counts the deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return 0.
+func (vmst *VMStorageWithTenantID) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return 0, nil
+	}
+	return vmst.vms.SeriesCount(qt, accountID, projectID, deadline)
+}
+
+// Tenants returns just one tenant consisting of the -accountID and -projectID
+// flag values.
+func (vmst *VMStorageWithTenantID) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	tenantID := fmt.Sprintf("%d:%d", vmst.accountID, vmst.projectID)
+	return []string{tenantID}, nil
+}
+
+// TSDBStatus retrieves the status for metrics that match to the search query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return empty
+// status.
+func (vmst *VMStorageWithTenantID) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return &storage.TSDBStatus{}, nil
+	}
+	return vmst.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
+}
+
+// DeleteSeries marks as deleted metrics that match the search query.
+// The method returns the number of deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, no metrics will be deleted
+// and the method will return 0.
+func (vmst *VMStorageWithTenantID) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return 0, nil
+	}
+	return vmst.vms.DeleteSeries(qt, sq, deadline)
+}
+
+// RegisterMetricNames registers metric names in the index, the sample values
+// and timestamps are ignored.
+func (vmst *VMStorageWithTenantID) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
+	return vmst.vms.RegisterMetricNames(qt, mrs, deadline)
+}
+
+// GetMetricNamesUsageStats retrieves the usage stats for metrics whose name
+// matches the pattern.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return metricnamestats.StatsResult{}, nil
+	}
+	return vmst.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
+}
+
+// ResetMetricNamesUsageStats resets the metric name usage stats.
+func (vmst *VMStorageWithTenantID) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
+	return vmst.vms.ResetMetricNamesUsageStats(qt, deadline)
+}
+
+// GetMetadataRecords retrieves the metadata for the metricName.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return nil, nil
+	}
+	return vmst.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
+}
+
+// hasValidTenantID returns true if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) hasValidTenantID(sq *storage.SearchQuery) bool {
+	return sq.IsMultiTenant || vmst.isValidTenantID(sq.AccountID, sq.ProjectID)
+}
+
+// isValidTenantToken returns true if the TenantToken is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantToken(tt *storage.TenantToken) bool {
+	return tt == nil || vmst.isValidTenantID(tt.AccountID, tt.ProjectID)
+}
+
+// isValidTenantID returns true if the accountID and projectID match -accountID
+// and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantID(accountID, projectID uint32) bool {
+	return accountID == vmst.accountID && projectID == vmst.projectID
+}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.4 AS build-web-stage
+FROM golang:1.26.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.23.4
+FROM alpine:3.24.1
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.21",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.5",
-    "preact": "^10.29.2",
-    "qs": "^6.15.2",
+    "marked": "^18.0.6",
+    "preact": "^10.29.7",
+    "qs": "^6.15.3",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.17.0",
+    "react-router-dom": "^7.18.1",
     "uplot": "^1.6.32",
-    "vite": "^8.0.16",
+    "vite": "^8.1.4",
     "web-vitals": "^5.3.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.5",
+    "@eslint/eslintrc": "^3.3.6",
     "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.5",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.9.2",
+    "@types/node": "^26.1.1",
     "@types/qs": "^6.15.1",
     "@types/react": "^19.2.17",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.61.0",
-    "@typescript-eslint/parser": "^8.61.0",
+    "@typescript-eslint/eslint-plugin": "^8.64.0",
+    "@typescript-eslint/parser": "^8.64.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "globals": "^17.6.0",
-    "http-proxy-middleware": "^4.1.0",
+    "globals": "^17.7.0",
+    "http-proxy-middleware": "^4.2.0",
     "jsdom": "^29.1.1",
-    "postcss": "^8.5.15",
+    "postcss": "^8.5.19",
     "sass-embedded": "^1.100.0",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.10"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -183,7 +183,7 @@ const StepConfigurator: FC = () => {
           <div className="vm-step-control-popper-info">
             <p>
               <code>step</code> - the <Hyperlink
-                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
+                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations"
                 text="interval"
               /> between datapoints, which must be returned from the range query.
               The <code>query</code> is executed

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls.tsx

@@ -1,4 +1,4 @@
-import { FC, useEffect, useRef, useState } from "preact/compat";
+import { FC, useEffect, useRef } from "preact/compat";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { getAppModeEnable } from "../../../../utils/app-mode";
 import Button from "../../../Main/Button/Button";
@@ -9,27 +9,38 @@ import classNames from "classnames";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import useBoolean from "../../../../hooks/useBoolean";
+import { getMillisecondsFromDuration } from "../../../../utils/time";
+import { useSearchParams } from "react-router";
 
-interface AutoRefreshOption {
-  seconds: number
-  title: string
-}
-
-const delayOptions: AutoRefreshOption[] = [
-  { seconds: 0, title: "Off" },
-  { seconds: 1, title: "1s" },
-  { seconds: 2, title: "2s" },
-  { seconds: 5, title: "5s" },
-  { seconds: 10, title: "10s" },
-  { seconds: 30, title: "30s" },
-  { seconds: 60, title: "1m" },
-  { seconds: 300, title: "5m" },
-  { seconds: 900, title: "15m" },
-  { seconds: 1800, title: "30m" },
-  { seconds: 3600, title: "1h" },
-  { seconds: 7200, title: "2h" }
+const delayOptions = [
+  "Off",
+  "1s",
+  "2s",
+  "5s",
+  "10s",
+  "30s",
+  "1m",
+  "5m",
+  "15m",
+  "30m",
+  "1h",
+  "2h"
 ];
 
+const DEFAULT_OPTION = delayOptions[0];
+
+const MIN_REFRESH_MS = 1000;
+const MAX_REFRESH_MS = getMillisecondsFromDuration(delayOptions[delayOptions.length - 1]);
+const REFRESH_URL_PARAM = "refresh";
+
+const durationToMs = (dur: string | null) => {
+  return dur ? getMillisecondsFromDuration(dur) : 0;
+};
+
+const isValidDelay = (ms: number) => {
+  return ms >= MIN_REFRESH_MS && ms <= MAX_REFRESH_MS;
+};
+
 interface ExecutionControlsProps {
   tooltip: string;
   useAutorefresh?: boolean;
@@ -38,12 +49,14 @@ interface ExecutionControlsProps {
 
 export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAutorefresh, closeModal }) => {
   const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
 
   const dispatch = useTimeDispatch();
   const appModeEnable = getAppModeEnable();
-  const [autoRefresh, setAutoRefresh] = useState(false);
 
-  const [selectedDelay, setSelectedDelay] = useState<AutoRefreshOption>(delayOptions[0]);
+  const rawDelay = searchParams.get(REFRESH_URL_PARAM);
+  const msDelay = durationToMs(rawDelay);
+  const selectedDelay = isValidDelay(msDelay) ? rawDelay : DEFAULT_OPTION;
 
   const {
     value: openOptions,
@@ -52,11 +65,20 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   } = useBoolean(false);
   const optionsButtonRef = useRef<HTMLDivElement>(null);
 
-  const handleChange = (d: AutoRefreshOption) => {
-    if ((autoRefresh && !d.seconds) || (!autoRefresh && d.seconds)) {
-      setAutoRefresh(prev => !prev);
-    }
-    setSelectedDelay(d);
+  const handleChange = (dur: string) => () => {
+    setSearchParams(prev => {
+      const nextParams = new URLSearchParams(prev);
+      const ms = durationToMs(dur);
+
+      if (ms) {
+        nextParams.set(REFRESH_URL_PARAM, `${dur}`);
+      } else {
+        nextParams.delete(REFRESH_URL_PARAM);
+      }
+
+      return nextParams;
+    });
+
     handleCloseOptions();
   };
 
@@ -68,23 +90,19 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   };
 
   useEffect(() => {
-    const delay = selectedDelay.seconds;
+    const ms = durationToMs(selectedDelay);
     let timer: number;
-    if (autoRefresh) {
+
+    if (useAutorefresh && isValidDelay(ms)) {
       timer = setInterval(() => {
         dispatch({ type: "RUN_QUERY" });
-      }, delay * 1000) as unknown as number;
-    } else {
-      setSelectedDelay(delayOptions[0]);
+      }, ms) as unknown as number;
     }
-    return () => {
-      timer && clearInterval(timer);
-    };
-  }, [selectedDelay, autoRefresh]);
 
-  const createHandlerChange = (d: AutoRefreshOption) => () => {
-    handleChange(d);
-  };
+    return () => {
+      clearInterval(timer);
+    };
+  }, [selectedDelay, useAutorefresh]);
 
   return (
     <>
@@ -106,7 +124,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 <span className="vm-mobile-option__icon"><RestartIcon/></span>
                 <div className="vm-mobile-option-text">
                   <span className="vm-mobile-option-text__label">Auto-refresh</span>
-                  <span className="vm-mobile-option-text__value">{selectedDelay.title}</span>
+                  <span className="vm-mobile-option-text__value">{selectedDelay}</span>
                 </div>
                 <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
               </div>
@@ -139,7 +157,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                       )}
                       onClick={toggleOpenOptions}
                     >
-                      {selectedDelay.title}
+                      {selectedDelay}
                     </Button>
                   </div>
                 </Tooltip>
@@ -187,12 +205,12 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 className={classNames({
                   "vm-list-item": true,
                   "vm-list-item_mobile": isMobile,
-                  "vm-list-item_active": d.seconds === selectedDelay.seconds
+                  "vm-list-item_active": d === selectedDelay
                 })}
-                key={d.seconds}
-                onClick={createHandlerChange(d)}
+                key={d}
+                onClick={handleChange(d)}
               >
-                {d.title}
+                {d}
               </div>
             ))}
           </div>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseAlert/index.tsx

@@ -98,7 +98,7 @@ const BaseAlert = ({ item, group }: BaseAlertProps) => {
       {!!Object.keys(item.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -121,7 +121,7 @@ const BaseRule = ({ item, group }: BaseRuleProps) => {
       {!!Object.keys(item?.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/HeaderNav.tsx

@@ -17,10 +17,12 @@ interface HeaderNavProps {
 const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
   const { pathname } = useLocation();
   const [activeMenu, setActiveMenu] = useState(pathname);
+  const [openMenu, setOpenMenu] = useState<string | null>(null);
   const menu = useNavigationMenu();
 
   useEffect(() => {
     setActiveMenu(pathname);
+    setOpenMenu(null);
   }, [pathname]);
 
   return (
@@ -41,6 +43,8 @@ const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
               color={color}
               background={background}
               direction={direction}
+              openMenu={openMenu}
+              setOpenMenu={setOpenMenu}
             />
           )
           : (

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavSubItem.tsx

@@ -1,11 +1,8 @@
-import { FC, useRef, useState } from "preact/compat";
-import { useLocation } from "react-router-dom";
+import { FC, useRef, useState, Dispatch, SetStateAction } from "preact/compat";
 import classNames from "classnames";
 import { ArrowDropDownIcon } from "../../../components/Main/Icons";
 import Popper from "../../../components/Main/Popper/Popper";
 import NavItem from "./NavItem";
-import { useEffect } from "react";
-import useBoolean from "../../../hooks/useBoolean";
 import { NavigationItem, NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
@@ -15,6 +12,8 @@ interface NavItemProps {
   color?: string
   background?: string
   direction?: "row" | "column"
+  openMenu: string | null,
+  setOpenMenu: Dispatch<SetStateAction<string | null>>,
 }
 
 const NavSubItem: FC<NavItemProps> = ({
@@ -23,21 +22,18 @@ const NavSubItem: FC<NavItemProps> = ({
   color,
   background,
   submenu,
-  direction = "row"
+  direction = "row",
+  openMenu,
+  setOpenMenu,
 }) => {
-  const { pathname } = useLocation();
-
   const [menuTimeout, setMenuTimeout] = useState<NodeJS.Timeout | null>(null);
   const buttonRef = useRef<HTMLDivElement>(null);
 
-  const {
-    value: openSubmenu,
-    setFalse: handleCloseSubmenu,
-    setTrue: setOpenSubmenu,
-  } = useBoolean(false);
+  const openSubmenu = openMenu === label;
+  const handleCloseSubmenu = () => setOpenMenu(prev => (prev === label ? null : prev));
 
   const handleOpenSubmenu = () => {
-    if (direction === "row" || !openSubmenu) setOpenSubmenu();
+    if (direction === "row" || !openSubmenu) setOpenMenu(label);
     if (direction === "column" && openSubmenu) handleCloseSubmenu();
     if (direction === "row" && menuTimeout) clearTimeout(menuTimeout);
   };
@@ -52,10 +48,6 @@ const NavSubItem: FC<NavItemProps> = ({
     if (menuTimeout) clearTimeout(menuTimeout);
   };
 
-  useEffect(() => {
-    handleCloseSubmenu();
-  }, [pathname]);
-
   return (
     <div
       className={classNames({

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const match = searchParams.get("match");
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
-
+  const showMetricNameStats = !(match || focusLabel);
   const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
     },
   ].filter(t => t.display);
 
-  if (!totals.length) {
+  if (!totals.length && !showMetricNameStats) {
     return null;
   }
 
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           <h4 className="vm-cardinality-totals-card__title">
             {info && (
               <Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
-                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
+                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
               </Tooltip>
             )}
             {title}
@@ -99,7 +99,10 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           )}
         </div>
       ))}
-      <CardinalityMetricNameStats metricNameStats={metricNameStats}/>
+      {
+        showMetricNameStats &&
+        <CardinalityMetricNameStats metricNameStats={metricNameStats} />
+      }
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/style.scss

@@ -62,6 +62,13 @@
     padding-right: 40px;
   }
 
+  .vm-annotations-table {
+    tbody > tr > td {
+      vertical-align: top;
+      white-space: pre-line;
+    }
+  }
+
   display: flex;
   flex-direction: column;
   align-items: stretch;

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Relabel configs"
+            label="Config"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,8 +82,9 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="Labels"
+            label="A time series"
             value={labels}
+            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -97,22 +98,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
-              Documentation
+              Relabeling Stages
             </Button>
           </a>
           <Button

app/vmui/packages/vmui/src/utils/time.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import dayjs from "dayjs";
-import { getNanoTimestamp, parseSupportedDuration } from "./time";
+import { getMillisecondsFromDuration, getNanoTimestamp, parseSupportedDuration } from "./time";
 
 describe("Time utils", () => {
   describe("getNanoTimestamp", () => {
@@ -82,4 +82,19 @@ describe("Time utils", () => {
       expect(parseSupportedDuration("   ")).toBeUndefined();
     });
   });
+
+  describe("getMillisecondsFromDuration", () => {
+    it("should convert valid durations to milliseconds", () => {
+      expect(getMillisecondsFromDuration("1s")).toBe(1000);
+      expect(getMillisecondsFromDuration("1.5s")).toBe(1500);
+      expect(getMillisecondsFromDuration("1m30s")).toBe(90000);
+      expect(getMillisecondsFromDuration("1h 30m")).toBe(5400000);
+      expect(getMillisecondsFromDuration("500ms")).toBe(500);
+    });
+
+    it("should return zero when duration cannot be parsed", () => {
+      expect(getMillisecondsFromDuration("garbage")).toBe(0);
+      expect(getMillisecondsFromDuration("")).toBe(0);
+    });
+  });
 });

app/vmui/packages/vmui/src/utils/time.ts

@@ -16,7 +16,7 @@ const supportedValuesOf = Intl.supportedValuesOf;
 export const supportedTimezones = supportedValuesOf ? supportedValuesOf("timeZone") as string[] : timezones;
 
 // The list of supported units could be the following -
-// https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations
+// https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations
 export const supportedDurations = [
   { long: "years", short: "y", possible: "year" },
   { long: "weeks", short: "w", possible: "week" },
@@ -100,6 +100,10 @@ export const getSecondsFromDuration = (dur: string) => {
   return dayjs.duration(durObject).asSeconds();
 };
 
+export const getMillisecondsFromDuration = (dur: string): number => {
+  return getSecondsFromDuration(dur) * 1000;
+};
+
 const instantQueryViews = [DisplayType.table, DisplayType.code];
 export const getStepFromDuration = (dur: number, histogram?: boolean, displayType?: DisplayType): string => {
   if (displayType && instantQueryViews.includes(displayType)) return roundStep(dur);
@@ -276,4 +280,3 @@ export const getNanoTimestamp = (dateStr: string): bigint => {
   // Return the full timestamp in nanoseconds as a BigInt
   return BigInt(baseMs) * 1000000n + BigInt(extraNano);
 };
-

apptest/testdata.go

@@ -0,0 +1,358 @@
+package apptest
+
+import (
+	"fmt"
+	"slices"
+)
+
+type TestData struct {
+	Samples              []string
+	Step                 int64
+	WantSeries           []map[string]string
+	WantLabels           []string
+	WantLabelValues      []string
+	WantQueryResults     []*QueryResult
+	WantMetadata         map[string][]MetadataEntry
+	WantMetricNamesStats []MetricNamesStatsRecord
+}
+
+func GenerateTestData(prefix string, numMetrics, start, end int64) TestData {
+	d := TestData{
+		Samples:              []string{},
+		Step:                 (end - start) / numMetrics,
+		WantSeries:           make([]map[string]string, numMetrics),
+		WantLabels:           make([]string, numMetrics),
+		WantLabelValues:      make([]string, numMetrics),
+		WantQueryResults:     make([]*QueryResult, numMetrics),
+		WantMetadata:         make(map[string][]MetadataEntry),
+		WantMetricNamesStats: make([]MetricNamesStatsRecord, numMetrics),
+	}
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("%s_%04d", prefix, i)
+		metricHelp := fmt.Sprintf("# HELP %s some help message", metricName)
+		metricType := fmt.Sprintf("# TYPE %s gauge", metricName)
+		labelName := fmt.Sprintf("label_%04d", i)
+		labelValue := fmt.Sprintf("value_%04d", i)
+		value := i
+		timestamp := start + i*d.Step
+		sample := fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+
+		d.Samples = append(d.Samples, metricHelp, metricType, sample)
+		d.WantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		d.WantLabels[i] = labelName
+		d.WantLabelValues[i] = labelValue
+		d.WantQueryResults[i] = &QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+		d.WantMetadata[metricName] = []MetadataEntry{{Help: "some help message", Type: "gauge"}}
+		d.WantMetricNamesStats[i].MetricName = metricName
+	}
+	d.WantLabels = append(d.WantLabels, "__name__", "label")
+	slices.Sort(d.WantLabels)
+	return d
+}
+
+// AssertSeries retrieves metric names from the storage and compares the result
+// with the expected one.
+func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []map[string]string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Series(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			}).Sort()
+		},
+		Want: &PrometheusAPIV1SeriesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		Retries: 1000,
+		FailNow: true,
+	})
+}
+
+// AssertSeriesCount retrieves series count and compares it with expected one.
+func AssertSeriesCount(tc *TestCase, app PrometheusQuerier, tenantID string, start, end int64, want uint64) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series/count response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1SeriesCount(tc.T(), QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+		},
+		Want: &PrometheusAPIV1SeriesCountResponse{
+			Status: "success",
+			Data:   []uint64{want},
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabels retrieves label names from the storage and compares the result
+// with the expected one.
+func AssertLabels(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1Labels(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelsResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabelValues retrieves values for the label whose name is labelName for
+// the series whose name mathes metricNameRE, compares the result with the
+// expected one.
+func AssertLabelValues(tc *TestCase, app PrometheusQuerier, metricNameRE, labelName, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels/.../values response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1LabelValues(tc.T(), labelName, query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelValuesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertQueryResults sends a data query to storage and compares the query
+// result with the expected one.
+func AssertQueryResults(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end, step int64, want []*QueryResult) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/query_range response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1QueryRange(tc.T(), query, QueryOpts{
+				Tenant:      tenantID,
+				Start:       fmt.Sprintf("%d", start),
+				End:         fmt.Sprintf("%d", end),
+				Step:        fmt.Sprintf("%dms", step),
+				MaxLookback: fmt.Sprintf("%dms", step-1),
+				NoCache:     "1",
+			})
+		},
+		Want: &PrometheusAPIV1QueryResponse{
+			Status: "success",
+			Data: &QueryData{
+				ResultType: "matrix",
+				Result:     want,
+			},
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetadata(tc *TestCase, app PrometheusQuerier, metricName, tenantID string, want map[string][]MetadataEntry) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/metadata response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Metadata(tc.T(), metricName, 0, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: &PrometheusAPIV1Metadata{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetricNamesStats(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, want []MetricNamesStatsRecord) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/status/metric_names_stats response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1StatusMetricNamesStats(tc.T(), "", "", metricNameRE, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: MetricNamesStatsResponse{
+			Records: want,
+		},
+		FailNow: true,
+	})
+
+}
+
+// GraphiteTestData holds the data samples in Graphite Pickle format, distance
+// between samples in milliseconds and expected responses for various Graphite
+// API endpoints.
+type GraphiteTestData struct {
+	Samples             []string
+	Step                int64
+	WantMetricsIndex    []string
+	WantMetricsFind     []GraphiteMetric
+	WantMetricsExpand   []string
+	WantRenderedTargets []GraphiteRenderedTarget
+}
+
+// GenerateGraphiteTestData generates Graphite test data.
+func GenerateGraphiteTestData(prefix string, numMetrics, start, end int64) GraphiteTestData {
+	d := GraphiteTestData{
+		Samples:             make([]string, numMetrics),
+		Step:                (end - start) / numMetrics,
+		WantMetricsIndex:    make([]string, numMetrics),
+		WantMetricsFind:     make([]GraphiteMetric, numMetrics),
+		WantMetricsExpand:   make([]string, numMetrics),
+		WantRenderedTargets: make([]GraphiteRenderedTarget, numMetrics),
+	}
+
+	datapoints := make([][2]float64, numMetrics)
+	for i := range numMetrics {
+		timestamp := (start + i*d.Step) / 1000
+		datapoints[i][1] = float64(timestamp)
+	}
+
+	for i := range numMetrics {
+		suffix := fmt.Sprintf("%04d", i)
+		metricName := fmt.Sprintf("%s.%s", prefix, suffix)
+		value := i
+		timestamp := (start + i*d.Step) / 1000
+		sample := fmt.Sprintf(`%s %d %d`, metricName, value, timestamp)
+
+		d.Samples[i] = sample
+		d.WantMetricsIndex[i] = metricName
+		d.WantMetricsFind[i].Id = metricName
+		d.WantMetricsFind[i].Text = suffix
+		d.WantMetricsFind[i].Leaf = 1
+		d.WantMetricsExpand[i] = metricName
+		d.WantRenderedTargets[i].Target = metricName
+		d.WantRenderedTargets[i].Datapoints = slices.Clone(datapoints)
+		d.WantRenderedTargets[i].Datapoints[i][0] = float64(value)
+	}
+	return d
+}
+
+// AssertGraphiteMetricsIndex retrieves all metrics by sending a request to
+// /graphite/metrics/index.json and compares the result with the expected one.
+func AssertGraphiteMetricsIndex(tc *TestCase, app PrometheusQuerier, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/index.json response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsIndex(tc.T(), QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		Retries: 30,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind finds metric names by sending a request to
+// /graphite/metrics/find and compares the result with the expected one.
+func AssertGraphiteMetricsFind(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []GraphiteMetric) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/find response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsFind(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind expands metric names by sending a request to
+// /graphite/metrics/expand and compares the result with the expected one.
+func AssertGraphiteMetricsExpand(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/expand response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsExpand(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteRender retieves metric raw data by sending a request to
+// /graphite/render and compares the result with the expected one.
+func AssertGraphiteRender(tc *TestCase, app PrometheusQuerier, target, tenantID string, from, until, step int64, want []GraphiteRenderedTarget) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/render response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteRender(tc.T(), target, QueryOpts{
+				Tenant:      tenantID,
+				From:        fmt.Sprintf("%d", from/1000),
+				Until:       fmt.Sprintf("%d", until/1000),
+				StorageStep: fmt.Sprintf("%dms", step),
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+}

apptest/tests/max_backfill_age_test.go

@@ -0,0 +1,222 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+				"-retentionPeriod=" + retentionPeriod,
+				"-maxBackfillAge=" + maxBackfillAge,
+			})
+		},
+		stop: func() {
+			tc.StopApp("vmsingle")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+func TestClusterMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&apptest.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage1"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage2"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags:    []string{},
+			})
+		},
+		stop: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+type maxBackfillAgeOpts struct {
+	start func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier
+	stop  func()
+}
+
+func testMaxBackfillAge(tc *apptest.TestCase, opts maxBackfillAgeOpts) {
+	t := tc.T()
+
+	assertSeries := func(app apptest.PrometheusQuerier, prefix string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &apptest.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	assertQueryResults := func(app apptest.PrometheusQuerier, prefix string, start, end, step int64, want []*apptest.QueryResult) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
+				})
+			},
+			Want: &apptest.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &apptest.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	const numMetrics = 1000
+	now := time.Now().UTC()
+	var start, end, step int64
+	emptySeries := []map[string]string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+
+	// Start sut with the same -retentionPeriod and -maxBackfillAge.
+	sut := opts.start("1y", "1y")
+
+	// Verify that samples older than the retention period are rejected.
+	start = now.Add(-365 * 24 * time.Hour).Add(-time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	outsideRetention := genMaxBackfillAgeData("outside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, outsideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "outside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "outside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that samples within the retention period are accepted and
+	// searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention := genMaxBackfillAgeData("inside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Restart sut with -maxBackfillAge shorter than the -retentionPeriod.
+	opts.stop()
+	sut = opts.start("1y", "6M")
+
+	// Verify that new samples older than max backfill age but still within the
+	// retention period are rejected but existing samples are still searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention2 := genMaxBackfillAgeData("inside_retention2", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention2.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention2", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention2", start, end, step, emptyQueryResults)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Verify that the metrics that are outside the backfill window can still
+	// be deleted.
+	sut.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*inside_retention.*"}`, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that the samples that are within the backfill window are accepted
+	// and searchable.
+	start = now.Add(-180 * 24 * time.Hour).UnixMilli()
+	end = now.Add(-180 * 24 * time.Hour).Add(1 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideMaxBackfillAge := genMaxBackfillAgeData("inside_max_backfill_age", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideMaxBackfillAge.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_max_backfill_age", start, end, insideMaxBackfillAge.wantSeries)
+	assertQueryResults(sut, "inside_max_backfill_age", start, end, step, insideMaxBackfillAge.wantQueryResults)
+
+	opts.stop()
+}
+
+type maxBackfillAgeData struct {
+	samples          []string
+	wantSeries       []map[string]string
+	wantQueryResults []*apptest.QueryResult
+}
+
+func genMaxBackfillAgeData(prefix string, numMetrics, start, step int64) maxBackfillAgeData {
+	samples := make([]string, numMetrics)
+	wantSeries := make([]map[string]string, numMetrics)
+	wantQueryResults := make([]*apptest.QueryResult, numMetrics)
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("metric_%s_%04d", prefix, i)
+		labelName := fmt.Sprintf("label_%s_%04d", prefix, i)
+		labelValue := fmt.Sprintf("value_%s_%04d", prefix, i)
+		value := i
+		timestamp := start + i*step
+		samples[i] = fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+		wantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		wantQueryResults[i] = &apptest.QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*apptest.Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+	}
+	return maxBackfillAgeData{samples, wantSeries, wantQueryResults}
+}

apptest/tests/metricsmetadata_test.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -25,7 +26,11 @@ func TestSingleMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
-
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
 		`# HELP metric_name_1 some help message`,
@@ -40,6 +45,12 @@ func TestSingleMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
+
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -52,6 +63,9 @@ func TestSingleMetricsMetadata(t *testing.T) {
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 
@@ -137,6 +151,11 @@ func TestClusterMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
@@ -152,6 +171,11 @@ func TestClusterMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -164,6 +188,9 @@ func TestClusterMetricsMetadata(t *testing.T) {
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 

apptest/tests/prometheus_mock_storage.go

@@ -59,7 +59,7 @@ func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, so
 		}
 
 		if !notMatch {
-			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
+			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples, Histograms: s.Histograms})
 		}
 	}
 

apptest/tests/remote_read_server.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
@@ -86,10 +87,17 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 						samples = append(samples, sample)
 					}
 				}
+				var histograms []prompb.Histogram
+				for _, h := range s.Histograms {
+					if h.Timestamp >= startTs && h.Timestamp < endTs {
+						histograms = append(histograms, h)
+					}
+				}
 				var series prompb.TimeSeries
-				if len(samples) > 0 {
+				if len(samples) > 0 || len(histograms) > 0 {
 					series.Labels = s.Labels
 					series.Samples = samples
+					series.Histograms = histograms
 				}
 				ts[i] = &series
 			}
@@ -317,6 +325,37 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
 	return samples
 }
 
+// GenerateRemoteReadHistogramSeries generates a remote read series
+// with native histogram samples within the given time range.
+func GenerateRemoteReadHistogramSeries(start, end, numOfSamples int64) []*prompb.TimeSeries {
+	timeSeries := &prompb.TimeSeries{
+		Labels: []prompb.Label{
+			{Name: labels.MetricName, Value: "vm_histogram_metric"},
+			{Name: "job", Value: "0"},
+		},
+	}
+
+	delta := (end - start) / numOfSamples
+	mul := int64(0)
+	for t := start; t != end; t += delta {
+		mul++
+		h := &histogram.Histogram{
+			Schema:          0,
+			Count:           uint64(10 * mul),
+			Sum:             25.5 * float64(mul),
+			ZeroThreshold:   0.001,
+			ZeroCount:       uint64(2 * mul),
+			PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+			PositiveBuckets: []int64{mul, 2 * mul},
+			NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+			NegativeBuckets: []int64{4 * mul},
+		}
+		timeSeries.Histograms = append(timeSeries.Histograms, prompb.FromIntHistogram(t*1000, h))
+	}
+
+	return []*prompb.TimeSeries{timeSeries}
+}
+
 func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
 	result := make([]prompb.Label, 0, ls.Len())
 	ls.Range(func(l labels.Label) {

apptest/tests/vmagent_remotewrite_test.go

@@ -517,10 +517,15 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
 		fmt.Sprintf(`-remoteWrite.url=http://%s/insert/multitenant/prometheus/api/v1/write`, sut.Vminsert.HTTPAddr()),
 	})
-
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Metadata: []prompb.MetricMetadata{
 			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary, AccountID: 100},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset, AccountID: 100},
 		},
 	}
 	vmagent.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{Tenant: "multitenant"})

apptest/tests/vmctl_remote_read_mogration_test.go

@@ -75,6 +75,118 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
 	testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
 }
 
+func TestSingleVmctlRemoteReadNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadServer, vmctlFlags)
+}
+
+func TestSingleVmctlRemoteReadStreamNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--remote-read-use-stream=true`,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadStreamServer, vmctlFlags)
+}
+
+// testRemoteReadNativeHistograms verifies that native histograms are migrated
+// as _count, _sum and _bucket series with vmrange labels in the same way
+// as VictoriaMetrics converts native histograms received via Prometheus remote write protocol.
+func testRemoteReadNativeHistograms(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	series := GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2)
+
+	rrs := newRemoteReadServer(t, series)
+	defer rrs.Close()
+
+	vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	tc.Assert(&apptest.AssertOptions{
+		Retries: 300,
+		Msg:     `unexpected native histogram metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			got := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2025-06-11T15:31:10Z",
+				End:   "2025-06-11T15:32:20Z",
+			})
+			got.Sort()
+			return got.Data.Result
+		},
+		Want: expectedNativeHistogramQueryResult(),
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}
+
+// expectedNativeHistogramQueryResult returns the series expected to be stored in VictoriaMetrics
+// after migrating the series generated by GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2).
+func expectedNativeHistogramQueryResult() []*apptest.QueryResult {
+	metric := func(name, vmrange string) map[string]string {
+		m := map[string]string{
+			"__name__": name,
+			"job":      "0",
+		}
+		if vmrange != "" {
+			m["vmrange"] = vmrange
+		}
+		return m
+	}
+	samples := func(v1, v2 float64) []*apptest.Sample {
+		return []*apptest.Sample{
+			{Timestamp: 1749655870000, Value: v1},
+			{Timestamp: 1749655875000, Value: v2},
+		}
+	}
+	resp := &apptest.PrometheusAPIV1QueryResponse{
+		Data: &apptest.QueryData{
+			Result: []*apptest.QueryResult{
+				{Metric: metric("vm_histogram_metric_count", ""), Samples: samples(10, 20)},
+				{Metric: metric("vm_histogram_metric_sum", ""), Samples: samples(25.5, 51)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e+00...-5.000e-01"), Samples: samples(4, 8)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e-03...1.000e-03"), Samples: samples(2, 4)},
+				{Metric: metric("vm_histogram_metric_bucket", "5.000e-01...1.000e+00"), Samples: samples(1, 2)},
+				{Metric: metric("vm_histogram_metric_bucket", "1.000e+00...2.000e+00"), Samples: samples(3, 6)},
+			},
+		},
+	}
+	// sort in the same way as the exported result
+	resp.Sort()
+	return resp.Data.Result
+}
+
 func testRemoteReadProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
 	t := tc.T()
 	t.Helper()

apptest/tests/vmsingle_vmselect_rpc_test.go

@@ -0,0 +1,216 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestMixedPrometheusQueries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data := apptest.GenerateTestData("metric", numMetrics, start, end)
+	emptySeries := []map[string]string{}
+	emptyLabels := []string{}
+	emptyLabelValues := []string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+	emptyMetadata := map[string][]apptest.MetadataEntry{}
+	emptyMetricNamesStats := []apptest.MetricNamesStatsRecord{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	// Ensure vmsingle returns data.
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data.WantSeries)
+	apptest.AssertSeriesCount(tc, vmsingle, "", start, end, numMetrics)
+	apptest.AssertLabels(tc, vmsingle, "metric.*", "", start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmsingle, "metric.*", "label", "", start, end, data.WantLabelValues)
+	apptest.AssertQueryResults(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmsingle, "", "", data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 1
+	}
+	apptest.AssertMetricNamesStats(tc, vmsingle, "", "", data.WantMetricNamesStats)
+
+	// Check that current vmsingle tenant (configured via flags) is tenant1.
+	gotAdminTenantsResponse := vmselect.APIV1AdminTenants(t, apptest.QueryOpts{})
+	wantAdminTenantsResponse := &apptest.AdminTenantsResponse{
+		Status: "success",
+		Data:   []string{tenantID1},
+	}
+	if diff := cmp.Diff(wantAdminTenantsResponse, gotAdminTenantsResponse); diff != "" {
+		t.Fatalf("unexpected tenants (-want, +got):\n%s", diff)
+	}
+
+	// Ensure vmselect returns data for tenant1.
+	apptest.AssertSeries(tc, vmselect, "metric.*", tenantID1, start, end, data.WantSeries)
+	apptest.AssertSeriesCount(tc, vmselect, tenantID1, start, end, numMetrics)
+	apptest.AssertLabels(tc, vmselect, "metric.*", tenantID1, start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID1, start, end, data.WantLabelValues)
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", tenantID1, data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 2
+	}
+	apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID1, data.WantMetricNamesStats)
+
+	// Ensure vmselect does not return any data for tenant2.
+	apptest.AssertSeries(tc, vmselect, "metric.*", tenantID2, start, end, emptySeries)
+	apptest.AssertSeriesCount(tc, vmselect, tenantID2, start, end, 0)
+	apptest.AssertLabels(tc, vmselect, "metric.*", tenantID2, start, end, emptyLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID2, start, end, emptyLabelValues)
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", tenantID2, emptyMetadata)
+	apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID2, emptyMetricNamesStats)
+
+	// Ensure vmselect returns data for multitenant.
+	for _, v := range data.WantSeries {
+		v["vm_account_id"] = strconv.Itoa(accountID1)
+		v["vm_project_id"] = strconv.Itoa(projectID1)
+	}
+	apptest.AssertSeries(tc, vmselect, "metric.*", "multitenant", start, end, data.WantSeries)
+	data.WantLabels = append(data.WantLabels, "vm_account_id", "vm_project_id")
+	apptest.AssertLabels(tc, vmselect, "metric.*", "multitenant", start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", "multitenant", start, end, data.WantLabelValues)
+	for _, v := range data.WantQueryResults {
+		v.Metric["vm_account_id"] = strconv.Itoa(accountID1)
+		v.Metric["vm_project_id"] = strconv.Itoa(projectID1)
+	}
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", "multitenant", start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", "multitenant", data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 3
+	}
+	apptest.AssertMetricNamesStats(tc, vmselect, "", "multitenant", data.WantMetricNamesStats)
+}
+
+func TestMixedDeleteSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data1 := apptest.GenerateTestData("metric1", numMetrics, start, end)
+	data2 := apptest.GenerateTestData("metric2", numMetrics, start, end)
+	emptySeries := []map[string]string{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data1.Samples, apptest.QueryOpts{})
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data2.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	wantSeries12 := slices.Concat(data1.WantSeries, data2.WantSeries)
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, wantSeries12)
+
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric1.*"}`, apptest.QueryOpts{
+		Tenant: tenantID1,
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
+		Tenant: tenantID2,
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
+		Tenant: "multitenant",
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, emptySeries)
+}
+
+func TestMixedGraphiteQueries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data := apptest.GenerateGraphiteTestData("metric", numMetrics, start, end)
+	emptyMetricsIndex := []string{}
+	emptyMetricsFind := []apptest.GraphiteMetric{}
+	emptyMetricsExpand := []string{}
+	emptyRenderedTargets := []apptest.GraphiteRenderedTarget{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.GraphiteWrite(tc.T(), data.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	// Ensure vmsingle returns data.
+	apptest.AssertGraphiteMetricsIndex(tc, vmsingle, "", data.WantMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmsingle, "metric.*", "", data.WantMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmsingle, "metric.*", "", data.WantMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantRenderedTargets)
+
+	// Ensure vmselect returns data for tenant1.
+	apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID1, data.WantMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID1, data.WantMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID1, data.WantMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantRenderedTargets)
+
+	// Ensure vmselect does not return any data for tenant2.
+	apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID2, emptyMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID2, emptyMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID2, emptyMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyRenderedTargets)
+}

apptest/vmsingle.go

@@ -25,12 +25,14 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
 			"-httpListenAddr":     "127.0.0.1:0",
 			"-graphiteListenAddr": "127.0.0.1:0",
 			"-opentsdbListenAddr": "127.0.0.1:0",
+			"-vmselectAddr":       "127.0.0.1:0",
 		},
 		extractREs: []*regexp.Regexp{
 			storageDataPathRE,
 			httpListenAddrRE,
 			graphiteListenAddrRE,
 			openTSDBListenAddrRE,
+			vmselectAddrRE,
 		},
 		output: output,
 	})
@@ -43,6 +45,7 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
 		httpListenAddr:     stderrExtracts[1],
 		graphiteListenAddr: stderrExtracts[2],
 		openTSDBListenAddr: stderrExtracts[3],
+		vmselectAddr:       stderrExtracts[4],
 	}), nil
 }
 
@@ -51,6 +54,7 @@ type vmsingleRuntimeValues struct {
 	httpListenAddr     string
 	graphiteListenAddr string
 	openTSDBListenAddr string
+	vmselectAddr       string
 }
 
 func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
@@ -85,6 +89,7 @@ func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 		},
 		storageDataPath: rt.storageDataPath,
 		httpListenAddr:  rt.httpListenAddr,
+		vmselectAddr:    rt.vmselectAddr,
 	}
 }
 
@@ -99,6 +104,7 @@ type Vmsingle struct {
 
 	storageDataPath string
 	httpListenAddr  string
+	vmselectAddr    string
 }
 
 // HTTPAddr returns the address at which the vminsert process is
@@ -107,6 +113,12 @@ func (app *Vmsingle) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
+// VmselectAddr returns the address at which the vmsingle process is listening
+// for vmselect connections.
+func (app *Vmsingle) VmselectAddr() string {
+	return app.vmselectAddr
+}
+
 // String returns the string representation of the vmsingle app state.
 func (app *Vmsingle) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{

dashboards/query-stats.json

@@ -1,4 +1,59 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_VICTORIALOGS",
+      "label": "VictoriaLogs",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "victoriametrics-logs-datasource",
+      "pluginName": "VictoriaLogs"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "12.4.3"
+    },
+    {
+      "type": "panel",
+      "id": "logs",
+      "name": "Logs",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "text",
+      "name": "Text",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "victoriametrics-logs-datasource",
+      "name": "VictoriaLogs",
+      "version": "0.29.0"
+    }
+  ],
   "annotations": {
     "list": [
       {
@@ -18,7 +73,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 861,
   "links": [
     {
       "icon": "doc",
@@ -78,7 +132,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -109,7 +164,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -117,7 +172,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| type:=\"instant\"\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| type:=\"instant\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -140,7 +195,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -171,7 +227,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -179,7 +235,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| type:=\"range\"\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| type:=\"range\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -202,7 +258,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -233,7 +290,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -241,7 +298,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| series_fetched:=0\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| series_fetched:=0\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -265,7 +322,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           },
@@ -297,7 +355,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -305,7 +363,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| stats min(start_ms)\n",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| stats min(start_ms)\n",
           "queryType": "stats",
           "refId": "A"
         }
@@ -327,10 +385,6 @@
       "type": "row"
     },
     {
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "gridPos": {
         "h": 2,
         "w": 24,
@@ -347,8 +401,7 @@
         "content": "To filter by specific query copy its hash from the table and put it into `query_hash` filter on the top. To disable filtering enter `*`.",
         "mode": "markdown"
       },
-      "pluginVersion": "11.6.0",
-      "title": "",
+      "pluginVersion": "12.4.3",
       "transparent": true,
       "type": "text"
     },
@@ -367,6 +420,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -374,7 +430,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -423,18 +480,6 @@
                 "value": 204
               }
             ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "duration_max"
-            },
-            "properties": [
-              {
-                "id": "custom.width",
-                "value": 122
-              }
-            ]
           }
         ]
       },
@@ -447,18 +492,10 @@
       "id": 4,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -466,7 +503,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -478,7 +515,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -533,6 +570,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -547,7 +585,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -580,7 +619,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -588,7 +627,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(execution_duration_ms) execution_duration_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(execution_duration_ms) execution_duration_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -611,6 +650,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -618,7 +660,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -680,18 +723,10 @@
       "interval": "1m",
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -699,7 +734,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -711,7 +746,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -766,6 +801,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -780,7 +816,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -813,7 +850,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -821,7 +858,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(series_fetched) series_fetched_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(series_fetched) series_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -844,6 +881,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -851,7 +891,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -912,18 +953,10 @@
       "id": 5,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -931,7 +964,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -942,8 +975,10 @@
           "id": "extractFields",
           "options": {
             "delimiter": ",",
+            "format": "json",
+            "keepTime": false,
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -998,6 +1033,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1012,7 +1048,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1045,7 +1082,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1053,7 +1090,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(samples_fetched) samples_fetched_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(samples_fetched) samples_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1076,6 +1113,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1083,7 +1123,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1144,18 +1185,10 @@
       "id": 11,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1163,7 +1196,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1175,7 +1208,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1230,6 +1263,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1244,7 +1278,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1277,7 +1312,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1285,7 +1320,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(bytes) bytes_fetched",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(bytes) bytes_fetched",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1308,6 +1343,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1315,7 +1353,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1388,18 +1427,10 @@
       "id": 13,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1407,7 +1438,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1419,7 +1450,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1474,6 +1505,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1488,7 +1520,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1521,7 +1554,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1529,7 +1562,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1552,6 +1585,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1559,7 +1595,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1620,18 +1657,10 @@
       "id": 18,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1639,7 +1668,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1651,7 +1680,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1706,6 +1735,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1720,7 +1750,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1753,7 +1784,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1761,7 +1792,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| stats max(range_ms) range_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| stats max(range_ms) range_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1770,7 +1801,7 @@
       "type": "timeseries"
     },
     {
-      "collapsed": false,
+      "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1778,52 +1809,59 @@
         "y": 56
       },
       "id": 12,
-      "panels": [],
-      "title": "Query log",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "victoriametrics-logs-datasource",
-        "uid": "${ds}"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 14,
-        "w": 24,
-        "x": 0,
-        "y": 57
-      },
-      "id": 6,
-      "options": {
-        "dedupStrategy": "none",
-        "enableInfiniteScrolling": false,
-        "enableLogDetails": true,
-        "prettifyLogMessage": false,
-        "showCommonLabels": false,
-        "showLabels": false,
-        "showTime": false,
-        "sortOrder": "Descending",
-        "wrapLogMessage": false
-      },
-      "pluginVersion": "11.6.0",
-      "targets": [
+      "panels": [
         {
           "datasource": {
             "type": "victoriametrics-logs-datasource",
             "uid": "${ds}"
           },
-          "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| limit 200",
-          "queryType": "instant",
-          "refId": "A"
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 14,
+            "w": 24,
+            "x": 0,
+            "y": 57
+          },
+          "id": 6,
+          "options": {
+            "dedupStrategy": "none",
+            "detailsMode": "sidebar",
+            "enableInfiniteScrolling": false,
+            "enableLogDetails": true,
+            "fontSize": "small",
+            "prettifyLogMessage": false,
+            "showCommonLabels": false,
+            "showControls": true,
+            "showLabels": false,
+            "showTime": false,
+            "sortOrder": "Descending",
+            "syntaxHighlighting": false,
+            "unwrappedColumns": false,
+            "wrapLogMessage": true
+          },
+          "pluginVersion": "12.4.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-logs-datasource",
+                "uid": "${ds}"
+              },
+              "direction": "desc",
+              "editorMode": "code",
+              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| limit 200",
+              "queryType": "instant",
+              "refId": "A"
+            }
+          ],
+          "title": "Raw logs",
+          "type": "logs"
         }
       ],
-      "title": "Raw logs",
-      "type": "logs"
+      "title": "Query log",
+      "type": "row"
     },
     {
       "collapsed": true,
@@ -1831,7 +1869,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 71
+        "y": 57
       },
       "id": 17,
       "panels": [
@@ -1848,7 +1886,7 @@
             "h": 14,
             "w": 24,
             "x": 0,
-            "y": 70
+            "y": 58
           },
           "id": 15,
           "options": {
@@ -1857,12 +1895,14 @@
             "enableLogDetails": true,
             "prettifyLogMessage": false,
             "showCommonLabels": false,
+            "showControls": false,
             "showLabels": false,
             "showTime": false,
             "sortOrder": "Descending",
+            "unwrappedColumns": false,
             "wrapLogMessage": false
           },
-          "pluginVersion": "11.6.0",
+          "pluginVersion": "12.4.3",
           "targets": [
             {
               "datasource": {
@@ -1870,7 +1910,7 @@
                 "uid": "${ds}"
               },
               "editorMode": "code",
-              "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:in($query_hash)",
+              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:$query_hash",
               "queryType": "instant",
               "refId": "A"
             }
@@ -1885,7 +1925,7 @@
   ],
   "preload": false,
   "refresh": "",
-  "schemaVersion": 41,
+  "schemaVersion": 42,
   "tags": [
     "victoriametrics",
     "victorialogs"
@@ -1894,8 +1934,9 @@
     "list": [
       {
         "current": {
-          "text": "VictoriaLogs",
-          "value": "PD775F2863313E6C7"
+          "text": "",
+          "value": "${ds}",
+          "selected": true
         },
         "name": "ds",
         "options": [],
@@ -1933,21 +1974,17 @@
           }
         ],
         "query": "5,10,15,20",
-        "type": "custom"
+        "type": "custom",
+        "valuesFormat": "csv"
       },
       {
         "allValue": "*",
-        "current": {
-          "text": "All",
-          "value": [
-            "$__all"
-          ]
-        },
+        "current": {},
         "datasource": {
           "type": "victoriametrics-logs-datasource",
           "uid": "${ds}"
         },
-        "definition": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+        "definition": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
         "includeAll": true,
         "multi": true,
         "name": "tenant",
@@ -1955,12 +1992,13 @@
         "query": {
           "field": "tenant",
           "limit": 25,
-          "query": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+          "query": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
           "refId": "VictoriaLogsVariableQueryEditor-VariableQuery",
           "type": "fieldValue"
         },
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
+        "regexApplyTo": "value",
         "type": "query"
       },
       {
@@ -2000,5 +2038,6 @@
   "timezone": "browser",
   "title": "Query Stats (cluster)",
   "uid": "feg3od1zt1fy8e",
-  "version": 1
-}
+  "version": 1,
+  "weekStart": ""
+}