make linter happy

In https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10607 vmagent
metrics relabling debug UI has been refined. There are a few changes
that make sence in VMUI too:
- Rename "Labels" to "A time series" to highlight that a single
Prometheus time series can be provided in text format.
- Rename "Relabel Configs" to "Config". It should be clear from the page
contex what config is meant. Also there are links to documention.
- Add placeholder. It hints what could be placed into "A time series"
textarea.

.github/workflows/build.yml

@@ -65,7 +65,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/changelog-linter.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           fetch-depth: 0  # we need full history for commit verification
           persist-credentials: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/codeql-analysis-go.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           path: __vm
           persist-credentials: false
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: 'vm-runner'
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -79,7 +79,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -106,7 +106,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/vmui.yml

@@ -26,13 +26,11 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

.golangci.yml

@@ -23,3 +23,4 @@ linters:
         text: 'SA(4003|1019|5011):'
     paths:
       - ^app/vmui/
+      - app/vmui/packages/vmui/node_modules/

Makefile

@@ -447,7 +447,7 @@ vet:
 	go vet ./app/...
 	go vet ./apptest/...
 
-check-all: fmt vet golangci-lint govulncheck
+check-all: fmt vet golangci-lint
 
 clean-checkers: remove-golangci-lint remove-govulncheck
 
@@ -471,8 +471,9 @@ test-full-386:
 
 apptest:
 	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+	go test ./apptest/... -skip="^Test(Cluster|Mixed|Legacy).*"
 
+# App tests for legacy indexDB
 apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
@@ -489,6 +490,22 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
+# App tests for mixed setups where vmsingle and vmcluster coexist.
+apptest-mixed: victoria-metrics-race
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VMSELECT_PATH=$${DIR}/vmselect-prod \
+	go test ./apptest/tests -run="^TestMixed.*"
+
 benchmark:
 	go test -run=NO_TESTS -bench=. ./lib/...
 	go test -run=NO_TESTS -bench=. ./app/...

app/victoria-metrics/main.go

@@ -89,7 +89,7 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
-	vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()

app/vmagent/influx/request_handler.go

@@ -51,7 +51,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vmagent/main.go

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmagent/opentelemetry/request_handler.go

@@ -63,6 +63,7 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range tss {
@@ -82,7 +83,19 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	var metadataTotal int
 	if prommetadata.IsEnabled() {
-		ctx.WriteRequest.Metadata = mms
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: mm.AccountID,
+				ProjectID: mm.ProjectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
 		metadataTotal = len(mms)
 	}
 

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 }
 
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+	if len(extraLabels) == 0 && !prommetadata.IsEnabled() && at == nil {
+		return insertRowsFast(at, timeseries)
+	}
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -102,3 +105,17 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }
+
+func insertRowsFast(at *auth.Token, timeseries []prompb.TimeSeries) error {
+	rowsTotal := 0
+	for i := range timeseries {
+		rowsTotal += len(timeseries[i].Samples)
+	}
+	wr := &prompb.WriteRequest{Timeseries: timeseries}
+	if !remotewrite.TryPush(at, wr) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,
@@ -187,7 +193,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	return c
 }
 
-func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
+func (c *client) init(argIdx int, sanitizedURL string) {
 	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
@@ -204,11 +210,20 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
-	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(concurrency)
-	})
-	for range concurrency {
-		c.wg.Go(c.runWorker)
+	workers := queues.GetOptionalArg(argIdx)
+	if workers <= 0 {
+		workers = 1
+	}
+	inmemoryWorkers := inmemoryQueues.GetOptionalArg(argIdx)
+	for range inmemoryWorkers {
+		c.wg.Go(func() {
+			c.runWorker(c.fq.MustReadInMemoryBlockBlocking)
+		})
+	}
+	for range workers {
+		c.wg.Go(func() {
+			c.runWorker(c.fq.MustReadBlock)
+		})
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }
@@ -302,12 +317,12 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	return cfg, nil
 }
 
-func (c *client) runWorker() {
+func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 	var ok bool
 	var block []byte
 	ch := make(chan bool, 1)
 	for {
-		block, ok = c.fq.MustReadBlock(block[:0])
+		block, ok = readBlock(block[:0])
 		if !ok {
 			return
 		}

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmagent/remotewrite/remotewrite.go

@@ -12,19 +12,18 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -66,6 +65,9 @@ var (
 	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+	inmemoryQueues = flagutil.NewArrayInt("remoteWrite.inmemoryQueues", 0, "The number of additional workers per each -remoteWrite.url, which send only recently ingested data from the in-memory queue, "+
+		"while the file-based queue at -remoteWrite.tmpDataPath is drained by workers configured via -remoteWrite.queues. "+
+		"This reduces delivery lag for fresh samples when the file-based queue contains a backlog accumulated during remote storage outages.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -103,6 +105,9 @@ var (
 		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
 	disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
+
+	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
 )
 
 var (
@@ -159,8 +164,8 @@ func InitSecretFlags() {
 }
 
 var (
-	shardByURLLabelsMap       map[string]struct{}
-	shardByURLIgnoreLabelsMap map[string]struct{}
+	shardByURLLabelsFilter       []string
+	shardByURLIgnoreLabelsFilter []string
 )
 
 // Init initializes remotewrite.
@@ -207,8 +212,8 @@ func Init() {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
 	}
-	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
-	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
+	shardByURLLabelsFilter = slices.Clone(*shardByURLLabels)
+	shardByURLIgnoreLabelsFilter = slices.Clone(*shardByURLIgnoreLabels)
 
 	initLabelsGlobal()
 
@@ -304,6 +309,10 @@ func initRemoteWriteCtxs(urls []string) {
 	}
 	fs.RegisterPathFsMetrics(*tmpDataPath)
 
+	if slices.Contains(*enableMdx, true) && *shardByURL {
+		logger.Fatalf("-remoteWrite.mdx.enable and -remoteWrite.shardByURL cannot be set to true simultaneously.")
+	}
+
 	if *shardByURL {
 		consistentHashNodes := make([]string, 0, len(urls))
 		for i, url := range urls {
@@ -562,6 +571,14 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
 			mm.ProjectID = at.ProjectID
 		}
 	}
+	tmp := mms[:0]
+	for _, mm := range mms {
+		if timeserieslimits.IsMetricMetadataExceeding(&mm) {
+			continue
+		}
+		tmp = append(tmp, mm)
+	}
+	mms = tmp
 	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
 	// Since metadata is usually small and there is no guarantee that metadata can be sent to
 	// the same remote storage with the corresponding metrics.
@@ -695,18 +712,18 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 
 	for _, ts := range tssBlock {
 		hashLabels := ts.Labels
-		if len(shardByURLLabelsMap) > 0 {
+		if len(shardByURLLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLLabelsMap[label.Name]; ok {
+				if slices.Contains(shardByURLLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
 			tmpLabels.Labels = hashLabels
-		} else if len(shardByURLIgnoreLabelsMap) > 0 {
+		} else if len(shardByURLIgnoreLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
+				if !slices.Contains(shardByURLIgnoreLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
@@ -807,34 +824,26 @@ var (
 // it omits the '=' separator between label name and value for backward compatibility.
 // Changing it would re-shard all series across remoteWrite targets.
 func getLabelsHashForShard(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
 func getLabelsHash(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, '=')
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString("=")
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
-var labelsHashBufPool bytesutil.ByteBufferPool
-
 func logSkippedSeries(labels []prompb.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
@@ -859,6 +868,7 @@ type remoteWriteCtx struct {
 
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
+	mdxFilter    *mdx.Filter
 
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
@@ -873,6 +883,7 @@ type remoteWriteCtx struct {
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
+	mdxRowsPreserved       *metrics.Counter
 
 	pushFailures                 *metrics.Counter
 	metadataDroppedOnPushFailure *metrics.Counter
@@ -906,7 +917,8 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
-	queuesSize := queues.GetOptionalArg(argIdx)
+	inmemoryQueueSize := inmemoryQueues.GetOptionalArg(argIdx)
+	queuesSize := queues.GetOptionalArg(argIdx) + inmemoryQueueSize
 	if queuesSize > maxQueues {
 		queuesSize = maxQueues
 	} else if queuesSize <= 0 {
@@ -923,7 +935,13 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	if maxInmemoryBlocks < 2 {
 		maxInmemoryBlocks = 2
 	}
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
+	fqOpts := persistentqueue.OpenFastQueueOpts{
+		MaxInmemoryBlocks:      maxInmemoryBlocks,
+		MaxPendingBytes:        maxPendingBytes,
+		IsPQDisabled:           isPQDisabled,
+		PrioritizeInmemoryData: inmemoryQueueSize > 0,
+	}
+	fq := persistentqueue.MustOpenFastQueueWithOpts(queuePath, sanitizedURL, fqOpts)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
@@ -936,6 +954,9 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		}
 		return 0
 	})
+	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, sanitizedURL), func() float64 {
+		return float64(queuesSize)
+	})
 
 	var c *client
 	switch remoteWriteURL.Scheme {
@@ -944,7 +965,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, queuesSize, sanitizedURL)
+	c.init(argIdx, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
@@ -959,7 +980,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
-
 	rwctx := &remoteWriteCtx{
 		idx:            argIdx,
 		fq:             fq,
@@ -976,6 +996,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	}
 	rwctx.initStreamAggrConfig()
 
+	if enableMdx.GetOptionalArg(argIdx) {
+		mdxFilter := mdx.NewFilter()
+		rwctx.mdxFilter = mdxFilter
+		rwctx.mdxRowsPreserved = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_preserved_total{path=%q,url=%q}`, queuePath, sanitizedURL))
+		_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_mdx_tracked_instances{path=%q,url=%q}`, queuePath, sanitizedURL), func() float64 {
+			return float64(mdxFilter.VMInstancesCount())
+		})
+
+	}
+
 	return rwctx
 }
 
@@ -989,6 +1019,11 @@ func (rwctx *remoteWriteCtx) MustStop() {
 		rwctx.deduplicator.MustStop()
 		rwctx.deduplicator = nil
 	}
+	if rwctx.mdxFilter != nil {
+		rwctx.mdxFilter.MustStop()
+		rwctx.mdxFilter = nil
+		rwctx.mdxRowsPreserved = nil
+	}
 
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
@@ -1004,6 +1039,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 	rwctx.rowsPushedAfterRelabel = nil
 	rwctx.rowsDroppedByRelabel = nil
+
 }
 
 // TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1011,16 +1047,41 @@ func (rwctx *remoteWriteCtx) MustStop() {
 // TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
 func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	var rctx *relabelCtx
+	var mctx *mdx.Ctx
 	var v *[]prompb.TimeSeries
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if mctx != nil {
+			mdx.PutContext(mctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
+	if rwctx.mdxFilter != nil {
+		mctx = mdx.GetContext()
+		// Make a copy of tss before applying relabeling in order to prevent
+		// from affecting time series for other remoteWrite.mdx configs.
+		copyTimeSeriesIfNeeded()
+		tss = rwctx.mdxFilter.Filter(mctx, tss)
+		if len(tss) == 0 {
+			return true
+		}
+		rowsCount := getRowsCount(tss)
+		rwctx.mdxRowsPreserved.Add(rowsCount)
+	}
+
 	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
@@ -1030,8 +1091,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
@@ -1049,8 +1109,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		} else if rwctx.streamAggrDropInput {
@@ -1058,8 +1117,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropUnaggregatedSeries(tss, matchIdxs.B)
 		}
@@ -1178,15 +1236,6 @@ func getRowsCount(tss []prompb.TimeSeries) int {
 	}
 	return rowsCount
 }
-
-func newMapFromStrings(a []string) map[string]struct{} {
-	m := make(map[string]struct{}, len(a))
-	for _, s := range a {
-		m[s] = struct{}{}
-	}
-	return m
-}
-
 func getMaxHourlySeries() int {
 	limit := *maxHourlySeries
 	if limit == -1 || limit > math.MaxInt32 {

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )
@@ -282,7 +289,8 @@ func processFlags() {
 
 func setUp() {
 	const maxConcurrentRequests = 4
-	vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	maxQueueDuration := 5 * time.Second
+	vmstorage.Init(maxConcurrentRequests, maxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {

app/vmalert/config/config_test.go

@@ -145,10 +145,10 @@ func TestRuleValidate(t *testing.T) {
 }
 
 func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+	f := func(data []byte, validateExpressions bool, errStrExpected string) {
 		t.Helper()
 
-		err := group.Validate(nil, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, nil, validateExpressions)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -158,275 +158,238 @@ func TestGroupValidate_Failure(t *testing.T) {
 		}
 	}
 
-	f(&Group{}, false, "group name must be set")
+	f([]byte(`
+groups:
+- name: ""
+`), false, "group name must be set")
 
-	f(&Group{
-		Name: "both record and alert are not set",
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
+	f([]byte(`
+groups:
+- name: both record and alert are not set
+  rules:
+  - expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+  - expr: "sumSeries(time('foo.bar',10))"
+`), false, "invalid rule")
 
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutil.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
+	f([]byte(`
+groups:
+- name: negative interval
+  interval: -1ms
+`), false, "interval shouldn't be lower than 0")
 
-	f(&Group{
-		Name:       "too big eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big eval_offset
+  interval: 1m
+  eval_offset: 2m
+`), false, "eval_offset should be smaller than interval")
 
-	f(&Group{
-		Name:       "too big negative eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(-2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big negative eval_offset
+  interval: 1m
+  eval_offset: -2m
+`), false, "eval_offset should be smaller than interval")
 
-	limit := -1
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: &limit,
-	}, false, "invalid limit")
+	f([]byte(`
+groups:
+- name: wrong limit
+  limit: -1
+`), false, "invalid limit")
 
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
+	f([]byte(`
+groups:
+- name: wrong concurrency
+  concurrency: -1
+`), false, "invalid concurrency")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+  - alert: alert
+    expr: up == 1
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
+	f([]byte(`
+groups:
+- name: test thanos
+  type: thanos
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), true, "unknown datasource type")
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus expr",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus expr
+  type: prometheus
+  rules:
+  - record: record
+    expr: "up | 0"
+`), true, "bad MetricsQL expr")
 
-	f(&Group{
-		Name: "test graphite expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite expr
+  type: graphite
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: some-description
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "stats count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr multipart
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test graphite with prometheus expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				ID:     1,
-				Expr:   "sumSeries(time('foo.bar',10))",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Record: "r2",
-				ID:     2,
-				Expr:   "sum(up == 0 ) by (host)",
-			},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite with prometheus expr
+  type: graphite
+  rules:
+  - record: r1
+    expr: "sumSeries(time('foo.bar',10))"
+    for: 10ms
+  - record: r2
+    expr: "sum(up == 0 ) by (host)"
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs with prometheus exp",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "sum(up == 0 ) by (host)",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs with prometheus expr
+  type: vlogs
+  rules:
+  - record: r1
+    expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test prometheus with vlogs exp",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "* | stats by (path) count()",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus with vlogs expr
+  type: prometheus
+  rules:
+  - record: r1
+    expr: "* | stats by (path) count()"
+    for: 10ms
+`), true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+	f := func(data []byte, validateAnnotations, validateExpressions bool) {
 		t.Helper()
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
 			validateTplFn = notifier.ValidateTemplates
 		}
-		err := group.Validate(validateTplFn, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, validateTplFn, validateExpressions)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	}
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: "up | 0"
+`), false, false)
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
-				},
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, false)
 
 	// validate annotations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
-{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
-  {{ . | first | value | humanize1024 }}B
-{{ end }}`,
-				},
-			},
-		},
-	}, true, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "\n{{ with printf \"node_memory_MemTotal{job='node',instance='%s'}\" \"localhost\" | query }}\n  {{ . | first | value | humanize1024 }}B\n{{ end }}"
+`), true, false)
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
-	f(&Group{
-		Name: "test victorialogs",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
+	f([]byte(`
+groups:
+- name: test prometheus
+  type: prometheus
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
+
+	f([]byte(`
+groups:
+- name: test victorialogs
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: " _time: 1m | stats count(*) as requests"
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
 }
 
 func TestHashRule_NotEqual(t *testing.T) {

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmalert/rule/group.go

@@ -97,6 +97,7 @@ type groupMetrics struct {
 	iterationMissed   *metrics.Counter
 	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
+	iterationLimit    *metrics.Gauge
 }
 
 // merges group rule labels into result map
@@ -336,6 +337,12 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
+	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_rule_group_results_limit{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		limit := g.Limit
+		g.mu.RUnlock()
+		return float64(limit)
+	})
 	for i := range g.Rules {
 		g.Rules[i].registerMetrics(g.metrics.set)
 	}

app/vmalert/rule/recording_test.go

@@ -457,12 +457,10 @@ func TestSetIntervalAsTimeFilter(t *testing.T) {
 	f(`* | count()`, "vlogs", true)
 	f(`error OR _time:5m  | count()`, "vlogs", true)
 	f(`(_time: 5m AND error) OR (_time: 5m AND warn) | count()`, "vlogs", true)
-	f(`* | error OR _time:5m | count()`, "vlogs", true)
 
 	f(`_time:5m | count()`, "vlogs", false)
 	f(`_time:2023-04-25T22:45:59Z | count()`, "vlogs", false)
 	f(`error AND _time:5m | count()`, "vlogs", false)
-	f(`* | error AND _time:5m | count()`, "vlogs", false)
 }
 
 func TestRecordingRuleExec_Partial(t *testing.T) {

app/vmalert/rule/web.go

@@ -75,10 +75,13 @@ type GroupAlerts struct {
 // ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type ApiRule struct {
-	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
+	// Rule state must be one of these under following scenarios:
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state. (only for alerting rules)
+	//  "firing": at least 1 alert in the rule in firing state. (only for alerting rules)
+	//  "inactive": rule's last evaluation was successful but no alert in the rule in firing or pending state. (only for alerting rules)
+	//  "unhealthy": rule's last evaluation was failed with error. (for both alerting and recording rules)
+	// 	"nomatch": rule's last evaluation was successful but no time series matched the rule's expression. (for both alerting and recording rules)
+	//  "ok": the recording rule's last evaluation was successful. (only for recording rules)
 	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
@@ -237,6 +240,7 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 }
 
 func (r *ApiRule) ExtendState() {
+	// if alerting rule already has alerts, then state is already set to either "pending" or "firing" and we don't need to change it
 	if len(r.Alerts) > 0 {
 		return
 	}

app/vmalert/web.go

@@ -115,7 +115,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		WriteRule(w, r, rule)
 		return true
-	// current used by old vmalert UI and Grafana Alerts
+	// used by old vmalert UI
 	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
@@ -128,6 +128,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			state = rf.states[0]
 			rf.states = rf.states[:1]
 		}
+		// enable extendedStates by default for vmalert UI
+		rf.extendedStates = true
 		lr := rh.groups(rf)
 		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
@@ -543,6 +545,8 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
+			// extendedStates is used by the vmalert UI to extend the rule state with values such as "nomatch" and "unhealthy".
+			// those states are not supported by Grafana and not part of the Prometheus API spec yet
 			if rf.extendedStates {
 				rule.ExtendState()
 			}

app/vmalert/web.qtpl

@@ -130,9 +130,12 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
-                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
+                                {% if g.States["inactive"] > 0 %}<span class="badge bg-light text-success border border-success" title="None of the alert instances is in a pending or firing state">{%d g.States["inactive"] %} inactive</span> {% endif %}
+                                {% if g.States["pending"] > 0 %}<span class="badge bg-warning text-dark" title="At least one alert instance is pending">{%d g.States["pending"] %} pending</span> {% endif %}
+                                {% if g.States["firing"] > 0 %}<span class="badge bg-danger" title="At least one alert instance is firing">{%d g.States["firing"] %} firing</span> {% endif %}
+                                {% if g.States["ok"] > 0 %}<span class="badge bg-success" title="Recording rule last evaluation succeeded">{%d g.States["ok"] %} ok</span>{% endif %}
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Last evaluation failed with an error">{%d g.States["unhealthy"] %} unhealthy</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Rule expression matched no time series">{%d g.States["nomatch"] %} nomatch</span> {% endif %}
                             </span>
                         </span>
                     </span>
@@ -169,7 +172,7 @@
                             <thead>
                                 <tr>
                                     <th scope="col" class="w-60">Rule</th>
-                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
+                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series Returned</th>
                                     <th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -188,8 +191,21 @@
                                                     {% else %}
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
-                                                    |
+                                                    {% if r.State == "inactive" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "pending" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "firing" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "ok" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "unhealthy" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "nomatch" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% endif %}
                                                     {%= seriesFetchedWarn(prefix, &r) %}
+                                                    |
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">

app/vmauth/auth_config.go

@@ -118,9 +118,10 @@ type AccessLogFilters struct {
 }
 
 func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
-	if ui.AccessLog == nil {
+	if ui == nil || ui.AccessLog == nil {
 		return
 	}
+
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -134,6 +135,17 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
 		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
+// hasAnyURLs reports whether ui has at least one backend URL route configured.
+// It is used only for unauthorized_user config section, since other users
+// must always have either URLPrefix or URLMaps set.
+func (ui *UserInfo) hasAnyURLs() bool {
+	if ui == nil {
+		return false
+	}
+
+	return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`
@@ -983,8 +995,11 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-		if err := ui.initURLs(); err != nil {
-			return nil, err
+
+		if ui.hasAnyURLs() {
+			if err := ui.initURLs(); err != nil {
+				return nil, err
+			}
 		}
 
 		metricLabels, err := ui.getMetricLabels()

app/vmauth/example_config.yml

@@ -140,6 +140,18 @@ users:
   - "ProjectID: {{.MetricsProjectID}}"
   url_prefix: "http://vminsert:8480/insert/prometheus"
 
+# JWT-based routing that relies solely on custom claims.
+# The `vm_access` claim is missing, default value will be used.
+# e.g. {"role": "admin"}.
+- name: jwt-custom-claims
+  jwt:
+    skip_verify: true
+    vm_default_access_claim:
+        metrics_account_id: 1
+    match_claims:
+      role: admin
+  url_prefix: "http://vmselect-admin:8481/select/0/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -65,6 +65,8 @@ type JWTConfig struct {
 	MatchClaims       map[string]string `yaml:"match_claims,omitempty"`
 	parsedMatchClaims []*jwt.Claim
 
+	DefaultVMAccessClaim *jwt.VMAccessClaim `yaml:"default_vm_access_claim,omitempty"`
+
 	// verifierPool is used to verify JWT tokens.
 	// It is initialized from PublicKeys and/or PublicKeyFiles.
 	// In this case, it is initialized once at config reload and never updated until next reload
@@ -432,7 +434,6 @@ func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
 			}
 			if strings.Contains(p, placeholderPrefix) {
 				return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
-
 			}
 		}
 		for param, values := range bu.Query() {
@@ -487,7 +488,6 @@ func hasAnyPlaceholders(u *url.URL) bool {
 				return true
 			}
 		}
-
 	}
 	return false
 }

app/vmauth/main.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/textproto"
@@ -31,6 +32,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
 var (
@@ -173,11 +175,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
-		if ui != nil {
+		if ui.hasAnyURLs() {
 			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
+		ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 		handleMissingAuthorizationError(w)
 		return true
 	}
@@ -191,17 +194,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
 		defer putToken(tkn)
-		processUserRequest(w, r, ui, tkn)
-		return true
+		// Call processUserRequest only if the token contains the vm_access claim
+		// or a default claim is configured; otherwise fall through to unauthorized_user.
+		if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
+			processUserRequest(w, r, ui, tkn)
+			return true
+		}
 	}
 
 	uu := authConfig.Load().UnauthorizedUser
-	if uu != nil {
+	if uu.hasAnyURLs() {
 		processUserRequest(w, r, uu, nil)
 		return true
 	}
 
 	invalidAuthTokenRequests.Inc()
+	slowdownUnauthorizedResponse(r)
+	uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 	if *logInvalidAuthTokens {
 		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 		err = &httpserver.ErrorWithStatusCode{
@@ -377,7 +386,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 			return nil, &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
 					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
+				StatusCode: http.StatusRequestTimeout,
 			}
 		}
 
@@ -424,8 +433,12 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		}
 		targetURL := bu.url
 		if tkn != nil {
+			vmac := tkn.VMAccess()
+			if !tkn.HasVMAccessClaim() {
+				vmac = ui.JWT.DefaultVMAccessClaim
+			}
 			// for security reasons allow templating only for configured url values and headers
-			targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
+			targetURL, hc = replaceJWTPlaceholders(bu, hc, vmac)
 		}
 		if isDefault {
 			// Don't change path and add request_path query param for default route.
@@ -881,3 +894,20 @@ func debugInfo(u *url.URL, r *http.Request) string {
 	fmt.Fprint(s, ")")
 	return s.String()
 }
+
+// slowdownUnauthorizedResponse adds a random delay in the [2..3] seconds range before returning an unauthorized response.
+// This reduces the effectiveness of brute-force.
+//
+// Recommended by OWASP Top10:
+// https://owasp.org/Top10/2025/A07_2025-Authentication_Failures
+func slowdownUnauthorizedResponse(r *http.Request) {
+
+	d := 2*time.Second + time.Duration(rand.IntN(1000))*time.Millisecond
+	t := timerpool.Get(d)
+
+	select {
+	case <-t.C:
+	case <-r.Context().Done():
+	}
+	timerpool.Put(t)
+}

app/vmauth/main_synctest_test.go

@@ -0,0 +1,54 @@
+//go:build synctest
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+func TestBufferRequestBody_Timeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+
+		defaultMaxQueueDuration := *maxQueueDuration
+		defer func() {
+			*maxQueueDuration = defaultMaxQueueDuration
+		}()
+
+		*maxQueueDuration = 100 * time.Millisecond
+
+		ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
+		defer cancel()
+
+		_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		esc, ok := err.(*httpserver.ErrorWithStatusCode)
+		if !ok {
+			t.Fatalf("unexpected error type: %s", err)
+		}
+		if esc.StatusCode != http.StatusRequestTimeout {
+			t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
+		}
+	})
+}
+
+type timeoutBody struct {
+	delay time.Duration
+}
+
+func (r *timeoutBody) Read(_ []byte) (int, error) {
+	time.Sleep(r.delay)
+	return 0, context.DeadlineExceeded
+}
+
+func (r *timeoutBody) Close() error {
+	return nil
+}

app/vmauth/main_test.go

@@ -739,6 +739,12 @@ users:
 		"vm_access": map[string]any{},
 	}, false)
 
+	// token without vm_access claim, but with a custom claim usable for routing
+	roleToken := genToken(t, map[string]any{
+		"exp":  time.Now().Add(10 * time.Minute).Unix(),
+		"role": "admin",
+	}, true)
+
 	fullToken := genToken(t, map[string]any{
 		"exp": time.Now().Add(10 * time.Minute).Unix(),
 		"vm_access": map[string]any{
@@ -779,6 +785,45 @@ statusCode=401
 Unauthorized`
 	f(simpleCfgStr, request, responseExpected)
 
+	// token without vm_access claim should fall through to unauthorized_user
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	responseExpected = `
+statusCode=200
+path: /bar/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+unauthorized_user:
+  url_prefix: {BACKEND}/bar
+users:
+- jwt:
+    public_keys:
+    - %q
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
+	// token without vm_access claim is accepted when default_vm_access_claim configured
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+roleToken)
+	responseExpected = `
+statusCode=200
+path: /foo/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+    default_vm_access_claim:
+      metrics_account_id: 10
+      metrics_project_id: 10
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
 	// expired token
 	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
 	request.Header.Set(`Authorization`, `Bearer `+expiredToken)

app/vmctl/README.md

@@ -1,3 +1,3 @@
 See vmctl docs [here](https://docs.victoriametrics.com/victoriametrics/vmctl/).
 
-vmctl docs can be edited at [docs/vmctl.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmctl.md).
+vmctl docs can be edited at [docs/vmctl.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmctl/vmctl.md).

app/vmctl/influx/influx.go

@@ -259,7 +259,7 @@ func (cr *ChunkedResponse) Next() ([]int64, []float64, error) {
 
 	fieldValues, ok := r.values[cr.field]
 	if !ok {
-		return nil, nil, fmt.Errorf("response doesn't contain filed %q", cr.field)
+		return nil, nil, fmt.Errorf("response doesn't contain field %q", cr.field)
 	}
 	values := make([]float64, len(fieldValues))
 	for i, fv := range fieldValues {

app/vmctl/main.go

@@ -563,11 +563,11 @@ func main() {
 	}()
 
 	err = app.Run(os.Args)
+	pushmetrics.StopAndPush()
 	if err != nil {
 		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
-	pushmetrics.StopAndPush()
 }
 
 func initConfigVM(c *cli.Context) (vm.Config, error) {

app/vmctl/vm/vm.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -111,7 +112,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 		if strings.Contains(dst, "?") {
 			separator = "&"
 		}
-		dst += fmt.Sprintf("%sextra_label=%s", separator, extraLabel)
+		dst += fmt.Sprintf("%sextra_label=%s", separator, url.QueryEscape(extraLabel))
 	}
 	return dst, nil
 }

app/vmctl/vm/vm_test.go

@@ -33,11 +33,14 @@ func TestAddExtraLabelsToImportPath_Success(t *testing.T) {
 	f("/api/v1/import", nil, "/api/v1/import")
 
 	// ok one extra label
-	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance=host-1")
+	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance%3Dhost-1")
 
 	// ok two extra labels
-	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
 
 	// ok two extra with exist param
-	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
+
+	// ok special chars in label value
+	f("/api/v1/import", []string{"team=a&b"}, "/api/v1/import?extra_label=team%3Da%26b")
 }

app/vmctl/vm_native.go

@@ -405,7 +405,16 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
 			if len(tf.Key) == 0 {
 				continue
 			}
-			a = append(a, tf.String())
+			switch {
+			case tf.IsNegative && tf.IsRegexp:
+				a = append(a, fmt.Sprintf("%s!~%q", tf.Key, tf.Value))
+			case tf.IsNegative:
+				a = append(a, fmt.Sprintf("%s!=%q", tf.Key, tf.Value))
+			case tf.IsRegexp:
+				a = append(a, fmt.Sprintf("%s=~%q", tf.Key, tf.Value))
+			default:
+				a = append(a, fmt.Sprintf("%s=%q", tf.Key, tf.Value))
+			}
 		}
 		a = append(a, nameFilter)
 		filters = append(filters, strings.Join(a, ","))

app/vmestimator/cardinality_metrics.go

@@ -1,89 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	cardinalityMetricsWrites        = metrics.NewCounter(`vmestimator_write_cardinality_metrics_total`)
-	cardinalityMetricsWriteDuration = metrics.NewFloatCounter(`vmestimator_write_cardinality_metrics_duration_seconds_total`)
-	cardinalityMetricsWriteBytes    = metrics.NewCounter(`vmestimator_write_cardinality_metrics_size_bytes_total`)
-
-	cardinalityCacheMu         sync.Mutex
-	cardinalityMetricsCacheAt  time.Time
-	cardinalityMetricsCache    []byte
-	cardinalityMetricsCacheTTL = flag.Duration("cardinalityMetrics.cacheTTL", time.Second*30, "Duration for caching cardinality metrics response")
-	cardinalityMetricsExposeAt = flag.String(`cardinalityMetrics.exposeAt`, `/metrics`, "HTTP path for exposing cardinality metrics. "+
-		"If set to the default /metrics, cardinality metrics are merged with regular metrics and exposed together. "+
-		"If set to a different path, only cardinality metrics are exposed at that endpoint. "+
-		"If set to an empty value, cardinality metrics are not exposed via HTTP at all.")
-)
-
-func writeCardinalityMetrics(w io.Writer, es []*estimator, storageNodeURLs []string) {
-	startTime := time.Now()
-
-	cardinalityCacheMu.Lock()
-	if time.Since(cardinalityMetricsCacheAt) >= *cardinalityMetricsCacheTTL || *cardinalityMetricsCacheTTL == 0 {
-		plain := bytes.NewBuffer(cardinalityMetricsCache[:0])
-		for _, e := range es {
-			e.writeMetrics(plain)
-		}
-
-		if len(storageNodeURLs) > 0 {
-			ss := newSnapshots()
-			var wg sync.WaitGroup
-			for _, nodeURL := range storageNodeURLs {
-				wg.Add(1)
-				go func(url string) {
-					defer wg.Done()
-					if err := fetchAndMergeSnapshots(url, ss.add); err != nil {
-						logger.Errorf("fetch snapshots from %s: %s", url, err)
-					}
-				}(nodeURL)
-			}
-			wg.Wait()
-
-			if err := ss.writeMetrics(plain); err != nil {
-				logger.Errorf("write cardinality metrics: %s", err)
-			}
-		}
-
-		cardinalityMetricsCache = plain.Bytes()
-		cardinalityMetricsCacheAt = time.Now()
-	}
-	cm := make([]byte, len(cardinalityMetricsCache))
-	copy(cm, cardinalityMetricsCache)
-	cardinalityCacheMu.Unlock()
-
-	if _, err := w.Write(cm); err != nil {
-		logger.Warnf("writing cardinality metrics: %s", err)
-	}
-
-	cardinalityMetricsWrites.Inc()
-	cardinalityMetricsWriteDuration.Add(time.Since(startTime).Seconds())
-	cardinalityMetricsWriteBytes.Add(len(cm))
-}
-
-func fetchAndMergeSnapshots(storageNodeURL string, cb func(s *snapshot)) error {
-	url := fmt.Sprintf("%s/clusternative/snapshot", storageNodeURL)
-	resp, err := http.Get(url) //nolint:noctx
-	if err != nil {
-		return fmt.Errorf("http get %s: %w", url, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, url)
-	}
-
-	return decodeSnapshots(resp.Body, cb)
-}

app/vmestimator/config.go

@@ -1,60 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"sort"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"gopkg.in/yaml.v2"
-)
-
-type Config struct {
-	Streams []EstimatorConfig `yaml:"streams"`
-}
-
-type EstimatorConfig struct {
-	GroupBy      []string          `yaml:"group_by"`
-	GroupLimit   int               `yaml:"group_limit"`
-	Labels       map[string]string `yaml:"labels"`
-	Interval     time.Duration     `yaml:"interval"`
-	Buckets      int               `yaml:"buckets"`
-	HLLPrecision uint8             `yaml:"hll_precision"`
-	HLLSparse    *bool             `yaml:"hll_sparse"`
-}
-
-func loadConfig(path string) ([]*estimator, error) {
-	if path == "" && len(*storageNodes) > 0 {
-		return nil, nil
-	}
-	if path == "" {
-		return nil, fmt.Errorf("no -config flag specified")
-	}
-
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read config file %q: %w", path, err)
-	}
-	var cfg Config
-	if err := yaml.UnmarshalStrict(data, &cfg); err != nil {
-		return nil, fmt.Errorf("cannot parse config file %q: %w", path, err)
-	}
-	for _, stream := range cfg.Streams {
-		sort.Strings(stream.GroupBy)
-		if stream.HLLPrecision != 0 && (stream.HLLPrecision < 4 || stream.HLLPrecision > 18) {
-			return nil, fmt.Errorf("invalid precision %d: must be in range [4, 18]", stream.HLLPrecision)
-		}
-	}
-
-	es := make([]*estimator, 0, len(cfg.Streams))
-	for _, ec := range cfg.Streams {
-		e, err := newEstimator(ec)
-		if err != nil {
-			logger.Fatalf("cannot create estimator: %v", err)
-		}
-		es = append(es, e)
-	}
-
-	return es, nil
-}

app/vmestimator/estimator.go

@@ -1,560 +0,0 @@
-package main
-
-import (
-	"encoding/gob"
-	"fmt"
-	"io"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/axiomhq/hyperloglog"
-	"github.com/dgryski/go-metro"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmestimator/protoparser"
-)
-
-type estimator struct {
-	groupBy          []string
-	groupByKeysLabel string
-
-	groupLimit              int64
-	groupSize               atomic.Int64
-	groupRejectedMu         sync.Mutex
-	groupRejectedSketch     *hyperloglog.Sketch
-	groupRejectedSketchPrev *hyperloglog.Sketch
-
-	buckets []*estimatorBucket
-
-	metricsSet  *metrics.Set
-	insertTotal *metrics.Counter
-
-	stopCh chan struct{}
-}
-
-func newEstimator(cfg EstimatorConfig) (*estimator, error) {
-	if cfg.Interval == 0 {
-		cfg.Interval = time.Minute * 5
-	}
-	if cfg.GroupLimit <= 0 {
-		cfg.GroupLimit = 10000
-	}
-	if cfg.Buckets <= 0 {
-		cfg.Buckets = min(64, 2*cgroup.AvailableCPUs())
-	}
-	if cfg.HLLPrecision == 0 {
-		cfg.HLLPrecision = 14
-	}
-	if cfg.HLLSparse == nil {
-		cfg.HLLSparse = new(true)
-	}
-
-	metricPrefix := fmt.Sprintf("cardinality_estimate{interval=%q", cfg.Interval)
-	if len(cfg.Labels) > 0 {
-		keys := make([]string, 0, len(cfg.Labels))
-		for k := range cfg.Labels {
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-		for _, k := range keys {
-			metricPrefix += fmt.Sprintf(",%s=%q", k, cfg.Labels[k])
-		}
-	}
-
-	groupByKeysLabel := "__global__"
-	if len(cfg.GroupBy) > 0 {
-		groupByKeysLabel = strings.Join(cfg.GroupBy, `,`)
-	}
-
-	e := &estimator{
-		groupBy:                 cfg.GroupBy,
-		groupByKeysLabel:        groupByKeysLabel,
-		groupLimit:              int64(cfg.GroupLimit),
-		groupRejectedSketch:     mustNewGroupRejectSketch(),
-		groupRejectedSketchPrev: mustNewGroupRejectSketch(),
-		buckets:                 make([]*estimatorBucket, cfg.Buckets),
-		metricsSet:              metrics.NewSet(),
-		stopCh:                  make(chan struct{}),
-	}
-
-	e.insertTotal = e.metricsSet.NewCounter(
-		fmt.Sprintf(`vmestimator_estimator_insert_total{group_by_keys=%q}`, e.groupByKeysLabel),
-	)
-	e.metricsSet.NewGauge(fmt.Sprintf(`vmestimator_estimator_group_rejected_size{group_by_keys=%q}`, e.groupByKeysLabel), func() float64 {
-		e.groupRejectedMu.Lock()
-		defer e.groupRejectedMu.Unlock()
-		return float64(e.groupRejectedSketch.Estimate())
-	})
-
-	for i := 0; i < len(e.buckets); i++ {
-		eb := &estimatorBucket{
-			groupBy:             cfg.GroupBy,
-			extraLabels:         cfg.Labels,
-			interval:            cfg.Interval,
-			metricPrefix:        metricPrefix,
-			groupByKeysLabel:    groupByKeysLabel,
-			groupLimit:          int64(cfg.GroupLimit),
-			groupSize:           &e.groupSize,
-			groupRejectedMu:     &e.groupRejectedMu,
-			groupRejectedSketch: e.groupRejectedSketch,
-
-			precision: cfg.HLLPrecision,
-			sparse:    *cfg.HLLSparse,
-		}
-
-		if len(cfg.GroupBy) == 0 {
-			eb.sketch = eb.newSketch()
-		} else {
-			eb.groups = make(map[string]groupSketch)
-			eb.prevGroups = make(map[string]groupSketch)
-
-			e.metricsSet.NewGauge(fmt.Sprintf(`vmestimator_estimator_group_size{group_by_keys=%q,bucket="%d"}`, eb.groupByKeysLabel, i), func() float64 {
-				return float64(eb.groupSize.Load())
-			})
-			e.metricsSet.NewGauge(fmt.Sprintf(`vmestimator_estimator_group_limit{group_by_keys=%q,bucket="%d"}`, eb.groupByKeysLabel, i), func() float64 {
-				return float64(eb.groupLimit)
-			})
-		}
-
-		e.buckets[i] = eb
-	}
-
-	go e.runRotation(cfg.Interval)
-
-	metrics.RegisterSet(e.metricsSet)
-
-	return e, nil
-}
-
-func (e *estimator) stop() {
-	close(e.stopCh)
-	e.metricsSet.UnregisterAllMetrics()
-}
-
-var groupValuesPool = sync.Pool{}
-
-func getGroupValuesKeySlice() *[]byte {
-	v0 := groupValuesPool.Get()
-	if v0 == nil {
-		v := make([]byte, 128)
-		return &v
-	}
-
-	return v0.(*[]byte)
-}
-
-func putGroupValuesSlice(key *[]byte) {
-	if key == nil {
-		return
-	}
-
-	*key = (*key)[:0]
-	groupValuesPool.Put(key)
-}
-
-func (e *estimator) insertMany(tss []protoparser.TimeSerie) {
-	bucketsNum := uint64(len(e.buckets))
-
-	groupValuesKeyP := getGroupValuesKeySlice()
-	groupValuesKey := *groupValuesKeyP
-	defer func() {
-		*groupValuesKeyP = groupValuesKey
-		putGroupValuesSlice(groupValuesKeyP)
-	}()
-
-	groupValues := make([]string, len(e.groupBy))
-
-	var cnt int
-	for _, ts := range tss {
-		if len(e.groupBy) == 0 {
-			i := int(ts.Fingerprint % bucketsNum)
-			e.buckets[i].insert(ts, "", nil)
-			cnt++
-			continue
-		}
-
-		groupValuesKey = groupValuesKey[:0]
-		clear(groupValues)
-		var hasNames bool
-		for i, labelName := range e.groupBy {
-			if i > 0 {
-				groupValuesKey = append(groupValuesKey, ',')
-			}
-
-			for _, l := range ts.GroupLabels {
-				if l.Name == labelName {
-					hasNames = true
-
-					groupValuesKey = append(groupValuesKey, l.Value...)
-					groupValues[i] = l.Value
-					break
-				}
-			}
-		}
-
-		// time series does not contribute to this groupBy
-		if !hasNames {
-			continue
-		}
-
-		i := int(hash(groupValuesKey) % bucketsNum)
-		e.buckets[i].insert(ts, bytesutil.ToUnsafeString(groupValuesKey), groupValues)
-		cnt++
-	}
-
-	e.insertTotal.Add(cnt)
-}
-
-func (e *estimator) reset() {
-	e.groupSize.Store(0)
-	for _, b := range e.buckets {
-		b.reset()
-	}
-
-	e.groupRejectedMu.Lock()
-	e.groupRejectedSketch.Reset()
-	e.groupRejectedMu.Unlock()
-}
-
-func (e *estimator) writeMetrics(w io.Writer) {
-	eb0 := e.buckets[0]
-
-	if len(e.groupBy) == 0 {
-		formatBuf := make([]byte, 0, 1024)
-		resSK := eb0.newSketch()
-		for _, eb := range e.buckets {
-			eb.writeNoGroupMetric(resSK)
-		}
-
-		formatBuf = appendGlobalMetric(formatBuf, eb0.metricPrefix)
-		formatBuf = strconv.AppendUint(formatBuf, resSK.Estimate(), 10)
-		formatBuf = append(formatBuf, "\n"...)
-		if _, err := w.Write(formatBuf); err != nil {
-			logger.Errorf("writing metrics failed: %s; written cardinality metrics might be incomplete or invalid", err)
-		}
-		return
-	}
-
-	formatBuf := make([]byte, 0, 16384)
-	formatBuf = appendGroupByKeysAndValuesPrefix(formatBuf, eb0.metricPrefix, eb0.groupByKeysLabel)
-
-	prefixLen := len(formatBuf)
-	resSK := eb0.newSketch()
-	for _, eb := range e.buckets {
-		formatBuf = eb.writeGroupMetrics(w, resSK, formatBuf[:prefixLen])
-	}
-
-	groupSize := e.groupSize.Load()
-	if groupSize >= int64(float64(e.groupLimit)*0.8) {
-		e.groupRejectedMu.Lock()
-		res := mustNewGroupRejectSketch()
-		if err := res.Merge(e.groupRejectedSketch); err != nil {
-			logger.Fatalf("BUG: groupRejectedSketch merge failed: %s", err)
-		}
-		if err := res.Merge(e.groupRejectedSketchPrev); err != nil {
-			logger.Fatalf("BUG: groupRejectedSketchPrev merge failed: %s", err)
-		}
-		e.groupRejectedMu.Unlock()
-
-		groupSize += int64(res.Estimate())
-	}
-
-	formatBuf = formatBuf[:0]
-	formatBuf = appendGroupMetric(formatBuf, eb0.metricPrefix, eb0.groupByKeysLabel)
-	formatBuf = strconv.AppendInt(formatBuf, groupSize, 10)
-	formatBuf = append(formatBuf, "\n"...)
-	if _, err := w.Write(formatBuf); err != nil {
-		logger.Errorf("writing metrics failed: %s; written cardinality metrics might be incomplete or invalid", err)
-	}
-}
-
-func (e *estimator) runRotation(interval time.Duration) {
-	t := time.NewTicker(interval / 2)
-	defer t.Stop()
-	for {
-		select {
-		case <-t.C:
-			e.rotate()
-		case <-e.stopCh:
-			return
-		}
-	}
-}
-
-func (e *estimator) rotate() {
-	e.groupSize.Store(0)
-
-	var wg sync.WaitGroup
-	for i := range e.buckets {
-		wg.Go(e.buckets[i].rotate)
-	}
-	wg.Wait()
-
-	e.groupRejectedMu.Lock()
-	prevSK := e.groupRejectedSketchPrev
-	prevSK.Reset()
-	e.groupRejectedSketchPrev = e.groupRejectedSketch
-	e.groupRejectedSketch = prevSK
-	e.groupRejectedMu.Unlock()
-}
-
-func (e *estimator) writeSnapshot(enc *gob.Encoder) error {
-	if len(e.groupBy) == 0 {
-		s := newSnapshot()
-		if err := enc.Encode(convertNoGroupToSnapshot(e, s)); err != nil {
-			return fmt.Errorf("encode snapshot: %w", err)
-		}
-
-		return nil
-	}
-
-	eb0 := e.buckets[0]
-
-	formatBuf := make([]byte, 0, 16384)
-	formatBuf = appendGroupByKeysAndValuesPrefix(formatBuf, eb0.metricPrefix, eb0.groupByKeysLabel)
-
-	s := newSnapshot()
-	for i, eb := range e.buckets {
-		s.reset()
-		if i == 0 {
-			eb.groupRejectedMu.Lock()
-			if eb.groupRejectedSketch != nil {
-				s.GroupRejectedSketch = eb.groupRejectedSketch.Clone()
-			}
-			eb.groupRejectedMu.Unlock()
-		}
-		if err := enc.Encode(convertGroupBucketToSnapshot(eb, s, formatBuf)); err != nil {
-			return fmt.Errorf("encode snapshot: %w", err)
-		}
-	}
-
-	return nil
-}
-
-type estimatorBucket struct {
-	mu sync.Mutex
-
-	groupBy          []string
-	groupLimit       int64
-	extraLabels      map[string]string
-	interval         time.Duration
-	metricPrefix     string
-	groupByKeysLabel string
-	precision        uint8
-	sparse           bool
-
-	sketch     *hyperloglog.Sketch
-	prevSketch *hyperloglog.Sketch
-
-	groupSize  *atomic.Int64
-	groups     map[string]groupSketch
-	prevGroups map[string]groupSketch
-
-	groupRejectedMu     *sync.Mutex
-	groupRejectedSketch *hyperloglog.Sketch
-}
-
-func (eb *estimatorBucket) String() string {
-	return fmt.Sprintf(
-		"interval: %s; group_by: %v; extra_labels: %v", eb.interval, eb.groupBy, eb.extraLabels)
-}
-
-func (eb *estimatorBucket) reset() {
-	eb.mu.Lock()
-	defer eb.mu.Unlock()
-
-	if len(eb.groupBy) == 0 {
-		eb.prevSketch.Reset()
-		eb.sketch.Reset()
-		return
-	}
-
-	eb.groups = make(map[string]groupSketch)
-	eb.prevGroups = make(map[string]groupSketch)
-}
-
-func (eb *estimatorBucket) rotate() {
-	if len(eb.groupBy) == 0 {
-		eb.mu.Lock()
-		eb.prevSketch = eb.sketch
-		eb.sketch = eb.newSketch()
-		eb.mu.Unlock()
-		return
-	}
-
-	eb.mu.Lock()
-	eb.prevGroups = eb.groups
-	eb.groups = make(map[string]groupSketch, len(eb.groups))
-	eb.mu.Unlock()
-
-	eb.groupSize.Add(int64(len(eb.prevGroups)))
-}
-
-func (eb *estimatorBucket) insert(ts protoparser.TimeSerie, groupValuesKey string, groupValues []string) {
-	eb.mu.Lock()
-	defer eb.mu.Unlock()
-
-	if len(eb.groupBy) == 0 {
-		eb.sketch.InsertHash(ts.Fingerprint)
-		return
-	}
-
-	gsk, ok := eb.groups[groupValuesKey]
-	if !ok {
-		if _, ok := eb.prevGroups[groupValuesKey]; !ok {
-			groupSize := eb.groupSize.Load()
-			if groupSize+1 > eb.groupLimit {
-				eb.groupRejectedMu.Lock()
-				eb.groupRejectedSketch.InsertHash(hash([]byte(groupValuesKey)))
-				eb.groupRejectedMu.Unlock()
-				return
-			}
-
-			eb.groupSize.Add(1)
-		}
-
-		formatBuf := make([]byte, 0, 1024)
-		formatBuf = strconv.AppendQuote(formatBuf, groupValuesKey)
-		for i := range groupValues {
-			formatBuf = append(formatBuf, ',')
-			if eb.groupBy[i] == `__name__` {
-				formatBuf = append(formatBuf, `by__name__`...)
-			} else {
-				formatBuf = append(formatBuf, `by_`...)
-				formatBuf = append(formatBuf, eb.groupBy[i]...)
-			}
-			formatBuf = append(formatBuf, '=')
-			formatBuf = strconv.AppendQuote(formatBuf, groupValues[i])
-		}
-		formatBuf = append(formatBuf, `} `...)
-
-		gsk = groupSketch{
-			groupValueLabels: bytesutil.ToUnsafeString(formatBuf),
-
-			Sketch: eb.newSketch(),
-		}
-
-		eb.groups[strings.Clone(groupValuesKey)] = gsk
-	}
-	gsk.InsertHash(ts.Fingerprint)
-}
-
-func (eb *estimatorBucket) writeNoGroupMetric(res *hyperloglog.Sketch) {
-	eb.mu.Lock()
-	defer eb.mu.Unlock()
-
-	eb.mergeSketches(eb.sketch, eb.prevSketch, res)
-}
-
-func (eb *estimatorBucket) writeGroupMetrics(w io.Writer, res *hyperloglog.Sketch, formatBuf []byte) []byte {
-	eb.mu.Lock()
-	defer eb.mu.Unlock()
-
-	prefixLen := len(formatBuf)
-
-	for valuesKey, gsk := range eb.groups {
-		res.Reset()
-		formatBuf = append(formatBuf[:prefixLen], gsk.groupValueLabels...)
-
-		eb.mergeSketches(gsk.Sketch, eb.prevGroups[valuesKey].Sketch, res)
-		formatBuf = strconv.AppendUint(formatBuf, res.Estimate(), 10)
-		formatBuf = append(formatBuf, "\n"...)
-		if _, err := w.Write(formatBuf); err != nil {
-			logger.Errorf("writing metrics failed: %s; written cardinality metrics might be incomplete or invalid", err)
-		}
-	}
-
-	for valuesKey := range eb.prevGroups {
-		if _, ok := eb.groups[valuesKey]; ok {
-			continue
-		}
-
-		res.Reset()
-		formatBuf = formatBuf[:prefixLen]
-
-		gsk := eb.prevGroups[valuesKey]
-		formatBuf = append(formatBuf, gsk.groupValueLabels...)
-
-		eb.mergeSketches(nil, eb.prevGroups[valuesKey].Sketch, res)
-		formatBuf = strconv.AppendUint(formatBuf, res.Estimate(), 10)
-		formatBuf = append(formatBuf, "\n"...)
-		if _, err := w.Write(formatBuf); err != nil {
-			logger.Errorf("writing metrics failed: %s; written cardinality metrics might be incomplete or invalid", err)
-		}
-	}
-
-	return formatBuf[:prefixLen]
-}
-
-func (eb *estimatorBucket) mergeSketches(cur, prev, res *hyperloglog.Sketch) {
-	if err := res.Merge(cur); err != nil {
-		panic(err)
-	}
-	if prev != nil {
-		if err := res.Merge(prev); err != nil {
-			panic(err)
-		}
-	}
-}
-
-func (eb *estimatorBucket) newSketch() *hyperloglog.Sketch {
-	return mustNewSketch(eb.precision, eb.sparse)
-}
-
-type groupSketch struct {
-	groupValueLabels string
-
-	*hyperloglog.Sketch
-}
-
-func mustNewGroupRejectSketch() *hyperloglog.Sketch {
-	return mustNewSketch(10, true)
-}
-
-func mustNewSketch(precision uint8, sparse bool) *hyperloglog.Sketch {
-	sk, err := hyperloglog.NewSketch(precision, sparse)
-	if err != nil {
-		panic(fmt.Sprintf("cannot create HLL sketch with precision=%d and sparse=%v: %s", precision, sparse, err))
-	}
-
-	return sk
-}
-
-func hash(v []byte) uint64 {
-	return metro.Hash64(v, 1337)
-}
-
-// appendGlobalMetric produces:
-// 'cardinality_estimate{interval="5m",group_by_keys="__global__"} '
-func appendGlobalMetric(buf []byte, metricPrefix string) []byte {
-	buf = append(buf, metricPrefix...)
-	buf = append(buf, `,group_by_keys="__global__"} `...)
-	return buf
-}
-
-// appendGroupMetric produces:
-// 'cardinality_estimate{interval="5m",group_by_keys="__group__",group_by_values="fooKey,barKey"} '
-func appendGroupMetric(buf []byte, metricPrefix, groupByKeysLabel string) []byte {
-	buf = append(buf, metricPrefix...)
-	buf = append(buf, `,group_by_keys="__group__",group_by_values="`...)
-	buf = append(buf, groupByKeysLabel...)
-	buf = append(buf, `"} `...)
-	return buf
-}
-
-// appendGroupByKeysAndValuesPrefix produces:
-// 'cardinality_estimate{interval="5m",group_by_keys="fooKey,barKey",group_by_values='
-func appendGroupByKeysAndValuesPrefix(buf []byte, metricPrefix, groupByKeysLabel string) []byte {
-	buf = append(buf, metricPrefix...)
-	buf = append(buf, `,group_by_keys="`...)
-	buf = append(buf, groupByKeysLabel...)
-	buf = append(buf, `",group_by_values=`...)
-	return buf
-}

app/vmestimator/estimator_timing_test.go

@@ -1,274 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmestimator/protoparser"
-)
-
-func BenchmarkEstimator_WriteMetrics(b *testing.B) {
-	b.Run("NoGroup/NoPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{Interval: time.Hour})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 5_000, 0)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-
-	b.Run("NoGroup/WithPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{Interval: time.Hour})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 5_000, 0)
-		for _, eb := range e.buckets {
-			eb.rotate()
-		}
-		insertSeriesIntoEstimator(e, 5_000, 0)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-
-	b.Run("Group100/NoPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 5_000, 100)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-
-	b.Run("Group100/WithPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 5_000, 100)
-		for _, eb := range e.buckets {
-			eb.rotate()
-		}
-		insertSeriesIntoEstimator(e, 5_000, 100)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-
-	b.Run("Group10k/NoPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 50_000, 10_000)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-
-	b.Run("Group10k/WithPrev", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-		insertSeriesIntoEstimator(e, 50_000, 10_000)
-		for _, eb := range e.buckets {
-			eb.rotate()
-		}
-		insertSeriesIntoEstimator(e, 50_000, 10_000)
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			e.writeMetrics(io.Discard)
-		}
-	})
-}
-
-func BenchmarkEstimator_InsertManyParallel(b *testing.B) {
-	b.Run("NoGroup", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{Interval: time.Hour})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		b.RunParallel(func(pb *testing.PB) {
-			var i uint64
-			for pb.Next() {
-				e.insertMany([]protoparser.TimeSerie{{Fingerprint: i}})
-				i++
-			}
-		})
-	})
-
-	b.Run("Group100", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		b.RunParallel(func(pb *testing.PB) {
-			var i uint64
-			for pb.Next() {
-				e.insertMany([]protoparser.TimeSerie{{
-					GroupLabels: []protoparser.Label{{Name: "groupLabel", Value: fmt.Sprintf("%d", i%100)}},
-					Fingerprint: i,
-				}})
-				i++
-			}
-		})
-	})
-
-	b.Run("Group10k", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		b.RunParallel(func(pb *testing.PB) {
-			var i uint64
-			for pb.Next() {
-				e.insertMany([]protoparser.TimeSerie{{
-					GroupLabels: []protoparser.Label{{Name: "groupLabel", Value: fmt.Sprintf("%d", i%10_000)}},
-					Fingerprint: i,
-				}})
-				i++
-			}
-		})
-	})
-
-	b.Run("Group100k", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{
-			GroupBy:  []string{"groupLabel"},
-			Interval: time.Hour,
-		})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		b.RunParallel(func(pb *testing.PB) {
-			var i uint64
-			for pb.Next() {
-				e.insertMany([]protoparser.TimeSerie{{
-					GroupLabels: []protoparser.Label{{Name: "groupLabel", Value: fmt.Sprintf("%d", i%100_000)}},
-					Fingerprint: i,
-				}})
-				i++
-			}
-		})
-	})
-}
-
-// BenchmarkEstimator_InsertRotateCycle benchmarks the insert→rotate→insert cycle
-// for the global (no-group) estimator in two HLL regimes:
-//   - Sparse: 1 000 series per interval (sketch stays in sparse mode)
-//   - Normal: 30 000 series per interval (sketch converts to dense mode)
-func BenchmarkEstimator_InsertRotateCycle(b *testing.B) {
-	b.Run("SparseHLL", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{Interval: time.Hour})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			insertSeriesIntoEstimator(e, 1_000, 0)
-			e.rotate()
-		}
-	})
-
-	b.Run("NormalHLL", func(b *testing.B) {
-		e, err := newEstimator(EstimatorConfig{Interval: time.Hour})
-		if err != nil {
-			b.Fatalf("newEstimator: %v", err)
-		}
-		defer e.stop()
-
-		b.ResetTimer()
-		b.ReportAllocs()
-		for i := 0; i < b.N; i++ {
-			insertSeriesIntoEstimator(e, 30_000, 0)
-			e.rotate()
-		}
-	})
-}
-
-// insertSeriesIntoEstimator inserts numSeries time series into e.
-// When groupsNum > 0 each series gets a "groupLabel" cycling through groupsNum values.
-func insertSeriesIntoEstimator(e *estimator, numSeries, groupsNum int) {
-	for i := 0; i < numSeries; i++ {
-		var labels []protoparser.Label
-		if groupsNum > 0 {
-			labels = append(labels, protoparser.Label{
-				Name:  "groupLabel",
-				Value: fmt.Sprintf("%d", i%groupsNum),
-			})
-		}
-		e.insertMany([]protoparser.TimeSerie{
-			{
-				GroupLabels: labels,
-				Fingerprint: hash([]byte(fmt.Sprintf("foobarbaz%d", i))),
-			},
-		})
-	}
-}

app/vmestimator/etimator_test.go

@@ -1,595 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"sort"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmestimator/protoparser"
-)
-
-func TestGlobalEstimate(t *testing.T) {
-	genCard := func(cardinality int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			var tss []protoparser.TimeSerie
-			fpBuf := make([]byte, 8, 8+len(seed))
-			for i := 0; i < cardinality; i++ {
-				binary.LittleEndian.PutUint64(fpBuf[:8], uint64(i))
-				fpBuf = append(fpBuf, seed...)
-				tss = append(tss, protoparser.TimeSerie{
-					Fingerprint: hash(fpBuf[:]),
-				})
-
-				if i%10 == 0 {
-					e.insertMany(tss)
-					tss = tss[:0]
-				}
-			}
-			if len(tss) > 0 {
-				e.insertMany(tss)
-			}
-		}
-	}
-
-	f := func(gen func(e *estimator), expMetric string) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval: time.Minute * 10,
-			Buckets:  5,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		if len(e.buckets) != cfg.Buckets {
-			t.Fatalf("expected buckets length to be %d but got %d", cfg.Buckets, len(e.buckets))
-		}
-		for i, eb := range e.buckets {
-			if len(eb.groupBy) > 0 {
-				t.Fatalf("expected bucket %d groupBy length to be 0 but got %d", i, len(eb.groupBy))
-			}
-			if eb.groups != nil {
-				t.Fatalf("expected bucket %d groups length to be 0 but got %d", i, len(eb.groups))
-			}
-			if eb.groupSize.Load() != 0 {
-				t.Fatalf("expected bucket %d groupSize to be 0 but got %d", i, eb.groupSize.Load())
-			}
-		}
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		assertMetricsSame(t, "", expMetric, buf.String())
-	}
-
-	// no previous
-	f(genCard(0, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genCard(1, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 1`)
-	f(genCard(10, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 10`)
-	f(genCard(100, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 100`)
-	f(genCard(1000, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 1000`)
-	f(genCard(5000, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 4998`)
-	f(genCard(10000, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 9920`)
-	f(genCard(100000, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 99658`)
-	f(genCard(500000, ""), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 496552`)
-
-	// rotate once
-	genRotateOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality, "")(e)
-			e.rotate()
-		}
-	}
-	f(genRotateOnce(0), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateOnce(1), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 1`)
-	f(genRotateOnce(10), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 10`)
-	f(genRotateOnce(100), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 100`)
-	f(genRotateOnce(1000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 1000`)
-	f(genRotateOnce(5000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 4998`)
-	f(genRotateOnce(10000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 9920`)
-	f(genRotateOnce(100000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 99658`)
-	f(genRotateOnce(500000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 496552`)
-
-	// insert, rotate insert the same
-	genInsertRotateInsertSameOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality/2, "")(e)
-			e.rotate()
-			genCard(cardinality/2, "")(e)
-		}
-	}
-	f(genInsertRotateInsertSameOnce(0), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genInsertRotateInsertSameOnce(1), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genInsertRotateInsertSameOnce(10), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 5`)
-	f(genInsertRotateInsertSameOnce(100), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 50`)
-	f(genInsertRotateInsertSameOnce(1000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 500`)
-	f(genInsertRotateInsertSameOnce(5000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 2499`)
-	f(genInsertRotateInsertSameOnce(10000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 4998`)
-	f(genInsertRotateInsertSameOnce(100000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 49529`)
-	f(genInsertRotateInsertSameOnce(200000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 99658`)
-
-	// insert, rotate insert
-	genInsertRotateInsertOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality/2, "one")(e)
-			e.rotate()
-			genCard(cardinality/2, "two")(e)
-		}
-	}
-	f(genInsertRotateInsertOnce(0), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genInsertRotateInsertOnce(1), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genInsertRotateInsertOnce(10), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 10`)
-	f(genInsertRotateInsertOnce(100), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 100`)
-	f(genInsertRotateInsertOnce(1000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 1000`)
-	f(genInsertRotateInsertOnce(5000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 5000`)
-	f(genInsertRotateInsertOnce(10000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 10058`)
-	f(genInsertRotateInsertOnce(100000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 99543`)
-	f(genInsertRotateInsertOnce(200000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 198814`)
-
-	// insert, rotate insert
-	genRotateTwoTimes := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality, "")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f(genRotateTwoTimes(0), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(1), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(10), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(100), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(1000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(5000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(10000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(100000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-	f(genRotateTwoTimes(500000), `cardinality_estimate{interval="10m0s",group_by_keys="__global__"} 0`)
-}
-
-func TestGroupEstimate(t *testing.T) {
-	genCard := func(fooCard, barCard, bazCard int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			var tss []protoparser.TimeSerie
-			for fooI := 0; fooI < max(1, fooCard); fooI++ {
-				for barI := 0; barI < max(1, barCard); barI++ {
-					for bazI := 0; bazI < max(1, bazCard); bazI++ {
-						ts := protoparser.TimeSerie{}
-						ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "__name__", Value: "the_metric_name"})
-						if fooCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "foo", Value: fmt.Sprintf("%s%d", seed, fooI)})
-						}
-						if barCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "bar", Value: fmt.Sprintf("%s%d", seed, barI)})
-						}
-						if bazCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "baz", Value: fmt.Sprintf("%s%d", seed, bazI)})
-						}
-						var fpBuf []byte
-						for _, l := range ts.GroupLabels {
-							fpBuf = append(fpBuf, l.Name...)
-							fpBuf = append(fpBuf, '=')
-							fpBuf = append(fpBuf, l.Value...)
-							fpBuf = append(fpBuf, ',')
-						}
-						fpBuf = append(fpBuf, seed...)
-						ts.Fingerprint = hash(fpBuf)
-						tss = append(tss, ts)
-					}
-				}
-			}
-			e.insertMany(tss)
-		}
-	}
-
-	f := func(groupBy []string, gen func(e *estimator), expMetrics string) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval: time.Minute * 10,
-			GroupBy:  groupBy,
-			Buckets:  5,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		if len(e.buckets) != cfg.Buckets {
-			t.Fatalf("expected buckets length to be %d but got %d", cfg.Buckets, len(e.buckets))
-		}
-		for i, eb := range e.buckets {
-			if eb.sketch != nil {
-				t.Fatalf("expected bucket %d sketch to be nil", i)
-			}
-			if eb.prevSketch != nil {
-				t.Fatalf("expected bucket %d prevSketch to be nil", i)
-			}
-		}
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		assertMetricsSame(t, "", expMetrics, buf.String())
-	}
-
-	// group by metric name
-	f([]string{"__name__"}, genCard(10, 10, 10, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="__name__"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="__name__",group_by_values="the_metric_name",by__name__="the_metric_name"} 1000`,
-	)
-
-	// time series does not contribute to a group
-	f([]string{"foo"}, genCard(0, 10, 10, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 0`,
-	)
-	f([]string{"foo", "bar"}, genCard(0, 0, 10, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 0`,
-	)
-
-	// group by one label
-	f([]string{"foo"}, genCard(1, 1, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1`,
-	)
-	f([]string{"foo"}, genCard(1, 2, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 2`,
-	)
-	f([]string{"foo"}, genCard(1, 10, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 10`,
-	)
-	f([]string{"foo"}, genCard(1, 100, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 100`,
-	)
-	f([]string{"foo"}, genCard(1, 1000, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1000`,
-	)
-	f([]string{"foo"}, genCard(1, 10000, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 9957`,
-	)
-	f([]string{"foo"}, genCard(1, 50000, 0, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 50387`,
-	)
-	f([]string{"foo"}, genCard(1, 1, 1, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1`,
-	)
-	f([]string{"foo"}, genCard(1, 2, 2, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 4`,
-	)
-	f([]string{"foo"}, genCard(1, 10, 10, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 100`,
-	)
-	f([]string{"foo"}, genCard(1, 100, 100, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 9954`,
-	)
-	f([]string{"foo"}, genCard(1, 1000, 1000, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1013124`,
-	)
-
-	// group by one label, rotate
-	genCardRotate := func(fooCard, barCard, bazCard int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(fooCard, barCard, bazCard, seed)(e)
-			e.rotate()
-		}
-	}
-	f([]string{"foo"}, genCardRotate(1, 10, 10, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 100`,
-	)
-	f([]string{"foo"}, genCardRotate(1, 1000, 1000, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1013124`,
-	)
-
-	// group by one label, rotate, insert same
-	genCardRotateInsertSame := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "")(e)
-			e.rotate()
-			genCard(1, barCard, bazCard, "")(e)
-		}
-	}
-	f([]string{"foo"}, genCardRotateInsertSame(10, 10), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 100`,
-	)
-	f([]string{"foo"}, genCardRotateInsertSame(1000, 1000), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="0",by_foo="0"} 1013124`,
-	)
-
-	// group by one label, rotate, insert diff
-	genCardRotateInsertDiff := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "one")(e)
-			e.rotate()
-			genCard(1, barCard, bazCard, "two")(e)
-		}
-	}
-	f([]string{"foo"}, genCardRotateInsertDiff(10, 10), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 2
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="one0",by_foo="one0"} 100
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="two0",by_foo="two0"} 100`,
-	)
-	f([]string{"foo"}, genCardRotateInsertDiff(1000, 1000), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 2
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="one0",by_foo="one0"} 995153
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="two0",by_foo="two0"} 992158`,
-	)
-
-	// group by one label, rotate, insert diff
-	genCardRotateTwice := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "one")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f([]string{"foo"}, genCardRotateTwice(10, 10), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 0`,
-	)
-	f([]string{"foo"}, genCardRotateTwice(1000, 1000), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 0`,
-	)
-
-	// group by two labels
-	f([]string{"foo", "bar"}, genCard(1, 1, 1000, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,0",by_foo="0",by_bar="0"} 1000`,
-	)
-	f([]string{"foo", "bar"}, genCard(2, 1, 1000, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 2
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,0",by_foo="0",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,0",by_foo="1",by_bar="0"} 1000`,
-	)
-	f([]string{"foo", "bar"}, genCard(2, 2, 1000, ""), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 4
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,0",by_foo="0",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,1",by_foo="0",by_bar="1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,0",by_foo="1",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,1",by_foo="1",by_bar="1"} 1000`,
-	)
-
-	// group by two labels, rotate
-	genCardTwoLabelsRotate := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "")(e)
-			e.rotate()
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotate(), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 4
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,0",by_foo="0",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,1",by_foo="0",by_bar="1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,0",by_foo="1",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,1",by_foo="1",by_bar="1"} 1000`,
-	)
-
-	// group by two labels, rotate, insert same
-	genCardTwoLabelsRotateInsertSame := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "")(e)
-			e.rotate()
-			genCard(2, 2, 1000, "")(e)
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotateInsertSame(), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 4
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,0",by_foo="0",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="0,1",by_foo="0",by_bar="1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,0",by_foo="1",by_bar="0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="1,1",by_foo="1",by_bar="1"} 1000`,
-	)
-
-	// group by two labels, rotate, insert diff
-	genCardTwoLabelsRotateInsertDiff := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "one")(e)
-			e.rotate()
-			genCard(2, 2, 1000, "two")(e)
-		}
-	}
-	f(
-		[]string{"foo", "bar"}, genCardTwoLabelsRotateInsertDiff(), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 8
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="one0,one0",by_foo="one0",by_bar="one0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="one0,one1",by_foo="one0",by_bar="one1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="one1,one0",by_foo="one1",by_bar="one0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="one1,one1",by_foo="one1",by_bar="one1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="two0,two0",by_foo="two0",by_bar="two0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="two0,two1",by_foo="two0",by_bar="two1"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="two1,two0",by_foo="two1",by_bar="two0"} 1000
-cardinality_estimate{interval="10m0s",group_by_keys="foo,bar",group_by_values="two1,two1",by_foo="two1",by_bar="two1"} 1000`,
-	)
-
-	// group by two labels, rotate, insert diff
-	genCardTwoLabelsRotateTwice := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "one")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotateTwice(), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo,bar"} 0`,
-	)
-
-	// quote values: label values with special characters must be properly escaped
-	genSpecialCard := func(fooVal string) func(e *estimator) {
-		return func(e *estimator) {
-			e.insertMany([]protoparser.TimeSerie{
-				{
-					GroupLabels: []protoparser.Label{{Name: "foo", Value: fooVal}},
-					Fingerprint: hash([]byte("foo=" + fooVal + ",")),
-				},
-			})
-		}
-	}
-
-	// double quote in value
-	f([]string{"foo"}, genSpecialCard(`a"b`), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a\"b",by_foo="a\"b"} 1`,
-	)
-
-	f([]string{"foo"}, genSpecialCard(`a\b`), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a\\b",by_foo="a\\b"} 1`,
-	)
-
-	f([]string{"foo"}, genSpecialCard("a\nb"), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a\nb",by_foo="a\nb"} 1`,
-	)
-
-	f([]string{"foo"}, genSpecialCard("a\tb"), `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a\tb",by_foo="a\tb"} 1`,
-	)
-}
-
-func TestGroupEstimateGroupLimit(t *testing.T) {
-	makeTS := func(fooVal string) protoparser.TimeSerie {
-		return protoparser.TimeSerie{
-			GroupLabels: []protoparser.Label{{Name: "foo", Value: fooVal}},
-			Fingerprint: hash([]byte("foo=" + fooVal + ",")),
-		}
-	}
-
-	f := func(groupLimit int, gen func(e *estimator), expRejected int, expMetrics string) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval:   time.Minute * 10,
-			GroupBy:    []string{"foo"},
-			GroupLimit: groupLimit,
-			Buckets:    3,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		assertMetricsSame(t, "", expMetrics, buf.String())
-
-		var actRejected int
-		if e.buckets[0].groupRejectedSketch != nil {
-			actRejected = int(e.buckets[0].groupRejectedSketch.Estimate())
-		}
-		if expRejected != actRejected {
-			t.Fatalf("rejected expected: %d; got: %d", expRejected, actRejected)
-		}
-	}
-
-	// all groups accepted
-	f(3, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 0, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 3
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a",by_foo="a"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="b",by_foo="b"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="c",by_foo="c"} 1`,
-	)
-
-	// 2 groups only accepted
-	f(2, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 1, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 3
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a",by_foo="a"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="b",by_foo="b"} 1`,
-	)
-
-	// one group only accepted
-	f(1, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 2, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 3
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a",by_foo="a"} 1`,
-	)
-
-	// after rotate: groups in prevGroups bypass the limit; new groups are still checked
-	f(2, func(e *estimator) {
-		// fills limit
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b")})
-		e.rotate()
-		// "a" bypasses, "c" rejected
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("c")})
-	}, 1, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 3
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a",by_foo="a"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="b",by_foo="b"} 1`,
-	)
-
-	// after rotate: new group accepted when remaining capacity allows
-	f(3, func(e *estimator) {
-		// 2 groups, limit=3
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b")})
-		e.rotate()
-		// "a" bypasses, "c" accepted (2+1=3 <= 3)
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("c")})
-	}, 0, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 3
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a",by_foo="a"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="b",by_foo="b"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="c",by_foo="c"} 1`,
-	)
-
-	// reject 100
-	f(3, func(e *estimator) {
-		var tss []protoparser.TimeSerie
-		for i := 0; i < 103; i++ {
-			tss = append(tss, makeTS(fmt.Sprintf("a%d", i)))
-		}
-		e.insertMany(tss)
-	}, 100, `
-cardinality_estimate{interval="10m0s",group_by_keys="__group__",group_by_values="foo"} 103
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a0",by_foo="a0"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a1",by_foo="a1"} 1
-cardinality_estimate{interval="10m0s",group_by_keys="foo",group_by_values="a2",by_foo="a2"} 1`,
-	)
-}
-
-func assertMetricsSame(t *testing.T, msg, exp, act string) {
-	t.Helper()
-
-	expLines := strings.Split(strings.TrimSpace(exp), "\n")
-	sort.Strings(expLines)
-	expSorted := strings.TrimSpace(strings.Join(expLines, "\n"))
-
-	actLines := strings.Split(strings.TrimSpace(act), "\n")
-	sort.Strings(actLines)
-	actSorted := strings.TrimSpace(strings.Join(actLines, "\n"))
-
-	if expSorted != actSorted {
-		t.Fatalf("%s\nexpected:\n%s\n\ngot:\n%s", msg, expSorted, actSorted)
-	}
-}

app/vmestimator/main.go

@@ -1,135 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/gob"
-	"flag"
-	"io"
-	"net/http"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmestimator/protoparser"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming HTTP requests")
-	configPath      = flag.String("config", "", "Path to YAML configuration file")
-	storageNodes    = flagutil.NewArrayString("storageNode", "HTTP URLs of remote vmestimator nodes to query for cardinality snapshots, e.g. http://vmestimator-2:8490")
-
-	prometheusWriteRequests = metrics.NewCounter(`vmestimator_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
-)
-
-func main() {
-	flag.CommandLine.SetOutput(os.Stdout)
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	es, err := loadConfig(*configPath)
-	if err != nil {
-		logger.Fatalf("cannot load config: %v", err)
-	}
-
-	if *cardinalityMetricsExposeAt == `/metrics` {
-		metrics.RegisterMetricsWriter(func(w io.Writer) {
-			writeCardinalityMetrics(w, es, *storageNodes)
-		})
-	}
-
-	groupLabelsMap := make(map[string]struct{})
-	for _, e := range es {
-		for _, l := range e.groupBy {
-			groupLabelsMap[l] = struct{}{}
-		}
-	}
-
-	groupLabels := make([]string, 0, len(groupLabelsMap))
-	for k := range groupLabelsMap {
-		groupLabels = append(groupLabels, k)
-	}
-
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8490"}
-	}
-
-	logger.Infof("starting vmestimator at %q", listenAddrs)
-	startTime := time.Now()
-
-	go httpserver.Serve(listenAddrs, func(w http.ResponseWriter, r *http.Request) bool {
-		cmPath := *cardinalityMetricsExposeAt
-		if cmPath != "/metrics" && cmPath != "" && r.URL.Path == cmPath {
-			w.WriteHeader(http.StatusOK)
-			writeCardinalityMetrics(w, es, *storageNodes)
-			return true
-		}
-
-		path, _ := strings.CutPrefix(r.URL.Path, `/cardinality`)
-		switch path {
-		case "/api/v1/write":
-			prometheusWriteRequests.Inc()
-			err := protoparser.Parse(r.Body, groupLabels, func(tss []protoparser.TimeSerie) {
-				for _, e := range es {
-					e.insertMany(tss)
-				}
-			})
-			if err != nil {
-				httpserver.Errorf(w, r, "error parsing remote write request: %s", err)
-				return true
-			}
-			w.WriteHeader(http.StatusNoContent)
-			return true
-		case "/clusternative/query", "/clusternative/snapshot":
-			w.Header().Set("Content-Type", "application/octet-stream")
-			w.Header().Set("Transfer-Encoding", "chunked")
-			w.WriteHeader(http.StatusOK)
-
-			bw := bufio.NewWriterSize(w, 64*1024)
-			enc := gob.NewEncoder(bw)
-			for _, e := range es {
-				if err := e.writeSnapshot(enc); err != nil {
-					logger.Errorf("write snapshot binary: %s", err)
-				}
-			}
-			if err := bw.Flush(); err != nil {
-				logger.Errorf("flush snapshot binary: %s", err)
-			}
-
-			return true
-		case "/reset":
-			for _, e := range es {
-				e.reset()
-			}
-			w.WriteHeader(http.StatusOK)
-			return true
-		}
-		return false
-	}, httpserver.ServeOptions{})
-
-	logger.Infof("started vmestimator in %.3f seconds", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Errorf("cannot stop http server: %s", err)
-	}
-	for _, e := range es {
-		e.stop()
-	}
-	logger.Infof("shutting down vmestimator")
-}

app/vmestimator/protoparser/streamparser.go

@@ -1,78 +0,0 @@
-package protoparser
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/snappy"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var maxInsertRequestSize = flagutil.NewBytes("maxInsertRequestSize", 32*1024*1024, "The maximum size in bytes of a single Prometheus remote_write API request")
-
-// Parse parses Prometheus remote_write message from reader and calls callback for the parsed timeseries.
-//
-// callback shouldn't hold tss after returning.
-func Parse(r io.Reader, groupLabels []string, callback func(tss []TimeSerie)) error {
-	startTime := fasttime.UnixTimestamp()
-
-	readCalls.Inc()
-	err := protoparserutil.ReadUncompressedData(r, "", maxInsertRequestSize, func(data []byte) error {
-		return parseRequestBody(data, groupLabels, callback)
-	})
-	if err != nil {
-		readErrors.Inc()
-		return fmt.Errorf("cannot read prometheus remote_write data from client in %d seconds: %w", fasttime.UnixTimestamp()-startTime, err)
-	}
-	return nil
-}
-
-func parseRequestBody(data []byte, groupLabels []string, callback func(tss []TimeSerie)) error {
-	// Synchronously process the request in order to properly return errors to Parse caller,
-	// so it could properly return HTTP 503 status code in response.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/896
-	bb := bodyBufferPool.Get()
-	defer bodyBufferPool.Put(bb)
-
-	if encoding.IsZstd(data) {
-		var err error
-		bb.B, err = encoding.DecompressZSTDLimited(bb.B[:0], data, maxInsertRequestSize.IntN())
-		if err != nil {
-			return fmt.Errorf("cannot decompress zstd-encoded request with length %d: %w", len(data), err)
-		}
-	} else {
-		var err error
-		bb.B, err = snappy.Decode(bb.B, data, maxInsertRequestSize.IntN())
-		if err != nil {
-			return fmt.Errorf("cannot decompress snappy-encoded request with length %d: %w", len(data), err)
-		}
-	}
-	if int64(len(bb.B)) > maxInsertRequestSize.N {
-		return fmt.Errorf("too big unpacked request; mustn't exceed `-maxInsertRequestSize=%d` bytes; got %d bytes", maxInsertRequestSize.N, len(bb.B))
-	}
-	wru := getWriteRequestUnmarshaler()
-	defer putWriteRequestUnmarshaler(wru)
-	if err := wru.UnmarshalProtobuf(bb.B, groupLabels, func(tss []TimeSerie) {
-		rowsRead.Add(len(tss))
-		callback(tss)
-	}); err != nil {
-		unmarshalErrors.Inc()
-		return fmt.Errorf("cannot unmarshal prompb.WriteRequest with size %d bytes: %w", len(bb.B), err)
-	}
-
-	return nil
-}
-
-var bodyBufferPool bytesutil.ByteBufferPool
-
-var (
-	readCalls       = metrics.NewCounter(`vm_protoparser_read_calls_total{type="promremotewrite"}`)
-	readErrors      = metrics.NewCounter(`vm_protoparser_read_errors_total{type="promremotewrite"}`)
-	rowsRead        = metrics.NewCounter(`vm_protoparser_rows_read_total{type="promremotewrite"}`)
-	unmarshalErrors = metrics.NewCounter(`vm_protoparser_unmarshal_errors_total{type="promremotewrite"}`)
-)

app/vmestimator/protoparser/streamparser_timing_test.go

@@ -1,67 +0,0 @@
-package protoparser
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/golang/snappy"
-)
-
-func BenchmarkParse(b *testing.B) {
-	data := buildSnappyEncodedWriteRequest(5000, 20, 20, 3)
-	groupLabels := []string{
-		"foo",
-		"bar",
-		"baz",
-		"__name__",
-		"job",
-		"groupLabel",
-	}
-
-	var cnt int
-
-	b.ResetTimer()
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	for b.Loop() {
-		err := Parse(bytes.NewReader(data), groupLabels, func(tss []TimeSerie) {
-			cnt += len(tss)
-		})
-		if err != nil {
-			b.Fatalf("stream.Parse: %v", err)
-		}
-	}
-}
-
-// buildSnappyEncodedWriteRequest builds a snappy-encoded protobuf WriteRequest
-// with numSeries time series, each having numLabels labels of labelSize bytes each.
-func buildSnappyEncodedWriteRequest(numSeries, numLabels, labelSize, groupsNum int) []byte {
-	labelValue := strings.Repeat("x", labelSize)
-
-	tss := make([]prompb.TimeSeries, numSeries)
-	for i := range tss {
-		labels := make([]prompb.Label, numLabels)
-		for j := range labels {
-			labels[j] = prompb.Label{
-				Name:  fmt.Sprintf("label%02d", j),
-				Value: fmt.Sprintf("val%05d_%s", i, labelValue),
-			}
-		}
-		labels = append(labels, prompb.Label{
-			Name:  "groupLabel",
-			Value: fmt.Sprintf("%d", i%groupsNum),
-		})
-
-		tss[i] = prompb.TimeSeries{
-			Labels:  labels,
-			Samples: []prompb.Sample{{Value: 1, Timestamp: 1000}},
-		}
-	}
-
-	wr := &prompb.WriteRequest{Timeseries: tss}
-	pbData := wr.MarshalProtobuf(nil)
-	return snappy.Encode(nil, pbData)
-}

app/vmestimator/protoparser/write_request.go

@@ -1,170 +0,0 @@
-package protoparser
-
-import (
-	"fmt"
-	"slices"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/easyproto"
-	"github.com/cespare/xxhash/v2"
-)
-
-type TimeSerie struct {
-	GroupLabels []Label
-	Fingerprint uint64
-}
-
-type Label struct {
-	Name  string
-	Value string
-}
-
-func getWriteRequestUnmarshaler() *writeRequestUnmarshaler {
-	v := wruPool.Get()
-	if v == nil {
-		return &writeRequestUnmarshaler{
-			tss:        make([]TimeSerie, 0, 1024),
-			labelsPool: make([]Label, 0, 4096),
-			d:          xxhash.New(),
-		}
-	}
-	return v.(*writeRequestUnmarshaler)
-}
-
-func putWriteRequestUnmarshaler(wru *writeRequestUnmarshaler) {
-	wru.Reset()
-	wruPool.Put(wru)
-}
-
-var wruPool sync.Pool
-
-// WriteRequestUnmarshaler is reusable unmarshaler for WriteRequest protobuf messages.
-//
-// It maintains internal pools for labels and samples to reduce memory allocations.
-// See UnmarshalProtobuf for details on how to use it.
-type writeRequestUnmarshaler struct {
-	tss        []TimeSerie
-	labelsPool []Label
-	d          *xxhash.Digest
-}
-
-// Reset resets wru, so it could be re-used.
-func (wru *writeRequestUnmarshaler) Reset() {
-	wru.tss = wru.tss[:0]
-	wru.labelsPool = wru.labelsPool[:0]
-	wru.d.Reset()
-}
-
-func (wru *writeRequestUnmarshaler) UnmarshalProtobuf(src []byte, groupLabels []string, callback func(tss []TimeSerie)) error {
-	wru.Reset()
-
-	var err error
-
-	tss := wru.tss
-
-	// message WriteRequest {
-	//    repeated TimeSeries timeseries = 1;
-	//    reserved 2;
-	//    repeated Metadata metadata = 3;
-	// }
-	labelsPool := wru.labelsPool
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		if len(tss) >= cap(tss) {
-			callback(tss)
-			tss = tss[:0]
-			labelsPool = labelsPool[:0]
-		}
-
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read the next field: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return fmt.Errorf("cannot read timeseries data")
-			}
-			tss = tss[:len(tss)+1]
-			ts := &tss[len(tss)-1]
-			d := wru.d
-			d.Reset()
-			labelsPool, err = ts.unmarshalProtobuf(data, groupLabels, labelsPool, d)
-			if err != nil {
-				return fmt.Errorf("cannot unmarshal timeseries: %w", err)
-			}
-		}
-	}
-
-	if len(tss) > 0 {
-		callback(tss)
-		tss = tss[:0]
-		labelsPool = labelsPool[:0]
-	}
-
-	wru.tss = tss[:0]
-	wru.labelsPool = labelsPool
-	wru.d.Reset()
-	return nil
-}
-
-func (ts *TimeSerie) unmarshalProtobuf(src []byte, groupLabels []string, labelsPool []Label, d *xxhash.Digest) ([]Label, error) {
-	// message TimeSeries {
-	//   repeated Label labels   = 1;
-	//   repeated Sample samples = 2;
-	// }
-
-	labelsPoolLen := len(labelsPool)
-	var fc easyproto.FieldContext
-	var lfc easyproto.FieldContext
-	for len(src) > 0 {
-		var err error
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelsPool, fmt.Errorf("cannot read the next field: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelsPool, fmt.Errorf("cannot read label data")
-			}
-
-			var nameBytes, valueBytes []byte
-			ldata := data
-			for len(ldata) > 0 {
-				ldata, err = lfc.NextField(ldata)
-				if err != nil {
-					return labelsPool, fmt.Errorf("cannot read label field: %w", err)
-				}
-				switch lfc.FieldNum {
-				case 1:
-					nameBytes, ok = lfc.Bytes()
-					if !ok {
-						return labelsPool, fmt.Errorf("cannot read label name")
-					}
-				case 2:
-					valueBytes, ok = lfc.Bytes()
-					if !ok {
-						return labelsPool, fmt.Errorf("cannot read label value")
-					}
-				}
-			}
-
-			_, _ = d.Write(data)
-
-			name := bytesutil.ToUnsafeString(nameBytes)
-			if slices.Contains(groupLabels, name) {
-				labelsPool = append(labelsPool, Label{
-					Name:  name,
-					Value: bytesutil.ToUnsafeString(valueBytes),
-				})
-			}
-		}
-	}
-	ts.GroupLabels = labelsPool[labelsPoolLen:]
-	ts.Fingerprint = d.Sum64()
-	return labelsPool, nil
-}

app/vmestimator/protoparser/write_request_timing_test.go

@@ -1,86 +0,0 @@
-package protoparser
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-)
-
-func BenchmarkWriteRequest_UnmarshalProtobuf(b *testing.B) {
-	var data = make([]byte, 0, 21_000_000)
-
-	f := func(rows, labels, labelSize, groupBy int) {
-		bName := fmt.Sprintf("Rows=%d/Labels=%d/LabelSize=%d/GroupBy=%d", rows, labels, labelSize, groupBy)
-		b.Run(bName, func(b *testing.B) {
-			data := buildEncodedWriteRequest(data, rows, labels, labelSize, groupBy)
-			groupLabels := []string{
-				"foo",
-				"bar",
-				"baz",
-				"__name__",
-				"job",
-				"groupLabel",
-			}
-
-			wru := getWriteRequestUnmarshaler()
-			cnt := 0
-
-			b.ResetTimer()
-			b.ReportAllocs()
-			b.SetBytes(int64(len(data)))
-			for b.Loop() {
-				wru.Reset()
-				if err := wru.UnmarshalProtobuf(data, groupLabels, func(tss []TimeSerie) {
-					cnt += len(tss)
-				}); err != nil {
-					b.Fatalf("unexpected error: %s", err)
-				}
-			}
-		})
-	}
-
-	f(5_000, 0, 0, 3)
-	f(5_000, 1, 20, 3)
-
-	f(1_000, 20, 20, 3)
-	f(5_000, 20, 20, 3)
-	f(10_000, 20, 20, 3)
-	f(20_000, 20, 20, 3)
-
-	// long label values
-	f(1_000, 20, 2000, 3)
-
-	// many labels
-	f(1_000, 2000, 100, 3)
-}
-
-// buildEncodedWriteRequest builds a snappy-encoded protobuf WriteRequest
-// with numSeries time series, each having numLabels labels of labelSize bytes each.
-func buildEncodedWriteRequest(dst []byte, numSeries, numLabels, labelSize, groupsNum int) []byte {
-	labelValue := strings.Repeat("x", labelSize)
-
-	tss := make([]prompb.TimeSeries, numSeries)
-	for i := range tss {
-		labels := make([]prompb.Label, numLabels)
-		for j := range labels {
-			labels[j] = prompb.Label{
-				Name:  fmt.Sprintf("label%02d", j),
-				Value: fmt.Sprintf("val%05d_%s", i, labelValue),
-			}
-		}
-		labels = append(labels, prompb.Label{
-			Name:  "groupLabel",
-			Value: fmt.Sprintf("%d", i%groupsNum),
-		})
-
-		tss[i] = prompb.TimeSeries{
-			Labels:  labels,
-			Samples: []prompb.Sample{{Value: 1, Timestamp: 1000}},
-		}
-	}
-
-	wr := &prompb.WriteRequest{Timeseries: tss}
-	return wr.MarshalProtobuf(dst[:0])
-}

app/vmestimator/snapshot.go

@@ -1,237 +0,0 @@
-package main
-
-import (
-	"encoding/gob"
-	"io"
-	"strconv"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/axiomhq/hyperloglog"
-)
-
-type snapshots struct {
-	mu sync.Mutex
-	m  map[string]*snapshot
-}
-
-func newSnapshots() *snapshots {
-	return &snapshots{m: make(map[string]*snapshot)}
-}
-
-func (ss *snapshots) add(newS *snapshot) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-
-	key := newS.GroupByKeysLabel
-	if s, found := ss.m[key]; found {
-		s.merge(newS)
-		return
-	}
-
-	s := newSnapshot()
-	s.merge(newS)
-	ss.m[key] = s
-}
-
-func (ss *snapshots) writeMetrics(w io.Writer) error {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-
-	for _, s := range ss.m {
-		if err := s.writeMetrics(w); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-type snapshot struct {
-	MetricPrefix        string
-	GroupByKeysLabel    string
-	GroupRejectedSketch *hyperloglog.Sketch
-	GroupBy             []string
-	// prom string metric => hll
-	Sketches map[string]*hyperloglog.Sketch
-}
-
-func newSnapshot() *snapshot {
-	return &snapshot{
-		Sketches: make(map[string]*hyperloglog.Sketch),
-	}
-}
-
-// decodeSnapshot reads a stream of gob-encoded EstimatorMerge objects from the response and merges them into the provided estimatorMerge object.
-func decodeSnapshots(r io.Reader, cb func(s *snapshot)) error {
-	d := gob.NewDecoder(r)
-	s := newSnapshot()
-	for {
-		s.reset()
-		if err := d.Decode(s); err != nil {
-			if err == io.EOF {
-				return nil
-			}
-			return err
-		}
-
-		cb(s)
-	}
-}
-
-func (s *snapshot) merge(other *snapshot) {
-	if s.GroupByKeysLabel != "" && s.GroupByKeysLabel != other.GroupByKeysLabel {
-		logger.Panicf("BUG: merge snapshots must have the same groupByKeysLabel; s: %s; other: %s", s.GroupByKeysLabel, other.GroupByKeysLabel)
-	}
-
-	for name, otherSK := range other.Sketches {
-		if existing, ok := s.Sketches[name]; ok {
-			existing.Merge(otherSK)
-		} else {
-			s.Sketches[name] = otherSK.Clone()
-		}
-	}
-
-	s.MetricPrefix = other.MetricPrefix
-	s.GroupByKeysLabel = other.GroupByKeysLabel
-	s.GroupBy = append(s.GroupBy, other.GroupBy...)
-	if other.GroupRejectedSketch != nil {
-		if s.GroupRejectedSketch == nil {
-			s.GroupRejectedSketch = other.GroupRejectedSketch.Clone()
-		} else {
-			s.GroupRejectedSketch.Merge(other.GroupRejectedSketch)
-		}
-	}
-}
-
-// writeMetrics writes metrics to w.
-// w must be a buffered writer.
-func (s *snapshot) writeMetrics(w io.Writer) error {
-	for name, sketch := range s.Sketches {
-		if _, err := w.Write(bytesutil.ToUnsafeBytes(name)); err != nil {
-			return err
-		}
-		if _, err := w.Write(strconv.AppendUint(nil, sketch.Estimate(), 10)); err != nil {
-			return err
-		}
-		if _, err := w.Write([]byte("\n")); err != nil {
-			return err
-		}
-	}
-
-	if len(s.GroupBy) > 0 {
-		groupSize := int64(len(s.Sketches))
-		if s.GroupRejectedSketch != nil {
-			groupSize += int64(s.GroupRejectedSketch.Estimate())
-		}
-
-		formatBuf := make([]byte, 0, 1024)
-		formatBuf = appendGroupMetric(formatBuf, s.MetricPrefix, s.GroupByKeysLabel)
-		formatBuf = strconv.AppendInt(formatBuf, groupSize, 10)
-		formatBuf = append(formatBuf, "\n"...)
-		if _, err := w.Write(formatBuf); err != nil {
-			logger.Errorf("writing metrics failed: %s; written cardinality metrics might be incomplete or invalid", err)
-		}
-	}
-
-	return nil
-}
-
-func (s *snapshot) reset() {
-	s.GroupByKeysLabel = ""
-	s.GroupRejectedSketch = nil
-	s.MetricPrefix = ""
-	s.GroupBy = s.GroupBy[:0]
-	clear(s.Sketches)
-}
-
-func convertNoGroupToSnapshot(e *estimator, s *snapshot) *snapshot {
-	if len(e.groupBy) != 0 {
-		panic("BUG: do not use this function for estimator with non-empty groupBy")
-	}
-	if s == nil {
-		s = newSnapshot()
-	}
-	s.reset()
-
-	eb0 := e.buckets[0]
-
-	resSK := eb0.newSketch()
-	for _, eb := range e.buckets {
-		eb.writeNoGroupMetric(resSK)
-	}
-
-	formatBuf := make([]byte, 0, 1024)
-	formatBuf = appendGlobalMetric(formatBuf, eb0.metricPrefix)
-	s.Sketches[string(formatBuf)] = resSK
-	s.GroupByKeysLabel = eb0.groupByKeysLabel
-	s.MetricPrefix = eb0.metricPrefix
-	s.GroupBy = append(s.GroupBy[:0], eb0.groupBy...)
-
-	return s
-}
-
-func convertGroupToSnapshot(e *estimator, s *snapshot) *snapshot {
-	if len(e.groupBy) == 0 {
-		panic("BUG: do not use this function for estimator with empty groupBy")
-	}
-
-	eb0 := e.buckets[0]
-
-	formatBuf := make([]byte, 0, 16384)
-	formatBuf = appendGroupByKeysAndValuesPrefix(formatBuf, eb0.metricPrefix, eb0.groupByKeysLabel)
-
-	if s == nil {
-		s = newSnapshot()
-	}
-	s.reset()
-
-	for _, eb := range e.buckets {
-		eb.groupRejectedMu.Lock()
-		if eb.groupRejectedSketch != nil {
-			s.GroupRejectedSketch = eb.groupRejectedSketch.Clone()
-		}
-		eb.groupRejectedMu.Unlock()
-		s = convertGroupBucketToSnapshot(eb, s, formatBuf)
-	}
-	return s
-}
-
-func convertGroupBucketToSnapshot(eb *estimatorBucket, s *snapshot, formatBuf []byte) *snapshot {
-	if len(eb.groupBy) == 0 {
-		panic("BUG: do not use this function for estimator with empty groupBy")
-	}
-
-	prefixLen := len(formatBuf)
-	resSK := eb.newSketch()
-
-	eb.mu.Lock()
-	defer eb.mu.Unlock()
-	for valuesKey, gsk := range eb.groups {
-		resSK.Reset()
-		formatBuf = append(formatBuf[:prefixLen], gsk.groupValueLabels...)
-		eb.mergeSketches(gsk.Sketch, eb.prevGroups[valuesKey].Sketch, resSK)
-		s.Sketches[string(formatBuf)] = resSK.Clone()
-	}
-	for valuesKey := range eb.prevGroups {
-		if _, ok := eb.groups[valuesKey]; ok {
-			continue
-		}
-
-		resSK.Reset()
-		formatBuf = formatBuf[:prefixLen]
-
-		gsk := eb.prevGroups[valuesKey]
-		formatBuf = append(formatBuf, gsk.groupValueLabels...)
-
-		eb.mergeSketches(nil, eb.prevGroups[valuesKey].Sketch, resSK)
-		s.Sketches[string(formatBuf)] = resSK.Clone()
-	}
-
-	s.GroupByKeysLabel = eb.groupByKeysLabel
-	s.MetricPrefix = eb.metricPrefix
-	s.GroupBy = append(s.GroupBy[:0], eb.groupBy...)
-
-	return s
-}

app/vmestimator/snapshot_test.go

@@ -1,471 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/gob"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmestimator/protoparser"
-)
-
-func TestGlobalSnapshot(t *testing.T) {
-	genCard := func(cardinality int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			var tss []protoparser.TimeSerie
-			fpBuf := make([]byte, 8, 8+len(seed))
-			for i := 0; i < cardinality; i++ {
-				binary.LittleEndian.PutUint64(fpBuf[:8], uint64(i))
-				fpBuf = append(fpBuf, seed...)
-				tss = append(tss, protoparser.TimeSerie{
-					Fingerprint: hash(fpBuf[:]),
-				})
-
-				if i%10 == 0 {
-					e.insertMany(tss)
-					tss = tss[:0]
-				}
-			}
-			if len(tss) > 0 {
-				e.insertMany(tss)
-			}
-		}
-	}
-
-	f := func(gen func(e *estimator)) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval: time.Minute * 10,
-			Buckets:  5,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		if len(e.buckets) != cfg.Buckets {
-			t.Fatalf("expected buckets length to be %d but got %d", cfg.Buckets, len(e.buckets))
-		}
-		for i, eb := range e.buckets {
-			if len(eb.groupBy) > 0 {
-				t.Fatalf("expected bucket %d groupBy length to be 0 but got %d", i, len(eb.groupBy))
-			}
-			if eb.groups != nil {
-				t.Fatalf("expected bucket %d groups length to be 0 but got %d", i, len(eb.groups))
-			}
-			if eb.groupSize.Load() != 0 {
-				t.Fatalf("expected bucket %d groupSize to be 0 but got %d", i, eb.groupSize.Load())
-			}
-		}
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		expMetric := buf.String()
-
-		buf.Reset()
-		if err := convertNoGroupToSnapshot(e, nil).writeMetrics(buf); err != nil {
-			t.Fatalf("convertNoGroupToSnapshot: %v", err)
-		}
-		assertMetricsSame(t, "convertNoGroupToSnapshot", expMetric, buf.String())
-
-		// test encode/decode snapshot produce same result
-		buf.Reset()
-		if err := e.writeSnapshot(gob.NewEncoder(buf)); err != nil {
-			t.Fatalf("writeSnapshot: %v", err)
-		}
-		ss := newSnapshots()
-		if err := decodeSnapshots(buf, func(s *snapshot) {
-			ss.add(s)
-		}); err != nil {
-			t.Fatalf("decodeSnapshot: %v", err)
-		}
-		buf.Reset()
-		if err := ss.writeMetrics(buf); err != nil {
-			t.Fatalf("writeMetrics: %v", err)
-		}
-		assertMetricsSame(t, "encode/decode", expMetric, buf.String())
-	}
-
-	f(func(e *estimator) {})
-
-	// no previous
-	f(genCard(0, ""))
-	f(genCard(100, ""))
-	f(genCard(10000, ""))
-	f(genCard(100000, ""))
-
-	// rotate once
-	genRotateOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality, "")(e)
-			e.rotate()
-		}
-	}
-	f(genRotateOnce(0))
-	f(genRotateOnce(100))
-	f(genRotateOnce(10000))
-	f(genRotateOnce(100000))
-
-	// insert, rotate insert the same
-	genInsertRotateInsertSameOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality/2, "")(e)
-			e.rotate()
-			genCard(cardinality/2, "")(e)
-		}
-	}
-	f(genInsertRotateInsertSameOnce(0))
-	f(genInsertRotateInsertSameOnce(100))
-	f(genInsertRotateInsertSameOnce(10000))
-	f(genInsertRotateInsertSameOnce(100000))
-
-	// insert, rotate insert
-	genInsertRotateInsertOnce := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality/2, "one")(e)
-			e.rotate()
-			genCard(cardinality/2, "two")(e)
-		}
-	}
-	f(genInsertRotateInsertOnce(0))
-	f(genInsertRotateInsertOnce(100))
-	f(genInsertRotateInsertOnce(10000))
-	f(genInsertRotateInsertOnce(100000))
-
-	// insert, rotate insert
-	genRotateTwoTimes := func(cardinality int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(cardinality, "")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f(genRotateTwoTimes(0))
-	f(genRotateTwoTimes(100))
-	f(genRotateTwoTimes(10000))
-	f(genRotateTwoTimes(100000))
-}
-
-func TestGroupSnapshot(t *testing.T) {
-	genCard := func(fooCard, barCard, bazCard int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			var tss []protoparser.TimeSerie
-			for fooI := 0; fooI < max(1, fooCard); fooI++ {
-				for barI := 0; barI < max(1, barCard); barI++ {
-					for bazI := 0; bazI < max(1, bazCard); bazI++ {
-						ts := protoparser.TimeSerie{}
-						ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "__name__", Value: "the_metric_name"})
-						if fooCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "foo", Value: fmt.Sprintf("%s%d", seed, fooI)})
-						}
-						if barCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "bar", Value: fmt.Sprintf("%s%d", seed, barI)})
-						}
-						if bazCard > 0 {
-							ts.GroupLabels = append(ts.GroupLabels, protoparser.Label{Name: "baz", Value: fmt.Sprintf("%s%d", seed, bazI)})
-						}
-						var fpBuf []byte
-						for _, l := range ts.GroupLabels {
-							fpBuf = append(fpBuf, l.Name...)
-							fpBuf = append(fpBuf, '=')
-							fpBuf = append(fpBuf, l.Value...)
-							fpBuf = append(fpBuf, ',')
-						}
-						fpBuf = append(fpBuf, seed...)
-						ts.Fingerprint = hash(fpBuf)
-						tss = append(tss, ts)
-					}
-				}
-			}
-			e.insertMany(tss)
-		}
-	}
-
-	f := func(groupBy []string, gen func(e *estimator)) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval: time.Minute * 10,
-			GroupBy:  groupBy,
-			Buckets:  5,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		if len(e.buckets) != cfg.Buckets {
-			t.Fatalf("expected buckets length to be %d but got %d", cfg.Buckets, len(e.buckets))
-		}
-		for i, eb := range e.buckets {
-			if eb.sketch != nil {
-				t.Fatalf("expected bucket %d sketch to be nil", i)
-			}
-			if eb.prevSketch != nil {
-				t.Fatalf("expected bucket %d prevSketch to be nil", i)
-			}
-		}
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		expMetrics := buf.String()
-
-		buf.Reset()
-		if err := convertGroupToSnapshot(e, nil).writeMetrics(buf); err != nil {
-			t.Fatalf("failed to write metrics: %v", err)
-		}
-		assertMetricsSame(t, "convertGroupToSnapshot", expMetrics, buf.String())
-
-		// test encode/decode snapshot produce same result
-		buf.Reset()
-		if err := e.writeSnapshot(gob.NewEncoder(buf)); err != nil {
-			t.Fatalf("writeSnapshot: %v", err)
-		}
-		ss := newSnapshots()
-		if err := decodeSnapshots(buf, func(s *snapshot) {
-			ss.add(s)
-		}); err != nil {
-			t.Fatalf("decodeSnapshot: %v", err)
-		}
-		buf.Reset()
-		if err := ss.writeMetrics(buf); err != nil {
-			t.Fatalf("writeMetrics: %v", err)
-		}
-		assertMetricsSame(t, "encode/decode", expMetrics, buf.String())
-	}
-
-	f([]string{"__name__"}, func(e *estimator) {})
-
-	// group by metric name
-	f([]string{"__name__"}, genCard(10, 10, 10, ""))
-
-	// time series does not contribute to a group
-	f([]string{"foo"}, genCard(0, 10, 10, ""))
-
-	// group by one label
-	f([]string{"foo"}, genCard(1, 100, 0, ""))
-	f([]string{"foo"}, genCard(1, 10000, 0, ""))
-	f([]string{"foo"}, genCard(1, 1000, 1000, ""))
-
-	// group by one label, rotate
-	genCardRotate := func(fooCard, barCard, bazCard int, seed string) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(fooCard, barCard, bazCard, seed)(e)
-			e.rotate()
-		}
-	}
-	f([]string{"foo"}, genCardRotate(1, 10, 10, ""))
-	f([]string{"foo"}, genCardRotate(1, 1000, 1000, ""))
-
-	// group by one label, rotate, insert same
-	genCardRotateInsertSame := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "")(e)
-			e.rotate()
-			genCard(1, barCard, bazCard, "")(e)
-		}
-	}
-	f([]string{"foo"}, genCardRotateInsertSame(10, 10))
-	f([]string{"foo"}, genCardRotateInsertSame(1000, 1000))
-
-	// group by one label, rotate, insert diff
-	genCardRotateInsertDiff := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "one")(e)
-			e.rotate()
-			genCard(1, barCard, bazCard, "two")(e)
-		}
-	}
-	f([]string{"foo"}, genCardRotateInsertDiff(10, 10))
-	f([]string{"foo"}, genCardRotateInsertDiff(1000, 1000))
-
-	// group by one label, rotate, insert diff
-	genCardRotateTwice := func(barCard, bazCard int) func(e *estimator) {
-		return func(e *estimator) {
-			genCard(1, barCard, bazCard, "one")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f([]string{"foo"}, genCardRotateTwice(10, 10))
-	f([]string{"foo"}, genCardRotateTwice(1000, 1000))
-
-	// group by two labels
-	f([]string{"foo", "bar"}, genCard(1, 1, 1000, ""))
-	f([]string{"foo", "bar"}, genCard(2, 1, 1000, ""))
-	f([]string{"foo", "bar"}, genCard(2, 2, 1000, ""))
-
-	// group by two labels, rotate
-	genCardTwoLabelsRotate := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "")(e)
-			e.rotate()
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotate())
-
-	// group by two labels, rotate, insert same
-	genCardTwoLabelsRotateInsertSame := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "")(e)
-			e.rotate()
-			genCard(2, 2, 1000, "")(e)
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotateInsertSame())
-
-	// group by two labels, rotate, insert diff
-	genCardTwoLabelsRotateInsertDiff := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "one")(e)
-			e.rotate()
-			genCard(2, 2, 1000, "two")(e)
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotateInsertDiff())
-
-	// group by two labels, rotate, insert diff
-	genCardTwoLabelsRotateTwice := func() func(e *estimator) {
-		return func(e *estimator) {
-			genCard(2, 2, 1000, "one")(e)
-			e.rotate()
-			e.rotate()
-		}
-	}
-	f([]string{"foo", "bar"}, genCardTwoLabelsRotateTwice())
-
-	// quote values: label values with special characters must be properly escaped
-	genSpecialCard := func(fooVal string) func(e *estimator) {
-		return func(e *estimator) {
-			e.insertMany([]protoparser.TimeSerie{
-				{
-					GroupLabels: []protoparser.Label{{Name: "foo", Value: fooVal}},
-					Fingerprint: hash([]byte("foo=" + fooVal + ",")),
-				},
-			})
-		}
-	}
-
-	// double quote in value
-	f([]string{"foo"}, genSpecialCard(`a"b`))
-	f([]string{"foo"}, genSpecialCard(`a\b`))
-	f([]string{"foo"}, genSpecialCard("a\nb"))
-	f([]string{"foo"}, genSpecialCard("a\tb"))
-}
-
-func TestGroupSnapshotGroupLimit(t *testing.T) {
-	makeTS := func(fooVal string) protoparser.TimeSerie {
-		return protoparser.TimeSerie{
-			GroupLabels: []protoparser.Label{{Name: "foo", Value: fooVal}},
-			Fingerprint: hash([]byte("foo=" + fooVal + ",")),
-		}
-	}
-
-	f := func(groupLimit int, gen func(e *estimator), expRejected int) {
-		t.Helper()
-
-		cfg := EstimatorConfig{
-			Interval:   time.Minute * 10,
-			GroupBy:    []string{"foo"},
-			GroupLimit: groupLimit,
-			Buckets:    3,
-		}
-
-		e, err := newEstimator(cfg)
-		if err != nil {
-			t.Fatalf("failed to create new estimator: %v", err)
-		}
-		defer e.stop()
-
-		gen(e)
-
-		buf := bytes.NewBuffer(nil)
-		e.writeMetrics(buf)
-		expMetrics := buf.String()
-
-		buf.Reset()
-		s := convertGroupToSnapshot(e, nil)
-		if err := s.writeMetrics(buf); err != nil {
-			t.Fatalf("failed to write metrics: %v", err)
-		}
-		assertMetricsSame(t, "convertGroupToSnapshot", expMetrics, buf.String())
-
-		var actRejected int
-		if s.GroupRejectedSketch != nil {
-			actRejected = int(s.GroupRejectedSketch.Estimate())
-		}
-		if expRejected != actRejected {
-			t.Fatalf("rejected expected: %d; got: %d", expRejected, actRejected)
-		}
-
-		// test encode/decode snapshot produce same result
-		buf.Reset()
-		if err := e.writeSnapshot(gob.NewEncoder(buf)); err != nil {
-			t.Fatalf("writeSnapshot: %v", err)
-		}
-		ss := newSnapshots()
-		if err := decodeSnapshots(buf, func(s *snapshot) {
-			ss.add(s)
-		}); err != nil {
-			t.Fatalf("decodeSnapshot: %v", err)
-		}
-		buf.Reset()
-		if err := ss.writeMetrics(buf); err != nil {
-			t.Fatalf("writeMetrics: %v", err)
-		}
-		assertMetricsSame(t, "encode/decode", expMetrics, buf.String())
-	}
-
-	// all groups accepted
-	f(3, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 0)
-
-	// 2 groups only accepted
-	f(2, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 1)
-
-	// one group only accepted
-	f(1, func(e *estimator) {
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b"), makeTS("c")})
-	}, 2)
-
-	// after rotate: groups in prevGroups bypass the limit; new groups are still checked
-	f(2, func(e *estimator) {
-		// fills limit
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b")})
-		e.rotate()
-		// "a" bypasses, "c" rejected
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("c")})
-	}, 1)
-
-	// after rotate: new group accepted when remaining capacity allows
-	f(3, func(e *estimator) {
-		// 2 groups, limit=3
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("b")})
-		e.rotate()
-		// "a" bypasses, "c" accepted (2+1=3 <= 3)
-		e.insertMany([]protoparser.TimeSerie{makeTS("a"), makeTS("c")})
-	}, 0)
-
-	// reject 100
-	f(3, func(e *estimator) {
-		var tss []protoparser.TimeSerie
-		for i := 0; i < 103; i++ {
-			tss = append(tss, makeTS(fmt.Sprintf("a%d", i)))
-		}
-		e.insertMany(tss)
-	}, 100)
-}

app/vminsert/common/insert_ctx.go

@@ -175,13 +175,19 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmpbs))
-	for idx, mmpb := range mmpbs {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmpbs {
+		if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
+			continue
+		}
+		mm := &mms[cnt]
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
 
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
@@ -201,14 +207,19 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmps))
-	for idx, mmpb := range mmps {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmps {
+		mm := &mms[cnt]
+		if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
+			continue
+		}
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
-
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{

app/vminsert/influx/request_handler.go

@@ -49,7 +49,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vminsert/main.go

@@ -68,9 +68,11 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented.")
+	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
+	maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
 )
 
 var (
@@ -106,7 +108,7 @@ func Init() {
 	promscrape.Init(func(_ *auth.Token, wr *prompb.WriteRequest) {
 		prompush.Push(wr)
 	})
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	timeserieslimits.MustInit(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 }
 
 // Stop stops vminsert.

app/vmselect/main.go

@@ -541,7 +541,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmselect/prometheus/metadata_response.qtpl

@@ -15,7 +15,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-m
 			currentItem := 0
 		 %}
 		{% for _, row := range result %}
-			"{%s string(row.MetricFamilyName) %}": [
+			{%q= string(row.MetricFamilyName) %}: [
 					{
 						"type": {%q= row.Type.String() %},
             {% if len(row.Unit) > 0 -%}

app/vmselect/prometheus/metadata_response.qtpl.go

@@ -35,12 +35,10 @@ func StreamMetadataResponse(qw422016 *qt422016.Writer, result []*metricsmetadata
 
 //line app/vmselect/prometheus/metadata_response.qtpl:17
 	for _, row := range result {
-//line app/vmselect/prometheus/metadata_response.qtpl:17
-		qw422016.N().S(`"`)
 //line app/vmselect/prometheus/metadata_response.qtpl:18
-		qw422016.E().S(string(row.MetricFamilyName))
+		qw422016.N().Q(string(row.MetricFamilyName))
 //line app/vmselect/prometheus/metadata_response.qtpl:18
-		qw422016.N().S(`": [{"type":`)
+		qw422016.N().S(`: [{"type":`)
 //line app/vmselect/prometheus/metadata_response.qtpl:20
 		qw422016.N().Q(row.Type.String())
 //line app/vmselect/prometheus/metadata_response.qtpl:20

app/vmselect/prometheus/prometheus.go

@@ -956,6 +956,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	mayCache := !httputil.GetBool(r, "nocache")
+	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
@@ -977,18 +978,19 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	}
 
 	ec := &promql.EvalConfig{
-		Start:               start,
-		End:                 end,
-		Step:                step,
-		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           0, // let vmstorage use maxUniqueTimeseries by default
-		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
-		Deadline:            deadline,
-		MayCache:            mayCache,
-		LookbackDelta:       lookbackDelta,
-		RoundDigits:         getRoundDigits(r),
-		EnforcedTagFilterss: etfs,
-		CacheTagFilters:     etfs,
+		Start:                            start,
+		End:                              end,
+		Step:                             step,
+		MaxPointsPerSeries:               *maxPointsPerTimeseries,
+		MaxSeries:                        0, // let vmstorage use maxUniqueTimeseries by default
+		QuotedRemoteAddr:                 httpserver.GetQuotedRemoteAddr(r),
+		Deadline:                         deadline,
+		MayCache:                         mayCache,
+		OptimizeRepeatedBinaryOpSubexprs: optimizeRepeatedBinaryOpSubexprs,
+		LookbackDelta:                    lookbackDelta,
+		RoundDigits:                      getRoundDigits(r),
+		EnforcedTagFilterss:              etfs,
+		CacheTagFilters:                  etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},

app/vmselect/promql/binary_op.go

@@ -164,15 +164,21 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		switch true {
-		case metricsql.IsBinaryOpCmp(op):
-			// Do not remove empty series for comparison operations,
-			// since this may lead to missing result.
-		default:
+		isCmpOp := metricsql.IsBinaryOpCmp(op)
+		if !isCmpOp {
+			// Do not remove empty series for comparison operations, since NaN can be an
+			// explicitly present sample value. In particular, `value != NaN` is true.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -183,6 +189,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			logger.Panicf("BUG: len(left) must match len(right) and len(dst); got %d vs %d vs %d", len(left), len(right), len(dst))
 		}
 		isBool := bfa.be.Bool
+		fillLeft := bfa.be.FillLeft
+		fillRight := bfa.be.FillRight
+		// A NaN produced by a vector comparison denotes a filtered-out sample rather
+		// than an explicitly present NaN value. Drop it when this result is used as
+		// the right-hand side of another comparison.
+		// Note that filtered-out samples surviving as NaN inside other wrappers such as
+		// transform functions or arithmetic operations aren't detected, since such NaN
+		// is indistinguishable from an explicitly present NaN value there.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
+		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -193,6 +209,24 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			}
 			for j, a := range leftValues {
 				b := rightValues[j]
+				leftIsNaN := math.IsNaN(a)
+				rightIsNaN := math.IsNaN(b)
+				// Both sides are NaN.
+				if leftIsNaN && rightIsNaN {
+					dstValues[j] = bf(a, b, isBool)
+					continue
+				}
+				if dropNaNRight && rightIsNaN && fillRight == nil {
+					dstValues[j] = nan
+					continue
+				}
+				// Apply the fill value when either the left or right side is NaN, but not both.
+				if leftIsNaN && fillLeft != nil {
+					a = fillLeft.N
+				}
+				if rightIsNaN && fillRight != nil {
+					b = fillRight.N
+				}
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
@@ -202,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
+// isVectorComparisonExpr returns whether e is a comparison operation
+// with at least one non-scalar operand.
+func isVectorComparisonExpr(e metricsql.Expr) bool {
+	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
+		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
+		// the shape of the result. Subqueries are left as is, since rollup
+		// functions skip NaN values on their own.
+		e = re.Expr
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
+		return false
+	}
+	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
+}
+
+// isScalarLikeExpr returns whether e always evaluates to a scalar value.
+func isScalarLikeExpr(e metricsql.Expr) bool {
+	switch e := e.(type) {
+	case *metricsql.NumberExpr, *metricsql.DurationExpr:
+		return true
+	case *metricsql.BinaryOpExpr:
+		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
+	case *metricsql.FuncExpr:
+		switch strings.ToLower(e.Name) {
+		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
+			return true
+		}
+	}
+	return false
+}
+
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {
@@ -226,7 +292,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +305,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0])}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0])}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +370,28 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with NaN values for the fill_left()/fill_right()/fill() modifiers.
+// These NaN values will be replaced later with the fill value if needed.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = math.NaN()
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -132,6 +132,9 @@ type EvalConfig struct {
 	// Whether the response can be cached.
 	MayCache bool
 
+	// Whether repeated cacheable binary op subexpressions can be optimized.
+	OptimizeRepeatedBinaryOpSubexprs bool
+
 	// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
 	LookbackDelta int64
 
@@ -171,6 +174,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.MaxPointsPerSeries = src.MaxPointsPerSeries
 	ec.Deadline = src.Deadline
 	ec.MayCache = src.MayCache
+	ec.OptimizeRepeatedBinaryOpSubexprs = src.OptimizeRepeatedBinaryOpSubexprs
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
@@ -420,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -447,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -455,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -466,82 +486,185 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
-	if !canPushdownCommonFilters(be) {
-		// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
-		// from exprFirst to exprSecond.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
-		qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
-		defer qt.Done()
-		var wg sync.WaitGroup
-
-		var tssFirst []*timeseries
-		var errFirst error
-		qtFirst := qt.NewChild("expr1")
-		wg.Go(func() {
-			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
-			qtFirst.Done()
-		})
-
-		var tssSecond []*timeseries
-		var errSecond error
-		qtSecond := qt.NewChild("expr2")
-		wg.Go(func() {
-			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
-			qtSecond.Done()
-		})
-
-		wg.Wait()
-		if errFirst != nil {
-			return nil, nil, errFirst
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
 		}
-		if errSecond != nil {
-			return nil, nil, errSecond
+		return tssSecond, tssFirst, nil
+	}
+	if canPushdownCommonFilters(be) {
+		// Execute binary operation in the following way:
+		//
+		// 1) execute the exprFirst
+		// 2) get common label filters for series returned at step 1
+		// 3) push down the found common label filters to exprSecond. This filters out unneeded series
+		//    during exprSecond execution instead of spending compute resources on extracting and processing these series
+		//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+		// 4) execute the exprSecond with possible additional filters found at step 3
+		//
+		// Typical use cases:
+		// - Kubernetes-related: show pod creation time with the node name:
+		//
+		//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
+		//
+		//   Without the optimization `kube_pod_info` would select and spend compute resources
+		//   for more time series than needed. The selected time series would be dropped later
+		//   when matching time series on the right and left sides of binary operand.
+		//
+		// - Generic alerting queries, which rely on `info` metrics.
+		//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
+		//
+		// - Queries, which get additional labels from `info` metrics.
+		//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
+		tssFirst, err := evalExpr(qt, ec, exprFirst)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
+			// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
+			// since the "exprFirst op exprSecond" would return an empty result in any case.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
+			return nil, nil, nil
+		}
+		lfs := getCommonLabelFilters(tssFirst)
+		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
+		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
+		tssSecond, err := evalExpr(qt, ec, exprSecond)
+		if err != nil {
+			return nil, nil, err
+		}
+		return sortResult(tssFirst, tssSecond)
+	}
+
+	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
+	// in exprFirst and exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
+	if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
+		qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
+		defer qt.Done()
+
+		qtFirst := qt.NewChild("expr1")
+		tssFirst, err := evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		qtSecond := qt.NewChild("expr2")
+		tssSecond, err := evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+		if err != nil {
+			return nil, nil, err
 		}
 		return tssFirst, tssSecond, nil
 	}
 
-	// Execute binary operation in the following way:
-	//
-	// 1) execute the exprFirst
-	// 2) get common label filters for series returned at step 1
-	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
-	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-	// 4) execute the exprSecond with possible additional filters found at step 3
-	//
-	// Typical use cases:
-	// - Kubernetes-related: show pod creation time with the node name:
-	//
-	//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
-	//
-	//   Without the optimization `kube_pod_info` would select and spend compute resources
-	//   for more time series than needed. The selected time series would be dropped later
-	//   when matching time series on the right and left sides of binary operand.
-	//
-	// - Generic alerting queries, which rely on `info` metrics.
-	//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
-	//
-	// - Queries, which get additional labels from `info` metrics.
-	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-	tssFirst, err := evalExpr(qt, ec, exprFirst)
-	if err != nil {
-		return nil, nil, err
+	// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
+	// from exprFirst to exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
+	qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
+	defer qt.Done()
+	var wg sync.WaitGroup
+
+	var tssFirst []*timeseries
+	var errFirst error
+	qtFirst := qt.NewChild("expr1")
+	wg.Go(func() {
+		tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+	})
+
+	var tssSecond []*timeseries
+	var errSecond error
+	qtSecond := qt.NewChild("expr2")
+	wg.Go(func() {
+		tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+	})
+
+	wg.Wait()
+	if errFirst != nil {
+		return nil, nil, errFirst
 	}
-	if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
-		// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
-		// since the "exprFirst op exprSecond" would return an empty result in any case.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
-		return nil, nil, nil
+	if errSecond != nil {
+		return nil, nil, errSecond
 	}
-	lfs := getCommonLabelFilters(tssFirst)
-	lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
-	exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
-	tssSecond, err := evalExpr(qt, ec, exprSecond)
-	if err != nil {
-		return nil, nil, err
+	return sortResult(tssFirst, tssSecond)
+}
+
+func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {
+	if !ec.OptimizeRepeatedBinaryOpSubexprs {
+		return false
 	}
-	return tssFirst, tssSecond, nil
+	if ec.Start == ec.End {
+		return false
+	}
+	if !ec.mayCache() {
+		return false
+	}
+
+	candidatesFirst := make(map[string]struct{}, 1)
+	var b []byte
+	visitOptimizedAggrs(exprFirst, func(ae *metricsql.AggrFuncExpr) {
+		if hasUnseededVolatileFunc(ae) {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		candidatesFirst[string(b)] = struct{}{}
+	})
+	if len(candidatesFirst) == 0 {
+		return false
+	}
+
+	repeated := false
+	visitOptimizedAggrs(exprSecond, func(ae *metricsql.AggrFuncExpr) {
+		if repeated {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		_, repeated = candidatesFirst[string(b)]
+	})
+	return repeated
+}
+
+func visitOptimizedAggrs(e metricsql.Expr, f func(ae *metricsql.AggrFuncExpr)) {
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		ae, ok := expr.(*metricsql.AggrFuncExpr)
+		if !ok {
+			return
+		}
+		if getIncrementalAggrFuncCallbacks(ae.Name) == nil {
+			return
+		}
+		fe, _ := tryGetArgRollupFuncWithMetricExpr(ae)
+		if fe == nil {
+			return
+		}
+		f(ae)
+	})
+}
+
+func hasUnseededVolatileFunc(e metricsql.Expr) bool {
+	found := false
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		if found {
+			return
+		}
+		fe, ok := expr.(*metricsql.FuncExpr)
+		if !ok {
+			return
+		}
+		switch strings.ToLower(fe.Name) {
+		case "now":
+			found = true
+		case "rand", "rand_normal", "rand_exponential":
+			found = len(fe.Args) == 0
+		}
+	})
+	return found
 }
 
 func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {

app/vmselect/promql/eval_test.go

@@ -170,3 +170,87 @@ func TestGetSumInstantValues(t *testing.T) {
 		[]*timeseries{ts("foo", 100, 1)},
 	)
 }
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsGate(t *testing.T) {
+	e, err := metricsql.Parse(`count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`)
+	if err != nil {
+		t.Fatalf("unexpected error in metricsql.Parse(): %s", err)
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok {
+		t.Fatalf("unexpected expr type; got %T; want *metricsql.BinaryOpExpr", e)
+	}
+
+	f := func(name string, ec *EvalConfig, resultExpected bool) {
+		t.Helper()
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v", name, result, resultExpected)
+		}
+	}
+
+	f("disabled optimization", &EvalConfig{
+		Start: 1000,
+		End:   2000,
+		Step:  1000,
+	}, false)
+	f("disabled cache", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("instant query", &EvalConfig{
+		Start:                            1000,
+		End:                              1000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("repeated cacheable aggregate subexpression", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, true)
+	f("unaligned range query", &EvalConfig{
+		Start:                            1001,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+}
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsExpressions(t *testing.T) {
+	f := func(name, q string, resultExpected bool) {
+		t.Helper()
+		e, err := metricsql.Parse(q)
+		if err != nil {
+			t.Fatalf("unexpected error in metricsql.Parse(%q) for %q: %s", q, name, err)
+		}
+		be, ok := e.(*metricsql.BinaryOpExpr)
+		if !ok {
+			t.Fatalf("unexpected expr type for %q; got %T; want *metricsql.BinaryOpExpr", name, e)
+		}
+		ec := &EvalConfig{Start: 1000, End: 2000, Step: 1000, MayCache: true, OptimizeRepeatedBinaryOpSubexprs: true}
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v; query: %q", name, result, resultExpected, q)
+		}
+	}
+
+	f("original issue query", `count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`, true)
+	f("right side contains repeated count aggregate", `count(foo) by (job) / (count(foo) by (job) + 1)`, true)
+	f("same sum aggregate", `sum(rate(foo[5m])) by (job) / sum(rate(foo[5m])) by (job)`, true)
+	f("same inner rollup but different aggregates", `sum(rate(foo[5m])) by (job) / count(rate(foo[5m])) by (job)`, false)
+	f("different count aggregates", `count(foo) by (job) / count(bar) by (job)`, false)
+	f("bare metric selector", `foo / foo`, false)
+	f("bare rollup function", `rate(a[5m]) / rate(a[5m])`, false)
+	f("now at modifier", `sum(rate(foo[5m] @ now())) by (job) / sum(rate(foo[5m] @ now())) by (job)`, false)
+	f("unseeded rand at modifier", `sum(rate(foo[5m] @ rand())) by (job) / sum(rate(foo[5m] @ rand())) by (job)`, false)
+	f("unseeded rand_normal at modifier", `sum(rate(foo[5m] @ rand_normal())) by (job) / sum(rate(foo[5m] @ rand_normal())) by (job)`, false)
+	f("unseeded rand_exponential at modifier", `sum(rate(foo[5m] @ rand_exponential())) by (job) / sum(rate(foo[5m] @ rand_exponential())) by (job)`, false)
+	f("seeded rand at modifier", `sum(rate(foo[5m] @ rand(1))) by (job) / sum(rate(foo[5m] @ rand(1))) by (job)`, true)
+}

app/vmselect/promql/exec_test.go

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != NaN`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != 1200`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (1 > 2)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
+		// A missing sample on the left side results in NaN for every comparison
+		// operation, so no special handling is needed there.
+		t.Parallel()
+		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
+		// Prometheus drops the timestamps filtered out by the comparison on the right.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`
@@ -4006,6 +4136,275 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill() both sides NaN case`, func(t *testing.T) {
+		t.Parallel()
+		q := `(
+			label_set(time() <= 1200, "foo", "common")
+		) + fill(10) (
+			label_set(time() >= 1600, "foo", "common")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1010, 1210, nan, 1610, 1810, 2010},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`
@@ -4833,13 +5232,137 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`buckets_limit(zero)`, func(t *testing.T) {
+	// buckets that are consecutively empty at left and right ends will not be preserved.
+	t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
 		t.Parallel()
-		q := `buckets_limit(0, (
-			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
-			alias(label_set(50, "le", "120", "x", "y"), "metric"),
-		))`
-		resultExpected := []netstorage.Result{}
+		q := `sort(buckets_limit(3, (
+			alias(label_set(36, "le", "+Inf"), "metric"),
+			alias(label_set(36, "le", "25"), "metric"),
+			alias(label_set(36, "le", "21"), "metric"),
+			alias(label_set(36, "le", "19"), "metric"),
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+
+	// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
+	t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(buckets_limit(5, (
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("6"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("metric")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.MetricGroup = []byte("metric")
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
 		f(q, resultExpected)
 	})
 	t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6228,50 +6751,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
 		f(q, resultExpected)
 	})
-	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
-		t.Parallel()
-		q := `sort_by_label(
-			buckets_limit(
-				3,
-				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
-			), "le"
-		)`
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("+Inf"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("1.000e+00"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("2.448e+00"),
-			},
-		}
-		resultExpected := []netstorage.Result{r1, r2, r3}
-		f(q, resultExpected)
-	})
 	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

app/vmselect/promql/rollup.go

@@ -1055,7 +1055,7 @@ func newRollupHoltWinters(args []any) (rollupFunc, error) {
 			return nan
 		}
 
-		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing .
+		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29 .
 		// TODO: determine whether this shit really works.
 		s0 := rfa.prevValue
 		if math.IsNaN(s0) {

app/vmselect/promql/timeseries.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"strconv"
 	"sync"
@@ -288,6 +289,9 @@ func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
 }
 
 func marshalBytesFast(dst []byte, s []byte) []byte {
+	if len(s) > math.MaxUint16 {
+		logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
+	}
 	dst = encoding.MarshalUint16(dst, uint16(len(s)))
 	dst = append(dst, s...)
 	return dst

app/vmselect/promql/transform.go

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	if limit <= 0 {
-		return nil, nil
+		return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
 	}
 	if limit < 3 {
 		// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,6 +461,23 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 				prevValue = value
 			}
 		}
+
+		// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
+		epsilon := 1e-9
+		isEmptyBucket := func(hits float64) bool {
+			return !math.IsNaN(hits) && math.Abs(hits) < epsilon
+		}
+		l := 0
+		r := len(leGroup) - 1
+		for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
+			r--
+		}
+		for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
+			l++
+		}
+		leGroup = leGroup[l : r+1]
+
 		for len(leGroup) > limit {
 			// Preserve the first and the last bucket for better accuracy for min and max values
 			xxMinIdx := 1
@@ -1121,29 +1138,29 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 
 func fixBrokenBuckets(i int, xss []leTimeseries) {
 	// Buckets are already sorted by le, so their values must be in ascending order,
-	// since the next bucket includes all the previous buckets.
-	// If the next bucket has lower value than the current bucket,
-	// then the next bucket must be substituted with the current bucket value.
+	// since the upper bucket includes all the lower buckets.
+	// If the upper bucket has lower value than the current bucket,
+	// then the upper bucket must be substituted with the current bucket value.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102
 	if len(xss) < 2 {
 		return
 	}
 
-	vNext := xss[0].ts.Values[i]
+	vPrev := xss[0].ts.Values[i]
 	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
 	// compared with upper buckets in the loop below.
-	if math.IsNaN(vNext) {
-		vNext = 0
-		xss[0].ts.Values[i] = vNext
+	if math.IsNaN(vPrev) {
+		vPrev = 0
+		xss[0].ts.Values[i] = vPrev
 	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
-	// or are bigger than the lower bucket values.
+	// or are smaller than the lower bucket values.
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
-		if math.IsNaN(v) || vNext > v {
-			xss[j].ts.Values[i] = vNext
+		if math.IsNaN(v) || vPrev > v {
+			xss[j].ts.Values[i] = vPrev
 		} else {
-			vNext = v
+			vPrev = v
 		}
 	}
 }

app/vmselect/vmui/assets/MetricsQL-CQAtdgI6.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmselect/vmui/assets/rolldown-runtime-CNC7AqOf.js

@@ -0,0 +1 @@
+var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/assets/rolldown-runtime-COnpUsM8.js

@@ -1 +0,0 @@
-var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-CoGukb-x.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
+  <script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -30,6 +30,17 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	maxBackfillAge = flagutil.NewRetentionDuration("maxBackfillAge", "0", "The maximum allowed age for the ingested samples with historical timestamps. "+
+		"Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. "+
+		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
+		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
+	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
+		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -108,7 +119,7 @@ func DataPath() string {
 }
 
 // Init initializes vmstorage.
-func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
@@ -143,6 +154,7 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	opts := storage.OpenOptions{
 		Retention:                   retentionPeriod.Duration(),
 		FutureRetention:             futureRetention.Duration(),
+		MaxBackfillAge:              maxBackfillAge.Duration(),
 		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
 		MaxHourlySeries:             getMaxHourlySeries(),
 		MaxDailySeries:              getMaxDailySeries(),
@@ -169,6 +181,21 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
 
+	if *vmselectAddr != "" {
+		var err error
+		limits := vmselectapi.Limits{
+			MaxConcurrentRequests:         vmselectMaxConcurrentRequests,
+			MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
+			MaxQueueDuration:              vmselectMaxQueueDuration,
+			MaxQueueDurationFlagName:      "search.maxQueueDuration",
+		}
+		api := newVMStorageWithTenantID(vmStorage)
+		vmselectSrv, err = vmselectapi.NewServer(*vmselectAddr, api, limits, *vmselectDisableRPCCompression)
+		if err != nil {
+			logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
+		}
+	}
+
 	VMInsertAPI = vmStorage
 	VMSelectAPI = vmStorage
 	GetSearch = vmStorage.GetSearch
@@ -191,6 +218,8 @@ var (
 
 	// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
 	DebugFlush func()
+
+	vmselectSrv *vmselectapi.Server
 )
 
 // Stop stops the vmstorage
@@ -201,6 +230,10 @@ func Stop() {
 
 	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime := time.Now()
+
+	if vmselectSrv != nil {
+		vmselectSrv.MustStop()
+	}
 	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -443,8 +476,10 @@ func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/inmemory"}`, tm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/small"}`, tm.SmallSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/big"}`, tm.BigSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/metaindex"}`, tm.MetaindexSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/metaindex"}`, idbm.MetaindexSizeBytes)
 
 	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)

app/vmstorage/vmstorage.go

@@ -164,6 +164,10 @@ func (vms *VMStorage) IsReadOnly() bool {
 }
 
 func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	return vms.initSearch(qt, sq, marshalDefault, deadline)
+}
+
+func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, marshal marshalFunc, deadline uint64) (vmselectapi.BlockIterator, error) {
 	vms.wg.Add(1)
 
 	tr := sq.GetTimeRange()
@@ -178,6 +182,7 @@ func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery
 		return nil, fmt.Errorf("missing tag filters")
 	}
 	bi := getBlockIterator()
+	bi.marshal = marshal
 	bi.wgDone = vms.wg.Done
 	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
 	if err := bi.sr.Error(); err != nil {
@@ -198,11 +203,19 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
 	return searchQueryLimit
 }
 
+type marshalFunc func(dst []byte, src *storage.MetricBlock) []byte
+
+// marshalDefault is the default implementation of the MetricBlock marshaling.
+func marshalDefault(dst []byte, src *storage.MetricBlock) []byte {
+	return src.Marshal(dst)
+}
+
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
-	sr     storage.Search
-	mb     storage.MetricBlock
-	wgDone func()
+	sr      storage.Search
+	mb      storage.MetricBlock
+	marshal marshalFunc
+	wgDone  func()
 }
 
 var blockIteratorsPool sync.Pool
@@ -228,10 +241,10 @@ func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	if !bi.sr.NextMetricBlock() {
 		return dst, false
 	}
-	mb := bi.mb
+	mb := &bi.mb
 	mb.MetricName = bi.sr.MetricBlockRef.MetricName
 	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	dst = mb.Marshal(dst[:0])
+	dst = bi.marshal(dst[:0], mb)
 	return dst, true
 }
 

app/vmstorage/vmstorage_with_tenant_id.go

@@ -0,0 +1,282 @@
+package vmstorage
+
+import (
+	"flag"
+	"fmt"
+	"math"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	accountID = flag.Uint64("accountID", 0, "The accountID of the stored data")
+	projectID = flag.Uint64("projectID", 0, "The projectID of the stored data")
+)
+
+func newVMStorageWithTenantID(vms *VMStorage) *VMStorageWithTenantID {
+	if *accountID > math.MaxUint32 {
+		logger.Fatalf("-accountID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *accountID)
+	}
+	if *projectID > math.MaxUint32 {
+		logger.Fatalf("-projectID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *projectID)
+	}
+	return &VMStorageWithTenantID{
+		vms:       vms,
+		accountID: uint32(*accountID),
+		projectID: uint32(*projectID),
+	}
+}
+
+// VMStorageWithTenantID is a thin wrapper around VMStorage type that overrides
+// its methods to properly serve requests coming from a vmselect (require
+// tenantID).
+//
+// A new instance of this type should be created using
+// newVMStorageWithTenantID(). The created instance does not require closing.
+// The instance also does not take ownership of vms and it is the responsibility
+// of the caller to close vms.
+type VMStorageWithTenantID struct {
+	vms *VMStorage
+
+	accountID uint32
+	projectID uint32
+}
+
+// InitSearch initializes a storage search for a request initiated by a
+// vmselect.
+//
+// The search is initialized only if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+// Otherwise, the method returns an interator that will return no data.
+//
+// The method also overrides the data format of the data returned by the
+// iterator by prepending accountID and projectID bytes to the metric name and
+// the data block (a format used in vmcluster).
+func (vmst *VMStorageWithTenantID) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return emptyBI, nil
+	}
+	return vmst.vms.initSearch(qt, sq, vmst.marshalMetricBlock, deadline)
+}
+
+var emptyBI = &emptyBlockIterator{}
+
+// emptyBlockIterator is an implementation of vmselectapi.BlockIterator that
+// always returns no data.
+type emptyBlockIterator struct{}
+
+func (*emptyBlockIterator) MustClose() {}
+
+func (*emptyBlockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	return dst, false
+}
+
+func (*emptyBlockIterator) Error() error {
+	return nil
+}
+
+// marshalMetricBlock serializes a metric block in the format expected by
+// vmselect.
+//
+// vmselect expects metric names and data blocks to have the tenantID but
+// vmsingle does not have it. Therefore the tenantID needs to be included to
+// every metric name and block.
+func (vmst *VMStorageWithTenantID) marshalMetricBlock(dst []byte, src *storage.MetricBlock) []byte {
+	// Marshal metric name:
+	// 1. Marshal metric name length + accountID length + projectID length (in
+	//    bytes).
+	// 2. append accountID and projectID bytes
+	// 3. Finally append metric name bytes
+	dst = encoding.MarshalVarUint64(dst, uint64(len(src.MetricName))+8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = append(dst, src.MetricName...)
+
+	// Marshal data block.
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = storage.MarshalBlock(dst, &src.Block)
+
+	return dst
+}
+
+// SearchMetricNames searches the storage for metric names that match the query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+//
+// Found metric names are prepended with accountID and projectID bytes (a format
+// used in vmcluster).
+func (vmst *VMStorageWithTenantID) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+
+	metricNames, err := vmst.vms.SearchMetricNames(qt, sq, deadline)
+	if err != nil {
+		return nil, err
+	}
+
+	// vmselect expects metric names to have the tenantID but vmsingle does not
+	// have it. Therefore the tenantID needs to be prepended to every metric
+	// name.
+	dst := make([]byte, 0, 8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	tenantID := string(dst)
+
+	for i, metricName := range metricNames {
+		metricNames[i] = tenantID + metricName
+	}
+	return metricNames, nil
+}
+
+// LabelValues searches the storage for label values that match the query and
+// correspond to a label whose name is `labelName`. The returned result
+// will contain not more than `maxLabelValues`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
+}
+
+// TagValueSuffixes searches the storage for Graphite tag value suffixes. The
+// returned result will contain not more than `maxSuffixes`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return nil, nil
+	}
+	return vmst.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+}
+
+// LabelNames searches the storage for label names that match the query.
+// The returned result will contain not more than `maxLabelNames`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelNames(qt, sq, maxLabelNames, deadline)
+}
+
+// SeriesCount returns the total number of metrics stored in the database.
+//
+// The method may return inflated numbers. How inflated the count depends
+// on the churn rate and the retention period. For example, if a metric lasts
+// for 2 months, it will be counted twice.
+//
+// The method also counts the deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return 0.
+func (vmst *VMStorageWithTenantID) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return 0, nil
+	}
+	return vmst.vms.SeriesCount(qt, accountID, projectID, deadline)
+}
+
+// Tenants returns just one tenant consisting of the -accountID and -projectID
+// flag values.
+func (vmst *VMStorageWithTenantID) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	tenantID := fmt.Sprintf("%d:%d", vmst.accountID, vmst.projectID)
+	return []string{tenantID}, nil
+}
+
+// TSDBStatus retrieves the status for metrics that match to the search query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return empty
+// status.
+func (vmst *VMStorageWithTenantID) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return &storage.TSDBStatus{}, nil
+	}
+	return vmst.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
+}
+
+// DeleteSeries marks as deleted metrics that match the search query.
+// The method returns the number of deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, no metrics will be deleted
+// and the method will return 0.
+func (vmst *VMStorageWithTenantID) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return 0, nil
+	}
+	return vmst.vms.DeleteSeries(qt, sq, deadline)
+}
+
+// RegisterMetricNames registers metric names in the index, the sample values
+// and timestamps are ignored.
+func (vmst *VMStorageWithTenantID) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
+	return vmst.vms.RegisterMetricNames(qt, mrs, deadline)
+}
+
+// GetMetricNamesUsageStats retrieves the usage stats for metrics whose name
+// matches the pattern.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return metricnamestats.StatsResult{}, nil
+	}
+	return vmst.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
+}
+
+// ResetMetricNamesUsageStats resets the metric name usage stats.
+func (vmst *VMStorageWithTenantID) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
+	return vmst.vms.ResetMetricNamesUsageStats(qt, deadline)
+}
+
+// GetMetadataRecords retrieves the metadata for the metricName.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return nil, nil
+	}
+	return vmst.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
+}
+
+// hasValidTenantID returns true if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) hasValidTenantID(sq *storage.SearchQuery) bool {
+	return sq.IsMultiTenant || vmst.isValidTenantID(sq.AccountID, sq.ProjectID)
+}
+
+// isValidTenantToken returns true if the TenantToken is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantToken(tt *storage.TenantToken) bool {
+	return tt == nil || vmst.isValidTenantID(tt.AccountID, tt.ProjectID)
+}
+
+// isValidTenantID returns true if the accountID and projectID match -accountID
+// and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantID(accountID, projectID uint32) bool {
+	return accountID == vmst.accountID && projectID == vmst.projectID
+}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.4 AS build-web-stage
+FROM golang:1.26.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.23.4
+FROM alpine:3.24.1
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.21",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.5",
-    "preact": "^10.29.2",
-    "qs": "^6.15.2",
+    "marked": "^18.0.6",
+    "preact": "^10.29.7",
+    "qs": "^6.15.3",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.17.0",
+    "react-router-dom": "^7.18.1",
     "uplot": "^1.6.32",
-    "vite": "^8.0.16",
+    "vite": "^8.1.4",
     "web-vitals": "^5.3.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.5",
+    "@eslint/eslintrc": "^3.3.6",
     "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.5",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.9.2",
+    "@types/node": "^26.1.1",
     "@types/qs": "^6.15.1",
     "@types/react": "^19.2.17",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.61.0",
-    "@typescript-eslint/parser": "^8.61.0",
+    "@typescript-eslint/eslint-plugin": "^8.64.0",
+    "@typescript-eslint/parser": "^8.64.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "globals": "^17.6.0",
-    "http-proxy-middleware": "^4.1.0",
+    "globals": "^17.7.0",
+    "http-proxy-middleware": "^4.2.0",
     "jsdom": "^29.1.1",
-    "postcss": "^8.5.15",
+    "postcss": "^8.5.19",
     "sass-embedded": "^1.100.0",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.10"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -183,7 +183,7 @@ const StepConfigurator: FC = () => {
           <div className="vm-step-control-popper-info">
             <p>
               <code>step</code> - the <Hyperlink
-                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
+                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations"
                 text="interval"
               /> between datapoints, which must be returned from the range query.
               The <code>query</code> is executed

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls.tsx

@@ -1,4 +1,4 @@
-import { FC, useEffect, useRef, useState } from "preact/compat";
+import { FC, useEffect, useRef } from "preact/compat";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { getAppModeEnable } from "../../../../utils/app-mode";
 import Button from "../../../Main/Button/Button";
@@ -9,27 +9,38 @@ import classNames from "classnames";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import useBoolean from "../../../../hooks/useBoolean";
+import { getMillisecondsFromDuration } from "../../../../utils/time";
+import { useSearchParams } from "react-router";
 
-interface AutoRefreshOption {
-  seconds: number
-  title: string
-}
-
-const delayOptions: AutoRefreshOption[] = [
-  { seconds: 0, title: "Off" },
-  { seconds: 1, title: "1s" },
-  { seconds: 2, title: "2s" },
-  { seconds: 5, title: "5s" },
-  { seconds: 10, title: "10s" },
-  { seconds: 30, title: "30s" },
-  { seconds: 60, title: "1m" },
-  { seconds: 300, title: "5m" },
-  { seconds: 900, title: "15m" },
-  { seconds: 1800, title: "30m" },
-  { seconds: 3600, title: "1h" },
-  { seconds: 7200, title: "2h" }
+const delayOptions = [
+  "Off",
+  "1s",
+  "2s",
+  "5s",
+  "10s",
+  "30s",
+  "1m",
+  "5m",
+  "15m",
+  "30m",
+  "1h",
+  "2h"
 ];
 
+const DEFAULT_OPTION = delayOptions[0];
+
+const MIN_REFRESH_MS = 1000;
+const MAX_REFRESH_MS = getMillisecondsFromDuration(delayOptions[delayOptions.length - 1]);
+const REFRESH_URL_PARAM = "refresh";
+
+const durationToMs = (dur: string | null) => {
+  return dur ? getMillisecondsFromDuration(dur) : 0;
+};
+
+const isValidDelay = (ms: number) => {
+  return ms >= MIN_REFRESH_MS && ms <= MAX_REFRESH_MS;
+};
+
 interface ExecutionControlsProps {
   tooltip: string;
   useAutorefresh?: boolean;
@@ -38,12 +49,14 @@ interface ExecutionControlsProps {
 
 export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAutorefresh, closeModal }) => {
   const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
 
   const dispatch = useTimeDispatch();
   const appModeEnable = getAppModeEnable();
-  const [autoRefresh, setAutoRefresh] = useState(false);
 
-  const [selectedDelay, setSelectedDelay] = useState<AutoRefreshOption>(delayOptions[0]);
+  const rawDelay = searchParams.get(REFRESH_URL_PARAM);
+  const msDelay = durationToMs(rawDelay);
+  const selectedDelay = isValidDelay(msDelay) ? rawDelay : DEFAULT_OPTION;
 
   const {
     value: openOptions,
@@ -52,11 +65,20 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   } = useBoolean(false);
   const optionsButtonRef = useRef<HTMLDivElement>(null);
 
-  const handleChange = (d: AutoRefreshOption) => {
-    if ((autoRefresh && !d.seconds) || (!autoRefresh && d.seconds)) {
-      setAutoRefresh(prev => !prev);
-    }
-    setSelectedDelay(d);
+  const handleChange = (dur: string) => () => {
+    setSearchParams(prev => {
+      const nextParams = new URLSearchParams(prev);
+      const ms = durationToMs(dur);
+
+      if (ms) {
+        nextParams.set(REFRESH_URL_PARAM, `${dur}`);
+      } else {
+        nextParams.delete(REFRESH_URL_PARAM);
+      }
+
+      return nextParams;
+    });
+
     handleCloseOptions();
   };
 
@@ -68,23 +90,19 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   };
 
   useEffect(() => {
-    const delay = selectedDelay.seconds;
+    const ms = durationToMs(selectedDelay);
     let timer: number;
-    if (autoRefresh) {
+
+    if (useAutorefresh && isValidDelay(ms)) {
       timer = setInterval(() => {
         dispatch({ type: "RUN_QUERY" });
-      }, delay * 1000) as unknown as number;
-    } else {
-      setSelectedDelay(delayOptions[0]);
+      }, ms) as unknown as number;
     }
-    return () => {
-      timer && clearInterval(timer);
-    };
-  }, [selectedDelay, autoRefresh]);
 
-  const createHandlerChange = (d: AutoRefreshOption) => () => {
-    handleChange(d);
-  };
+    return () => {
+      clearInterval(timer);
+    };
+  }, [selectedDelay, useAutorefresh]);
 
   return (
     <>
@@ -106,7 +124,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 <span className="vm-mobile-option__icon"><RestartIcon/></span>
                 <div className="vm-mobile-option-text">
                   <span className="vm-mobile-option-text__label">Auto-refresh</span>
-                  <span className="vm-mobile-option-text__value">{selectedDelay.title}</span>
+                  <span className="vm-mobile-option-text__value">{selectedDelay}</span>
                 </div>
                 <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
               </div>
@@ -139,7 +157,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                       )}
                       onClick={toggleOpenOptions}
                     >
-                      {selectedDelay.title}
+                      {selectedDelay}
                     </Button>
                   </div>
                 </Tooltip>
@@ -187,12 +205,12 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 className={classNames({
                   "vm-list-item": true,
                   "vm-list-item_mobile": isMobile,
-                  "vm-list-item_active": d.seconds === selectedDelay.seconds
+                  "vm-list-item_active": d === selectedDelay
                 })}
-                key={d.seconds}
-                onClick={createHandlerChange(d)}
+                key={d}
+                onClick={handleChange(d)}
               >
-                {d.title}
+                {d}
               </div>
             ))}
           </div>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseAlert/index.tsx

@@ -98,7 +98,7 @@ const BaseAlert = ({ item, group }: BaseAlertProps) => {
       {!!Object.keys(item.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -121,7 +121,7 @@ const BaseRule = ({ item, group }: BaseRuleProps) => {
       {!!Object.keys(item?.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/HeaderNav.tsx

@@ -17,10 +17,12 @@ interface HeaderNavProps {
 const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
   const { pathname } = useLocation();
   const [activeMenu, setActiveMenu] = useState(pathname);
+  const [openMenu, setOpenMenu] = useState<string | null>(null);
   const menu = useNavigationMenu();
 
   useEffect(() => {
     setActiveMenu(pathname);
+    setOpenMenu(null);
   }, [pathname]);
 
   return (
@@ -41,6 +43,8 @@ const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
               color={color}
               background={background}
               direction={direction}
+              openMenu={openMenu}
+              setOpenMenu={setOpenMenu}
             />
           )
           : (

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavSubItem.tsx

@@ -1,11 +1,8 @@
-import { FC, useRef, useState } from "preact/compat";
-import { useLocation } from "react-router-dom";
+import { FC, useRef, useState, Dispatch, SetStateAction } from "preact/compat";
 import classNames from "classnames";
 import { ArrowDropDownIcon } from "../../../components/Main/Icons";
 import Popper from "../../../components/Main/Popper/Popper";
 import NavItem from "./NavItem";
-import { useEffect } from "react";
-import useBoolean from "../../../hooks/useBoolean";
 import { NavigationItem, NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
@@ -15,6 +12,8 @@ interface NavItemProps {
   color?: string
   background?: string
   direction?: "row" | "column"
+  openMenu: string | null,
+  setOpenMenu: Dispatch<SetStateAction<string | null>>,
 }
 
 const NavSubItem: FC<NavItemProps> = ({
@@ -23,21 +22,18 @@ const NavSubItem: FC<NavItemProps> = ({
   color,
   background,
   submenu,
-  direction = "row"
+  direction = "row",
+  openMenu,
+  setOpenMenu,
 }) => {
-  const { pathname } = useLocation();
-
   const [menuTimeout, setMenuTimeout] = useState<NodeJS.Timeout | null>(null);
   const buttonRef = useRef<HTMLDivElement>(null);
 
-  const {
-    value: openSubmenu,
-    setFalse: handleCloseSubmenu,
-    setTrue: setOpenSubmenu,
-  } = useBoolean(false);
+  const openSubmenu = openMenu === label;
+  const handleCloseSubmenu = () => setOpenMenu(prev => (prev === label ? null : prev));
 
   const handleOpenSubmenu = () => {
-    if (direction === "row" || !openSubmenu) setOpenSubmenu();
+    if (direction === "row" || !openSubmenu) setOpenMenu(label);
     if (direction === "column" && openSubmenu) handleCloseSubmenu();
     if (direction === "row" && menuTimeout) clearTimeout(menuTimeout);
   };
@@ -52,10 +48,6 @@ const NavSubItem: FC<NavItemProps> = ({
     if (menuTimeout) clearTimeout(menuTimeout);
   };
 
-  useEffect(() => {
-    handleCloseSubmenu();
-  }, [pathname]);
-
   return (
     <div
       className={classNames({

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const match = searchParams.get("match");
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
-
+  const showMetricNameStats = !(match || focusLabel);
   const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
     },
   ].filter(t => t.display);
 
-  if (!totals.length) {
+  if (!totals.length && !showMetricNameStats) {
     return null;
   }
 
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           <h4 className="vm-cardinality-totals-card__title">
             {info && (
               <Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
-                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
+                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
               </Tooltip>
             )}
             {title}
@@ -99,7 +99,10 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           )}
         </div>
       ))}
-      <CardinalityMetricNameStats metricNameStats={metricNameStats}/>
+      {
+        showMetricNameStats &&
+        <CardinalityMetricNameStats metricNameStats={metricNameStats} />
+      }
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/style.scss

@@ -62,6 +62,13 @@
     padding-right: 40px;
   }
 
+  .vm-annotations-table {
+    tbody > tr > td {
+      vertical-align: top;
+      white-space: pre-line;
+    }
+  }
+
   display: flex;
   flex-direction: column;
   align-items: stretch;

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Relabel configs"
+            label="Config"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,8 +82,9 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="Labels"
+            label="A time series"
             value={labels}
+            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -97,22 +98,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
-              Documentation
+              Relabeling Stages
             </Button>
           </a>
           <Button

app/vmui/packages/vmui/src/utils/time.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import dayjs from "dayjs";
-import { getNanoTimestamp, parseSupportedDuration } from "./time";
+import { getMillisecondsFromDuration, getNanoTimestamp, parseSupportedDuration } from "./time";
 
 describe("Time utils", () => {
   describe("getNanoTimestamp", () => {
@@ -82,4 +82,19 @@ describe("Time utils", () => {
       expect(parseSupportedDuration("   ")).toBeUndefined();
     });
   });
+
+  describe("getMillisecondsFromDuration", () => {
+    it("should convert valid durations to milliseconds", () => {
+      expect(getMillisecondsFromDuration("1s")).toBe(1000);
+      expect(getMillisecondsFromDuration("1.5s")).toBe(1500);
+      expect(getMillisecondsFromDuration("1m30s")).toBe(90000);
+      expect(getMillisecondsFromDuration("1h 30m")).toBe(5400000);
+      expect(getMillisecondsFromDuration("500ms")).toBe(500);
+    });
+
+    it("should return zero when duration cannot be parsed", () => {
+      expect(getMillisecondsFromDuration("garbage")).toBe(0);
+      expect(getMillisecondsFromDuration("")).toBe(0);
+    });
+  });
 });

app/vmui/packages/vmui/src/utils/time.ts

@@ -16,7 +16,7 @@ const supportedValuesOf = Intl.supportedValuesOf;
 export const supportedTimezones = supportedValuesOf ? supportedValuesOf("timeZone") as string[] : timezones;
 
 // The list of supported units could be the following -
-// https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations
+// https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations
 export const supportedDurations = [
   { long: "years", short: "y", possible: "year" },
   { long: "weeks", short: "w", possible: "week" },
@@ -100,6 +100,10 @@ export const getSecondsFromDuration = (dur: string) => {
   return dayjs.duration(durObject).asSeconds();
 };
 
+export const getMillisecondsFromDuration = (dur: string): number => {
+  return getSecondsFromDuration(dur) * 1000;
+};
+
 const instantQueryViews = [DisplayType.table, DisplayType.code];
 export const getStepFromDuration = (dur: number, histogram?: boolean, displayType?: DisplayType): string => {
   if (displayType && instantQueryViews.includes(displayType)) return roundStep(dur);
@@ -276,4 +280,3 @@ export const getNanoTimestamp = (dateStr: string): bigint => {
   // Return the full timestamp in nanoseconds as a BigInt
   return BigInt(baseMs) * 1000000n + BigInt(extraNano);
 };
-

apptest/README.md

@@ -4,7 +4,7 @@ The `apptest` package contains the integration tests for the VictoriaMetrics
 applications (such as vmstorage, vminsert, and vmselect).
 
 An integration test aims at verifying the behavior of an application as a whole,
-as apposed to a unit test that verifies the behavior of a building block of an
+as opposed to a unit test that verifies the behavior of a building block of an
 application.
 
 To achieve that an integration test starts an application in a separate process
@@ -19,10 +19,10 @@ work together as a system.
 The package provides a collection of helpers to start applications and make
 queries to them:
 
--   `app.go` - contains the generic code for staring an application and should
+-   `app.go` - contains the generic code for starting an application and should
     not be used by integration tests directly.
 -   `{vmstorage,vminsert,etc}.go` - build on top of `app.go` and provide the
-    code for staring a specific application.
+    code for starting a specific application.
 -   `client.go` - provides helper functions for sending HTTP requests to
     applications.
 
@@ -36,7 +36,7 @@ the application binary files to be built and put into the `bin` directory. The
 build rule used for running integration tests, `make apptest`,
 accounts for that, it builds all application binaries before running the tests.
 But if you want to run the tests without `make`, i.e. by executing
-`go test ./app/apptest`, you will need to build the binaries first (for example,
+`go test ./apptest/tests`, you will need to build the binaries first (for example,
 by executing `make all`).
 
 Not all binaries can be built from `master` branch, cluster binaries can be built

apptest/testdata.go

@@ -0,0 +1,358 @@
+package apptest
+
+import (
+	"fmt"
+	"slices"
+)
+
+type TestData struct {
+	Samples              []string
+	Step                 int64
+	WantSeries           []map[string]string
+	WantLabels           []string
+	WantLabelValues      []string
+	WantQueryResults     []*QueryResult
+	WantMetadata         map[string][]MetadataEntry
+	WantMetricNamesStats []MetricNamesStatsRecord
+}
+
+func GenerateTestData(prefix string, numMetrics, start, end int64) TestData {
+	d := TestData{
+		Samples:              []string{},
+		Step:                 (end - start) / numMetrics,
+		WantSeries:           make([]map[string]string, numMetrics),
+		WantLabels:           make([]string, numMetrics),
+		WantLabelValues:      make([]string, numMetrics),
+		WantQueryResults:     make([]*QueryResult, numMetrics),
+		WantMetadata:         make(map[string][]MetadataEntry),
+		WantMetricNamesStats: make([]MetricNamesStatsRecord, numMetrics),
+	}
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("%s_%04d", prefix, i)
+		metricHelp := fmt.Sprintf("# HELP %s some help message", metricName)
+		metricType := fmt.Sprintf("# TYPE %s gauge", metricName)
+		labelName := fmt.Sprintf("label_%04d", i)
+		labelValue := fmt.Sprintf("value_%04d", i)
+		value := i
+		timestamp := start + i*d.Step
+		sample := fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+
+		d.Samples = append(d.Samples, metricHelp, metricType, sample)
+		d.WantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		d.WantLabels[i] = labelName
+		d.WantLabelValues[i] = labelValue
+		d.WantQueryResults[i] = &QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+		d.WantMetadata[metricName] = []MetadataEntry{{Help: "some help message", Type: "gauge"}}
+		d.WantMetricNamesStats[i].MetricName = metricName
+	}
+	d.WantLabels = append(d.WantLabels, "__name__", "label")
+	slices.Sort(d.WantLabels)
+	return d
+}
+
+// AssertSeries retrieves metric names from the storage and compares the result
+// with the expected one.
+func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []map[string]string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Series(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			}).Sort()
+		},
+		Want: &PrometheusAPIV1SeriesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		Retries: 1000,
+		FailNow: true,
+	})
+}
+
+// AssertSeriesCount retrieves series count and compares it with expected one.
+func AssertSeriesCount(tc *TestCase, app PrometheusQuerier, tenantID string, start, end int64, want uint64) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series/count response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1SeriesCount(tc.T(), QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+		},
+		Want: &PrometheusAPIV1SeriesCountResponse{
+			Status: "success",
+			Data:   []uint64{want},
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabels retrieves label names from the storage and compares the result
+// with the expected one.
+func AssertLabels(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1Labels(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelsResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabelValues retrieves values for the label whose name is labelName for
+// the series whose name mathes metricNameRE, compares the result with the
+// expected one.
+func AssertLabelValues(tc *TestCase, app PrometheusQuerier, metricNameRE, labelName, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels/.../values response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1LabelValues(tc.T(), labelName, query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelValuesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertQueryResults sends a data query to storage and compares the query
+// result with the expected one.
+func AssertQueryResults(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end, step int64, want []*QueryResult) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/query_range response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1QueryRange(tc.T(), query, QueryOpts{
+				Tenant:      tenantID,
+				Start:       fmt.Sprintf("%d", start),
+				End:         fmt.Sprintf("%d", end),
+				Step:        fmt.Sprintf("%dms", step),
+				MaxLookback: fmt.Sprintf("%dms", step-1),
+				NoCache:     "1",
+			})
+		},
+		Want: &PrometheusAPIV1QueryResponse{
+			Status: "success",
+			Data: &QueryData{
+				ResultType: "matrix",
+				Result:     want,
+			},
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetadata(tc *TestCase, app PrometheusQuerier, metricName, tenantID string, want map[string][]MetadataEntry) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/metadata response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Metadata(tc.T(), metricName, 0, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: &PrometheusAPIV1Metadata{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetricNamesStats(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, want []MetricNamesStatsRecord) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/status/metric_names_stats response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1StatusMetricNamesStats(tc.T(), "", "", metricNameRE, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: MetricNamesStatsResponse{
+			Records: want,
+		},
+		FailNow: true,
+	})
+
+}
+
+// GraphiteTestData holds the data samples in Graphite Pickle format, distance
+// between samples in milliseconds and expected responses for various Graphite
+// API endpoints.
+type GraphiteTestData struct {
+	Samples             []string
+	Step                int64
+	WantMetricsIndex    []string
+	WantMetricsFind     []GraphiteMetric
+	WantMetricsExpand   []string
+	WantRenderedTargets []GraphiteRenderedTarget
+}
+
+// GenerateGraphiteTestData generates Graphite test data.
+func GenerateGraphiteTestData(prefix string, numMetrics, start, end int64) GraphiteTestData {
+	d := GraphiteTestData{
+		Samples:             make([]string, numMetrics),
+		Step:                (end - start) / numMetrics,
+		WantMetricsIndex:    make([]string, numMetrics),
+		WantMetricsFind:     make([]GraphiteMetric, numMetrics),
+		WantMetricsExpand:   make([]string, numMetrics),
+		WantRenderedTargets: make([]GraphiteRenderedTarget, numMetrics),
+	}
+
+	datapoints := make([][2]float64, numMetrics)
+	for i := range numMetrics {
+		timestamp := (start + i*d.Step) / 1000
+		datapoints[i][1] = float64(timestamp)
+	}
+
+	for i := range numMetrics {
+		suffix := fmt.Sprintf("%04d", i)
+		metricName := fmt.Sprintf("%s.%s", prefix, suffix)
+		value := i
+		timestamp := (start + i*d.Step) / 1000
+		sample := fmt.Sprintf(`%s %d %d`, metricName, value, timestamp)
+
+		d.Samples[i] = sample
+		d.WantMetricsIndex[i] = metricName
+		d.WantMetricsFind[i].Id = metricName
+		d.WantMetricsFind[i].Text = suffix
+		d.WantMetricsFind[i].Leaf = 1
+		d.WantMetricsExpand[i] = metricName
+		d.WantRenderedTargets[i].Target = metricName
+		d.WantRenderedTargets[i].Datapoints = slices.Clone(datapoints)
+		d.WantRenderedTargets[i].Datapoints[i][0] = float64(value)
+	}
+	return d
+}
+
+// AssertGraphiteMetricsIndex retrieves all metrics by sending a request to
+// /graphite/metrics/index.json and compares the result with the expected one.
+func AssertGraphiteMetricsIndex(tc *TestCase, app PrometheusQuerier, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/index.json response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsIndex(tc.T(), QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		Retries: 30,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind finds metric names by sending a request to
+// /graphite/metrics/find and compares the result with the expected one.
+func AssertGraphiteMetricsFind(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []GraphiteMetric) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/find response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsFind(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind expands metric names by sending a request to
+// /graphite/metrics/expand and compares the result with the expected one.
+func AssertGraphiteMetricsExpand(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/expand response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsExpand(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteRender retieves metric raw data by sending a request to
+// /graphite/render and compares the result with the expected one.
+func AssertGraphiteRender(tc *TestCase, app PrometheusQuerier, target, tenantID string, from, until, step int64, want []GraphiteRenderedTarget) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/render response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteRender(tc.T(), target, QueryOpts{
+				Tenant:      tenantID,
+				From:        fmt.Sprintf("%d", from/1000),
+				Until:       fmt.Sprintf("%d", until/1000),
+				StorageStep: fmt.Sprintf("%dms", step),
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+}

apptest/tests/max_backfill_age_test.go

@@ -0,0 +1,222 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+				"-retentionPeriod=" + retentionPeriod,
+				"-maxBackfillAge=" + maxBackfillAge,
+			})
+		},
+		stop: func() {
+			tc.StopApp("vmsingle")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+func TestClusterMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&apptest.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage1"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage2"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags:    []string{},
+			})
+		},
+		stop: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+type maxBackfillAgeOpts struct {
+	start func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier
+	stop  func()
+}
+
+func testMaxBackfillAge(tc *apptest.TestCase, opts maxBackfillAgeOpts) {
+	t := tc.T()
+
+	assertSeries := func(app apptest.PrometheusQuerier, prefix string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &apptest.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	assertQueryResults := func(app apptest.PrometheusQuerier, prefix string, start, end, step int64, want []*apptest.QueryResult) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
+				})
+			},
+			Want: &apptest.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &apptest.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	const numMetrics = 1000
+	now := time.Now().UTC()
+	var start, end, step int64
+	emptySeries := []map[string]string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+
+	// Start sut with the same -retentionPeriod and -maxBackfillAge.
+	sut := opts.start("1y", "1y")
+
+	// Verify that samples older than the retention period are rejected.
+	start = now.Add(-365 * 24 * time.Hour).Add(-time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	outsideRetention := genMaxBackfillAgeData("outside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, outsideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "outside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "outside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that samples within the retention period are accepted and
+	// searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention := genMaxBackfillAgeData("inside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Restart sut with -maxBackfillAge shorter than the -retentionPeriod.
+	opts.stop()
+	sut = opts.start("1y", "6M")
+
+	// Verify that new samples older than max backfill age but still within the
+	// retention period are rejected but existing samples are still searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention2 := genMaxBackfillAgeData("inside_retention2", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention2.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention2", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention2", start, end, step, emptyQueryResults)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Verify that the metrics that are outside the backfill window can still
+	// be deleted.
+	sut.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*inside_retention.*"}`, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that the samples that are within the backfill window are accepted
+	// and searchable.
+	start = now.Add(-180 * 24 * time.Hour).UnixMilli()
+	end = now.Add(-180 * 24 * time.Hour).Add(1 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideMaxBackfillAge := genMaxBackfillAgeData("inside_max_backfill_age", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideMaxBackfillAge.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_max_backfill_age", start, end, insideMaxBackfillAge.wantSeries)
+	assertQueryResults(sut, "inside_max_backfill_age", start, end, step, insideMaxBackfillAge.wantQueryResults)
+
+	opts.stop()
+}
+
+type maxBackfillAgeData struct {
+	samples          []string
+	wantSeries       []map[string]string
+	wantQueryResults []*apptest.QueryResult
+}
+
+func genMaxBackfillAgeData(prefix string, numMetrics, start, step int64) maxBackfillAgeData {
+	samples := make([]string, numMetrics)
+	wantSeries := make([]map[string]string, numMetrics)
+	wantQueryResults := make([]*apptest.QueryResult, numMetrics)
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("metric_%s_%04d", prefix, i)
+		labelName := fmt.Sprintf("label_%s_%04d", prefix, i)
+		labelValue := fmt.Sprintf("value_%s_%04d", prefix, i)
+		value := i
+		timestamp := start + i*step
+		samples[i] = fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+		wantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		wantQueryResults[i] = &apptest.QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*apptest.Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+	}
+	return maxBackfillAgeData{samples, wantSeries, wantQueryResults}
+}

apptest/tests/metricsmetadata_test.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -25,7 +26,11 @@ func TestSingleMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
-
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
 		`# HELP metric_name_1 some help message`,
@@ -40,16 +45,27 @@ func TestSingleMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
+
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_5"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: `metric_name_7_!@"_suffix`}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 		},
 		Metadata: []prompb.MetricMetadata{
 			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 
@@ -59,12 +75,13 @@ func TestSingleMetricsMetadata(t *testing.T) {
 	expected := &apptest.PrometheusAPIV1Metadata{
 		Status: "success",
 		Data: map[string][]apptest.MetadataEntry{
-			"metric_name_1": {{Help: "some help message", Type: "gauge"}},
-			"metric_name_2": {{Help: "some help message", Type: "counter"}},
-			"metric_name_3": {{Help: "some help message", Type: "gauge"}},
-			"metric_name_4": {{Help: "some help message", Type: "summary"}},
-			"metric_name_5": {{Help: "some help message", Type: "summary"}},
-			"metric_name_6": {{Help: "some help message", Type: "stateset"}},
+			"metric_name_1":            {{Help: "some help message", Type: "gauge"}},
+			"metric_name_2":            {{Help: "some help message", Type: "counter"}},
+			"metric_name_3":            {{Help: "some help message", Type: "gauge"}},
+			"metric_name_4":            {{Help: "some help message", Type: "summary"}},
+			"metric_name_5":            {{Help: "some help message", Type: "summary"}},
+			"metric_name_6":            {{Help: "some help message", Type: "stateset"}},
+			`metric_name_7_!@"_suffix`: {{Help: "some help message", Type: "stateset"}},
 		},
 	}
 	gotStats := sut.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{})
@@ -134,6 +151,11 @@ func TestClusterMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
@@ -149,16 +171,26 @@ func TestClusterMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_5"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: `metric_name_7_!@"_suffix`}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 		},
 		Metadata: []prompb.MetricMetadata{
 			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 
@@ -171,12 +203,13 @@ func TestClusterMetricsMetadata(t *testing.T) {
 		expected := &apptest.PrometheusAPIV1Metadata{
 			Status: "success",
 			Data: map[string][]apptest.MetadataEntry{
-				"metric_name_1": {{Help: "some help message", Type: "gauge"}},
-				"metric_name_2": {{Help: "some help message", Type: "counter"}},
-				"metric_name_3": {{Help: "some help message", Type: "gauge"}},
-				"metric_name_4": {{Help: "some help message", Type: "summary"}},
-				"metric_name_5": {{Help: "some help message", Type: "summary"}},
-				"metric_name_6": {{Help: "some help message", Type: "stateset"}},
+				"metric_name_1":            {{Help: "some help message", Type: "gauge"}},
+				"metric_name_2":            {{Help: "some help message", Type: "counter"}},
+				"metric_name_3":            {{Help: "some help message", Type: "gauge"}},
+				"metric_name_4":            {{Help: "some help message", Type: "summary"}},
+				"metric_name_5":            {{Help: "some help message", Type: "summary"}},
+				"metric_name_6":            {{Help: "some help message", Type: "stateset"}},
+				`metric_name_7_!@"_suffix`: {{Help: "some help message", Type: "stateset"}},
 			},
 		}
 		gotStats := vmselect.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{Tenant: tenantID})