fix code review remarks

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -22,8 +22,7 @@ on:
       - '!app/vmui/**'
       - '.github/workflows/build.yml'
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -32,6 +31,8 @@ concurrency:
 jobs:
   build:
     name: ${{ matrix.os }}-${{ matrix.arch }}
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -63,11 +64,11 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/changelog-linter.yml

@@ -5,11 +5,15 @@ on:
     paths:
       - "docs/victoriametrics/changelog/CHANGELOG.md"
 
+permissions: {}
+
 jobs:
   tip-lint:
+    permissions:
+      contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: 'actions/checkout@v6'
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -3,12 +3,16 @@ name: check-commit-signed
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
   check-commit-signed:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/check-licenses.yml

@@ -6,20 +6,22 @@ on:
   pull_request:
     paths:
       - 'vendor'
-permissions:
-  contents: read
+
+permissions: {}
 
 jobs:
   build:
     name: Build
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           go-version-file: 'go.mod'
           cache: false
@@ -27,7 +29,7 @@ jobs:
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-go.yml

@@ -18,6 +18,8 @@ concurrency:
   cancel-in-progress: true
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
 
+permissions: {}
+
 jobs:
   analyze:
     name: Analyze
@@ -29,18 +31,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache: false
           go-version-file: 'go.mod'
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build
@@ -50,14 +52,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.35.2
+        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.35.2
+        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.35.2
+        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -7,28 +7,30 @@ on:
       - 'docs/**'
       - '.github/workflows/docs.yaml'
   workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
+
+permissions: {}
+
 jobs:
   build:
     name: Build
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
 
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v7
+        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0
         id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}

.github/workflows/test.yml

@@ -18,8 +18,7 @@ on:
       - 'go.*'
       - '.github/workflows/main.yml'
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -29,14 +28,16 @@ concurrency:
 jobs:
   lint:
     name: lint
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -47,7 +48,7 @@ jobs:
       - run: go version
 
       - name: Cache golangci-lint
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/golangci-lint
@@ -61,6 +62,8 @@ jobs:
 
   unit:
     name: unit
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
 
     strategy:
@@ -72,11 +75,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -90,15 +93,17 @@ jobs:
 
   apptest:
     name: apptest
+    permissions:
+      contents: read
     runs-on: apptest
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/vmui.yml

@@ -16,11 +16,7 @@ on:
       - 'app/vmui/packages/vmui/**'
       - '.github/workflows/vmui.yml'
 
-permissions:
-  contents: read
-  packages: read
-  pull-requests: read
-  checks: write
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -29,14 +25,18 @@ concurrency:
 jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
+    permissions:
+      checks: write
+      contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Cache node_modules
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: app/vmui/packages/vmui/node_modules
           key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
@@ -69,7 +69,7 @@ jobs:
           VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
+        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           report-json: app/vmui/packages/vmui/vmui-lint-report.json

Makefile

@@ -485,8 +485,8 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
-	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
-	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	VMSINGLE_V1_132_0_PATH=$${DIR}/victoria-metrics-prod \
+	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
@@ -535,6 +535,15 @@ remove-golangci-lint:
 govulncheck: install-govulncheck
 	govulncheck ./...
 
+govulncheck-docker:
+	docker run -w $(PWD) -v $(PWD):$(PWD) \
+		-v govulncheck-gomod-cache:/root/go/pkg/mod \
+		-v govulncheck-gobuild-cache:/root/.cache/go-build \
+		-v govulncheck-go-bin:/root/go/bin \
+		--env="GOCACHE=/root/.cache/go-build" \
+		--env="GOMODCACHE=/root/go/pkg/mod" \
+		"$(GO_BUILDER_IMAGE)" /bin/sh -c "which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./..."
+
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 

app/victoria-metrics/main.go

@@ -22,7 +22,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 var (
@@ -30,23 +29,26 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
 	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate.")
-	finalDedupScheduleInterval = flag.Duration("storage.finalDedupScheduleCheckInterval", time.Hour, "The interval for checking when final deduplication process should be started."+
-		"Storage unconditionally adds 25% jitter to the interval value on each check evaluation."+
-		" Changing the interval to the bigger values may delay downsampling, deduplication for historical data."+
-		" See also https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
+	vmselectMaxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
+		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
+	vmselectMaxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
+		"limit is reached; see also -search.maxQueryDuration")
 )
 
+func getDefaultMaxConcurrentRequests() int {
+	// A single request can saturate all the CPU cores, so there is no sense
+	// in allowing higher number of concurrent requests - they will just contend
+	// for unavailable CPU time.
+	n := min(cgroup.AvailableCPUs()*2, 16)
+	return n
+}
+
 func main() {
 	// VictoriaMetrics is optimized for reduced memory allocations,
 	// so it can run with the reduced GOGC in order to reduce the used memory,
@@ -87,14 +89,8 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
-	storage.SetDedupInterval(*minScrapeInterval)
-	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
-	if *finalDedupScheduleInterval < time.Hour {
-		logger.Fatalf("-dedup.finalDedupScheduleCheckInterval cannot be smaller than 1 hour; got %s", *finalDedupScheduleInterval)
-	}
-	storage.SetFinalDedupScheduleInterval(*finalDedupScheduleInterval)
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	vmselect.Init()
+	vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()
 

app/victoria-metrics/self_scraper.go

@@ -93,7 +93,7 @@ func selfScraper(scrapeInterval time.Duration) {
 				mr.Value = r.Value
 			}
 		}
-		if err := vmstorage.AddRows(mrs); err != nil {
+		if err := vmstorage.VMInsertAPI.WriteRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 		if len(metadataRows.Rows) > 0 {
@@ -105,7 +105,7 @@ func selfScraper(scrapeInterval time.Duration) {
 					Type:             mm.Type,
 				})
 			}
-			if err := vmstorage.AddMetadataRows(mms); err != nil {
+			if err := vmstorage.VMInsertAPI.WriteMetadata(mms); err != nil {
 				logger.Errorf("cannot store self-scraped metrics metadata: %s", err)
 			}
 		}

app/vmagent/main.go

@@ -118,6 +118,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	opentelemetry.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {

app/vmagent/opentelemetry/request_handler.go

@@ -25,6 +25,11 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {

app/vmagent/remotewrite/client.go

@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -310,11 +311,6 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
-			continue
-		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)
@@ -330,15 +326,20 @@ func (c *client) runWorker() {
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			return
 		case <-c.stopCh:
-			// c must be stopped. Wait for a while in the hope the block will be sent.
-			graceDuration := 5 * time.Second
+			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
+			// If it succeeds, drain the remaining in-memory queue before returning.
+			stopCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
 			select {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
 					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				} else {
+					c.drainInMemoryQueue(stopCtx, block[:0])
 				}
-			case <-time.After(graceDuration):
+			case <-stopCtx.Done():
 				// Return unsent block to the queue.
 				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			}
@@ -508,6 +509,32 @@ again:
 	goto again
 }
 
+func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
+	var ok bool
+	for {
+		select {
+		case <-stopCtx.Done():
+			return
+		default:
+		}
+
+		block, ok = c.fq.MustReadInMemoryBlock(block[:0])
+		if !ok {
+			// The in memory queue has already been drained,
+			// or persisted queue is being used.
+			// In this case it is guaranteed that fq will be empty
+			return
+		}
+
+		// at this stage c.stopCh should be closed
+		// so sendBlock function should not perform retries
+		if ok := c.sendBlock(block); !ok {
+			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			return
+		}
+	}
+}
+
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 

app/vmagent/remotewrite/client_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestParseRetryAfterHeader(t *testing.T) {
@@ -36,6 +37,40 @@ func TestParseRetryAfterHeader(t *testing.T) {
 	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
 }
 
+func TestInitSecretFlags(t *testing.T) {
+	showRemoteWriteURLOrig := *showRemoteWriteURL
+	defer func() {
+		*showRemoteWriteURL = showRemoteWriteURLOrig
+		flagutil.UnregisterAllSecretFlags()
+	}()
+
+	flagutil.UnregisterAllSecretFlags()
+	*showRemoteWriteURL = false
+	InitSecretFlags()
+	if !flagutil.IsSecretFlag("remotewrite.url") {
+		t.Fatalf("expecting remoteWrite.url to be secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.headers") {
+		t.Fatalf("expecting remoteWrite.headers to be secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
+		t.Fatalf("expecting remoteWrite.proxyURL to be secret")
+	}
+
+	flagutil.UnregisterAllSecretFlags()
+	*showRemoteWriteURL = true
+	InitSecretFlags()
+	if flagutil.IsSecretFlag("remotewrite.url") {
+		t.Fatalf("remoteWrite.url must remain visible when -remoteWrite.showURL is set")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.headers") {
+		t.Fatalf("expecting remoteWrite.headers to remain secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
+		t.Fatalf("expecting remoteWrite.proxyURL to remain secret")
+	}
+}
+
 func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 

app/vmagent/remotewrite/remotewrite.go

@@ -79,7 +79,8 @@ var (
 		"writing them to remote storage. "+
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
 		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
-		"This option may be used for improving data compression for the stored metrics")
+		"This option may be used for improving data compression for the stored metrics. "+
+		"See also -remoteWrite.significantFigures")
 	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
@@ -151,6 +152,10 @@ func InitSecretFlags() {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
+	// remoteWrite.proxyURL can contain authentication codes.
+	flagutil.RegisterSecretFlag("remoteWrite.proxyURL")
+	// remoteWrite.headers can contain auth headers such as Authorization and API keys.
+	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 var (
@@ -167,6 +172,18 @@ func Init() {
 	if len(*remoteWriteURLs) == 0 {
 		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
+	if *shardByURL && len(*disableOnDiskQueue) > 1 {
+		disableOnDiskQueues := *disableOnDiskQueue
+
+		firstValue := disableOnDiskQueues[0]
+		for _, v := range disableOnDiskQueues[1:] {
+			if firstValue != v {
+				logger.Fatalf("all -remoteWrite.url targets must have the same -remoteWrite.disableOnDiskQueue setting when -remoteWrite.shardByURL is enabled; " +
+					"either enable or disable -remoteWrite.disableOnDiskQueue for all targets")
+			}
+		}
+	}
+
 	if limit := getMaxHourlySeries(); limit > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
@@ -499,7 +516,9 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 //
 // calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	if !disableOnDiskQueueAny {
+	// When -remoteWrite.shardByURL=true always use all configured remote writes to preserve stable metrics distribution across shards.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507
+	if !disableOnDiskQueueAny || *shardByURL {
 		return rwctxsGlobal, true
 	}
 
@@ -514,12 +533,6 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
-			if *shardByURL {
-				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
-				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
-				// data to all rwctxs evenly.
-				rowsCount = rowsCount / len(rwctxsGlobal)
-			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}

app/vmalert-tool/unittest/input.go

@@ -52,7 +52,7 @@ func writeInputSeries(input []series, interval *promutil.Duration, startStamp ti
 	data := testutil.Compress(r)
 	// write input series to vm
 	httpWrite(dst, bytes.NewBuffer(data))
-	vmstorage.Storage.DebugFlush()
+	vmstorage.DebugFlush()
 	return nil
 }
 

app/vmalert-tool/unittest/unittest.go

@@ -61,7 +61,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	}
 	eu, err := url.Parse(externalURL)
 	if err != nil {
-		logger.Fatalf("failed to parse external URL: %w", err)
+		logger.Fatalf("failed to parse external URL: %s", err)
 	}
 	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
@@ -108,7 +108,9 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	storagePath = tmpFolder
 	processFlags()
 	vminsert.Init()
-	vmselect.Init()
+	const maxConcurrentRequests = 4
+	maxQueueDuration := 5 * time.Second
+	vmselect.Init(maxConcurrentRequests, maxQueueDuration)
 	// storagePath will be created again when closing vmselect, so remove it again.
 	defer fs.MustRemoveDir(storagePath)
 	defer vminsert.Stop()
@@ -279,7 +281,8 @@ func processFlags() {
 }
 
 func setUp() {
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	const maxConcurrentRequests = 4
+	vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {
@@ -384,7 +387,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 				}
 			}
 			// flush series after each group evaluation
-			vmstorage.Storage.DebugFlush()
+			vmstorage.DebugFlush()
 		}
 
 		// check alert_rule_test case at every eval time

app/vmalert/config/config.go

@@ -113,15 +113,15 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid expression for rule %q: %w", ruleName, err)
 			}
 		}
 		if validateTplFn != nil {
 			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid annotations for rule %q: %w", ruleName, err)
 			}
 			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid labels for rule %q: %w", ruleName, err)
 			}
 		}
 	}

app/vmalert/config/config_test.go

@@ -121,7 +121,7 @@ func TestParse_Failure(t *testing.T) {
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad GraphiteQL expr")
 	f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr")
 	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
 	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
@@ -283,7 +283,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "up | 0",
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 
 	f(&Group{
 		Name: "test graphite expr",
@@ -293,7 +293,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				"description": "some-description",
 			}},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs expr",
@@ -327,7 +327,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "sum(up == 0 ) by (host)",
 			},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs with prometheus exp",
@@ -351,7 +351,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				For:    promutil.NewDuration(10 * time.Millisecond),
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -66,11 +66,11 @@ func (t *Type) ValidateExpr(expr string) error {
 	switch t.String() {
 	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
-			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad GraphiteQL expr: %q, err: %w", expr, err)
 		}
 	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
-			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad MetricsQL expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
 		q, err := logstorage.ParseStatsQuery(expr, 0)

app/vmalert/datasource/init.go

@@ -64,6 +64,7 @@ func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")
 	}
+	flagutil.RegisterSecretFlag("datasource.headers")
 }
 
 // ShowDatasourceURL whether to show -datasource.url with sensitive information

app/vmalert/notifier/config_watcher.go

@@ -105,7 +105,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn
 			}
 			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 			for _, err := range errors {
-				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
+				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
@@ -274,7 +274,7 @@ func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targ
 	for addr, metadata := range targetMts {
 		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
-			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
+			logger.Errorf("failed to init %s notifier with addr %q: %s", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{

app/vmalert/notifier/init.go

@@ -194,6 +194,7 @@ func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")
 	}
+	flagutil.RegisterSecretFlag("notifier.headers")
 }
 
 func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {

app/vmalert/remoteread/init.go

@@ -59,6 +59,7 @@ func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")
 	}
+	flagutil.RegisterSecretFlag("remoteRead.headers")
 }
 
 // Init creates a Querier from provided flag values.

app/vmalert/remotewrite/init.go

@@ -62,6 +62,7 @@ func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
+	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 // Init creates Client object from given flags.

app/vmalert/rule/group.go

@@ -95,6 +95,7 @@ type groupMetrics struct {
 	iterationTotal    *metrics.Counter
 	iterationDuration *metrics.Summary
 	iterationMissed   *metrics.Counter
+	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
 }
 
@@ -330,6 +331,7 @@ func (g *Group) Init() {
 	g.metrics.iterationTotal = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	g.metrics.iterationDuration = g.metrics.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	g.metrics.iterationMissed = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	g.metrics.iterationReset = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_reset_total{%s}`, labels))
 	g.metrics.iterationInterval = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		i := g.Interval.Seconds()
 		return i
@@ -474,14 +476,16 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			if missed < 0 {
 				// missed can become < 0 due to irregular delays during evaluation
 				// which can result in time.Since(evalTS) < g.Interval;
-				// or the system wall clock was changed backward
-				missed = 0
+				// or the system wall clock was changed backward,
+				// Reset the evalTS to the current time.
 				evalTS = time.Now()
+				g.metrics.iterationReset.Inc()
+			} else {
+				evalTS = evalTS.Add((missed + 1) * g.Interval)
 			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
 			}
-			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
 			eval(evalCtx, evalTS)
 		}

app/vmalert/web.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
@@ -160,12 +162,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
-		gf, err := newGroupsFilter(r)
+		af, err := newAlertsFilter(r)
 		if err != nil {
 			errJson(w, r, err)
 			return true
 		}
-		data, err := rh.listAlerts(gf)
+		data, err := rh.listAlerts(af)
 		if err != nil {
 			errJson(w, r, err)
 			return true
@@ -325,6 +327,48 @@ func (gf *groupsFilter) matches(group *rule.Group) bool {
 	return true
 }
 
+type alertsFilter struct {
+	gf    *groupsFilter
+	match [][]metricsql.LabelFilter
+}
+
+func getMatchFilters(matches []string) ([][]metricsql.LabelFilter, *httpserver.ErrorWithStatusCode) {
+	if len(matches) == 0 {
+		return nil, nil
+	}
+	tfss := make([][]metricsql.LabelFilter, 0, len(matches))
+	for _, s := range matches {
+		expr, err := metricsql.Parse(s)
+		if err != nil {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": failed to parse %q: %w`, s, err), http.StatusBadRequest)
+		}
+		me, ok := expr.(*metricsql.MetricExpr)
+		if !ok {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": expecting metricSelector; got %q`, expr.AppendString(nil)), http.StatusBadRequest)
+		}
+		if len(me.LabelFilterss) == 0 {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": labelFilterss cannot be empty`), http.StatusBadRequest)
+		}
+		tfss = append(tfss, me.LabelFilterss...)
+	}
+	return tfss, nil
+}
+
+func newAlertsFilter(r *http.Request) (*alertsFilter, *httpserver.ErrorWithStatusCode) {
+	gf, err := newGroupsFilter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var af alertsFilter
+	af.gf = gf
+	af.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
+	return &af, nil
+}
+
 // see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
 type rulesFilter struct {
 	gf             *groupsFilter
@@ -335,6 +379,7 @@ type rulesFilter struct {
 	maxGroups      int
 	pageNum        int
 	search         string
+	match          [][]metricsql.LabelFilter
 	extendedStates bool
 }
 
@@ -355,7 +400,10 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
 			return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
 		}
 	}
-
+	rf.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
 	states := vs["state"]
 	if len(states) == 0 {
 		states = vs["filter"]
@@ -416,12 +464,47 @@ func (rf *rulesFilter) matchesRule(r *rule.ApiRule) bool {
 	if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
 		return false
 	}
+	if !areLabelsMatch(r.Labels, rf.match) {
+		return false
+	}
 	if len(rf.states) == 0 {
 		return true
 	}
 	return slices.Contains(rf.states, r.State)
 }
 
+func areLabelsMatch(labels map[string]string, matches [][]metricsql.LabelFilter) bool {
+	if len(matches) == 0 {
+		return true
+	}
+	// labels need to match at least one of the provided match[] arg
+	return slices.ContainsFunc(matches, func(filters []metricsql.LabelFilter) bool {
+		for _, mf := range filters {
+			if !isLabelFilterMatch(labels[mf.Label], mf) {
+				return false
+			}
+		}
+		return true
+	})
+}
+
+func isLabelFilterMatch(s string, match metricsql.LabelFilter) bool {
+	if !match.IsRegexp {
+		if match.IsNegative {
+			return s != match.Value
+		}
+		return s == match.Value
+	}
+	re, err := metricsql.CompileRegexpAnchored(match.Value)
+	if err != nil {
+		return false
+	}
+	if match.IsNegative {
+		return !re.MatchString(s)
+	}
+	return re.MatchString(s)
+}
+
 func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
@@ -543,14 +626,14 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	return gAlerts
 }
 
-func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
+func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
 	lr.Data.Alerts = make([]*rule.ApiAlert, 0)
 	for _, group := range rh.m.groups {
-		if !gf.matches(group) {
+		if !af.gf.matches(group) {
 			continue
 		}
 		g := group.ToAPI()
@@ -558,7 +641,11 @@ func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.Erro
 			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
+			for _, alert := range r.Alerts {
+				if areLabelsMatch(alert.Labels, af.match) {
+					lr.Data.Alerts = append(lr.Data.Alerts, alert)
+				}
+			}
 		}
 	}
 

app/vmalert/web.qtpl

@@ -348,7 +348,7 @@
                 typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
                 count := len(ns)
             %}
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-{%s typeK %}">
                     <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
                     <span
@@ -361,7 +361,7 @@
                 <div id="item-{%s typeK %}" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web.qtpl.go

@@ -1115,7 +1115,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 
 //line app/vmalert/web.qtpl:350
 			qw422016.N().S(`
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-`)
 //line app/vmalert/web.qtpl:352
 			qw422016.E().S(typeK)
@@ -1152,7 +1152,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 			qw422016.N().S(`" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -37,12 +39,14 @@ func TestHandler(t *testing.T) {
 			Concurrency: 1,
 			Rules: []config.Rule{
 				{
-					ID:    0,
-					Alert: "alert",
+					ID:     0,
+					Alert:  "alert",
+					Labels: map[string]string{"job": "foo"},
 				},
 				{
 					ID:     1,
 					Record: "record",
+					Labels: map[string]string{"job": "bar"},
 				},
 			},
 		}, fq, 1*time.Minute, nil)
@@ -128,6 +132,18 @@ func TestHandler(t *testing.T) {
 		if length := len(lr.Data.Alerts); length != 2 {
 			t.Fatalf("expected 2 alert got %d", length)
 		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="foo"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 3 {
+			t.Fatalf("expected 3 alerts got %d", length)
+		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="bar"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 0 {
+			t.Fatalf("expected 0 alerts got %d", length)
+		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
 		expAlert := rule.NewAlertAPI(ar, ar.GetAlerts()[0])
@@ -242,6 +258,13 @@ func TestHandler(t *testing.T) {
 		check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
 		check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
 
+		// invalid match[] params
+		check(`/vmalert/api/v1/rules?match[]={job=!"foo"}`, 400, 0, 0)
+		check(`/vmalert/api/v1/rules?match[]={job="foo"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}&match[]={job="foo"}`, 200, 3, 6)
+		check(`/vmalert/api/v1/rules?match[]={job="barzz"}`, 200, 0, 0)
+
 		// no filtering expected due to bad params
 		check("/api/v1/rules?type=badParam", 400, 0, 0)
 		check("/api/v1/rules?foo=bar", 200, 3, 6)
@@ -367,3 +390,116 @@ func TestEmptyResponse(t *testing.T) {
 		}
 	})
 }
+
+func TestMatchesRule(t *testing.T) {
+	parseMatch := func(t *testing.T, selectors []string) [][]metricsql.LabelFilter {
+		t.Helper()
+		var match [][]metricsql.LabelFilter
+		for _, s := range selectors {
+			expr, err := metricsql.Parse(s)
+			if err != nil {
+				t.Fatalf("failed to parse selector %q: %v", s, err)
+			}
+			me, ok := expr.(*metricsql.MetricExpr)
+			if !ok {
+				t.Fatalf("expected MetricExpr for %q, got %T", s, expr)
+			}
+			match = append(match, me.LabelFilterss...)
+		}
+		return match
+	}
+
+	f := func(t *testing.T, selectors []string, labels map[string]string, wantMatch bool) {
+		t.Helper()
+		rf := &rulesFilter{
+			gf:    &groupsFilter{},
+			match: parseMatch(t, selectors),
+		}
+		r := &rule.ApiRule{Labels: labels}
+		got := rf.matchesRule(r)
+		if got != wantMatch {
+			t.Fatalf("matchesRule(%v) with selectors %v: got %v, want %v",
+				labels, selectors, got, wantMatch)
+		}
+	}
+
+	f(t, nil, map[string]string{"foo": "bar"}, true)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "baz"}, false)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"bar": "baz"}, false)
+	f(t, []string{`{foo=""}`}, map[string]string{"bar": "baz"}, true)
+
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "bar"}, false)
+
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "baz"}, false)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "baz"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "bar"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "foo"}, false)
+
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "bar"}, false)
+
+	// single match[] with multiple filters
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "foo", "instance": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "other", "instance": "bar"},
+		false,
+	)
+
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "bar", "baz": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "other", "baz": "bazinga"},
+		false,
+	)
+
+	// multiple matches[]
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "baz"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "unknown"},
+		false,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"bar": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "bartender"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "other", "bar": "other"},
+		false,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"foo": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo", instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"instance": "barr", "job": "foo"},
+		false,
+	)
+}

app/vmauth/auth_config.go

@@ -889,7 +889,8 @@ func reloadAuthConfig() (bool, error) {
 	}
 
 	mp := authUsers.Load()
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
+	jwtc := jwtAuthCache.Load()
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp)+len(jwtc.users), *authConfigPath)
 	return true, nil
 }
 

app/vmauth/example_config.yml

@@ -130,6 +130,16 @@ users:
   - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
   - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
 
+# JWT-based routing using header-based tenant identification (VictoriaMetrics cluster)
+# The AccountID and ProjectID from JWT vm_access claims are injected as HTTP headers.
+- name: jwt-header-tenant
+  jwt:
+    skip_verify: true
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"
+  url_prefix: "http://vminsert:8480/insert/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -17,6 +17,8 @@ import (
 
 const (
 	metricsTenantPlaceholder       = `{{.MetricsTenant}}`
+	metricsAccountIDPlaceholder    = `{{.MetricsAccountID}}`
+	metricsProjectIDPlaceholder    = `{{.MetricsProjectID}}`
 	metricsExtraLabelsPlaceholder  = `{{.MetricsExtraLabels}}`
 	metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
 
@@ -30,6 +32,8 @@ const (
 
 var allPlaceholders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	metricsExtraLabelsPlaceholder,
 	metricsExtraFiltersPlaceholder,
 	logsAccountIDPlaceholder,
@@ -40,6 +44,8 @@ var allPlaceholders = []string{
 
 var urlPathPlaceHolders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	logsAccountIDPlaceholder,
 	logsProjectIDPlaceholder,
 }
@@ -371,6 +377,8 @@ func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
 	data := map[string][]string{
 		// TODO: optimize at parsing stage
 		metricsTenantPlaceholder:       {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
+		metricsAccountIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsAccountID)},
+		metricsProjectIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsProjectID)},
 		metricsExtraLabelsPlaceholder:  vma.MetricsExtraLabels,
 		metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
 

app/vmauth/jwt_test.go

@@ -170,13 +170,13 @@ users:
   url_prefix: http://foo.bar
 `, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
 
-	// unsupported placeholder in a header
+	// unsupported placeholder in a URL path
 	f(`
 users:
 - jwt: 
     skip_verify: true
   url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
-		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 	// unsupported placeholder in a header
 	f(`
@@ -187,7 +187,7 @@ users:
     - "AccountID: {{.UnsupportedPlaceholder}}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// spaces in templating not allowed
@@ -199,7 +199,19 @@ users:
     - "AccountID: {{ .LogsAccountID }}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+
+	// placeholder must match the entire header value
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  headers:
+    - "AccountID: foo {{.MetricsAccountID}}"
+  url_prefix: http://foo.bar
+`,
+		"request header: \"AccountID\" has unsupported placeholder: \"foo {{.MetricsAccountID}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// oidc is not an object
@@ -364,10 +376,25 @@ users:
   url_prefix: http://foo.bar
 `, validRSAPublicKey, validECDSAPublicKey))
 
+	// metrics header placeholders
 	f(`
 users:
 - jwt:
     skip_verify: true
+  headers:
+  - "MetricsAccountID: {{.MetricsAccountID}}"
+  - "MetricsProjectID: {{.MetricsProjectID}}"
+  url_prefix: http://foo.bar
+`)
+
+	// logs header placeholders
+	f(`
+users:
+- jwt:
+    skip_verify: true
+  headers:
+  - "LogsAccountID: {{.LogsAccountID}}"
+  - "LogsProjectID: {{.LogsProjectID}}"
   url_prefix: http://foo.bar
 `)
 

app/vmauth/main.go

@@ -317,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 	defer ui.endConcurrencyLimit()
 
 	// Process the request.
-	processRequest(w, r, ui, tkn)
+	processRequest(w, r, ui, tkn, userName)
 }
 
 func beginConcurrencyLimit(ctx context.Context) error {
@@ -391,7 +391,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 	return bb, nil
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token, userName string) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
 	isDefault := false
@@ -409,7 +409,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 			if ui.DumpRequestOnErrors {
 				di = debugInfo(u, r)
 			}
-			httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
+			httpserver.Errorf(w, r, "user %s missing route for %q%s", userName, u.String(), di)
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
@@ -455,7 +455,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), userName),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)

app/vmauth/main_test.go

@@ -307,6 +307,24 @@ statusCode=200
 requested_url={BACKEND}/bar/a/b`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// correct authorization but unexisted path, hence missing route error.
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_map:
+  - src_paths:
+    - "/api/v1/write"
+    url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:secret@some-host.com/a/b"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=400
+user foo missing route for "http://foo:secret@some-host.com/a/b"`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// verify how path cleanup works
 	cfgStr = `
 unauthorized_user:
@@ -403,7 +421,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg"`
+user unauthorized missing route for "http://some-host.com/abc?de=fg"`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// missing default_url and default url_prefix for unauthorized user with dump_request_on_errors enabled
@@ -419,7 +437,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
+user unauthorized missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
 Pass-Header: abc
 Some-Header: foobar
 X-Forwarded-For: 12.34.56.78
@@ -461,7 +479,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// all the backend_urls are unavailable for authorized user
@@ -501,7 +519,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 0 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 0 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	netutil.Resolver = origResolver
 
@@ -518,7 +536,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	if n := retries.Load(); n != 2 {
 		t.Fatalf("unexpected number of retries; got %d; want 2", n)
@@ -545,6 +563,31 @@ requested_url={BACKEND}/path2/foo/?de=fg`
 	if n := retries.Load(); n != 2 {
 		t.Fatalf("unexpected number of retries; got %d; want 2", n)
 	}
+
+	// make sure that empty config value erases client extra filters and extra labels
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz&extra_filters[]=&extra_label=&extra_filters=`
+	requestURL = "http://some-host.com/abc/def?some_arg=some_value&extra_filters[]=baz&extra_label=tenant=admin&extra_filters=bar"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("Connection", "close")
+		h.Set("Foo", "bar")
+
+		var bb bytes.Buffer
+		if err := r.Header.Write(&bb); err != nil {
+			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
+	}
+	responseExpected = `
+statusCode=200
+Foo: bar
+requested_url={BACKEND}/foo/abc/def?bar=baz&extra_filters=&extra_filters%5B%5D=&extra_label=&some_arg=some_value
+Pass-Header: abc
+User-Agent: vmauth
+X-Forwarded-For: 12.34.56.78, 42.2.3.84`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
 }
 
 func TestJWTRequestHandler(t *testing.T) {
@@ -851,6 +894,30 @@ users:
 		responseExpected,
 	)
 
+	// test header injection and URL templating with individual placeholders
+	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
+	request.Header.Set(`Authorization`, `Bearer `+fullToken)
+	responseExpected = `
+statusCode=200
+path: /select/123/234/api/v1/query
+query:
+headers:
+    AccountID=123
+    ProjectID=234`
+	f(fmt.Sprintf(
+		`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: {BACKEND}/select/{{.MetricsAccountID}}/{{.MetricsProjectID}}
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"`, string(publicKeyPEM)),
+		request,
+		responseExpected,
+	)
+
 	// extra_label and extra_filters from vm_access claim merged with statically defined
 	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
 	request.Header.Set(`Authorization`, `Bearer `+fullToken)

app/vmctl/flags.go

@@ -146,7 +146,8 @@ var (
 			Name:  vmRoundDigits,
 			Value: 100,
 			Usage: "Round metric values to the given number of decimal digits after the point. " +
-				"This option may be used for increasing on-disk compression level for the stored metrics",
+				"This option may be used for increasing on-disk compression level for the stored metrics. " +
+				"See also --vm-significant-figures option",
 		},
 		&cli.StringSliceFlag{
 			Name:  vmExtraLabel,
@@ -500,6 +501,96 @@ var (
 	}
 )
 
+const (
+	mimirPath             = "mimir-path"
+	mimirTenantID         = "mimir-tenant-id"
+	mimirConcurrency      = "mimir-concurrency"
+	mimirFilterTimeStart  = "mimir-filter-time-start"
+	mimirFilterTimeEnd    = "mimir-filter-time-end"
+	mimirFilterLabel      = "mimir-filter-label"
+	mimirFilterLabelValue = "mimir-filter-label-value"
+
+	mimirCredsFilePath           = "mimir-creds-file-path"
+	mimirConfigFilePath          = "mimir-config-file-path"
+	mimirConfigProfile           = "mimir-config-profile"
+	mimirCustomS3Endpoint        = "mimir-custom-s3-endpoint"
+	mimirS3ForcePathStyle        = "mimir-s3-force-path-style"
+	mimirS3TLSInsecureSkipVerify = "mimir-s3-tls-insecure-skip-verify"
+	mimirSSEKMSKeyID             = "mimir-s3-sse-kms-key-id"
+	mimirSSEAlgorithm            = "mimir-s3-sse-algorithm"
+)
+
+var (
+	mimirFlags = []cli.Flag{
+		&cli.StringFlag{
+			Name:     mimirPath,
+			Usage:    "Path to Mimir storage bucket or local folder.",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirTenantID,
+			Usage: "Tenant ID for Mimir storage",
+		},
+		&cli.IntFlag{
+			Name:  mimirConcurrency,
+			Usage: "Number of concurrently running block readers",
+			Value: 1,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeStart,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeEnd,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabel,
+			Usage: "Mimir label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabelValue,
+			Usage: fmt.Sprintf("Regular expression to filter label from %q flag.", mimirFilterLabel),
+			Value: ".*",
+		},
+		&cli.StringFlag{
+			Name:  mimirCredsFilePath,
+			Usage: "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigFilePath,
+			Usage: "Path to file with S3 configs. Configs are loaded from default location if not set. See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigProfile,
+			Usage: "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used",
+		},
+		&cli.StringFlag{
+			Name:  mimirCustomS3Endpoint,
+			Usage: "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set",
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3ForcePathStyle,
+			Usage: "Prefixing endpoint with bucket name when set false, true by default.",
+			Value: true,
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3TLSInsecureSkipVerify,
+			Usage: "Whether to skip TLS verification when connecting to the S3 endpoint.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEKMSKeyID,
+			Usage: "SSE KMS Key ID for use with S3-compatible storages.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEAlgorithm,
+			Usage: "SSE algorithm for use with S3-compatible storages.",
+		},
+	}
+)
+
 const (
 	vmNativeFilterMatch       = "vm-native-filter-match"
 	vmNativeFilterTimeStart   = "vm-native-filter-time-start"

app/vmctl/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/mimir"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -297,12 +298,12 @@ func main() {
 				},
 			},
 			{
-				Name:   "thanos",
-				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
-				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Name:   "mimir",
+				Usage:  "Migrate time series from Mimir object storage or local filesystem",
+				Flags:  mergeFlags(globalFlags, mimirFlags, vmFlags),
 				Before: beforeFn,
 				Action: func(c *cli.Context) error {
-					fmt.Println("Thanos import mode")
+					fmt.Println("Mimir import mode")
 
 					vmCfg, err := initConfigVM(c)
 					if err != nil {
@@ -314,6 +315,54 @@ func main() {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
+					mCfg := mimir.Config{
+						Filter: mimir.Filter{
+							TimeMin:    c.String(mimirFilterTimeStart),
+							TimeMax:    c.String(mimirFilterTimeEnd),
+							Label:      c.String(mimirFilterLabel),
+							LabelValue: c.String(mimirFilterLabelValue),
+						},
+						Path:                    c.String(mimirPath),
+						TenantID:                c.String(mimirTenantID),
+						CredsFilePath:           c.String(mimirCredsFilePath),
+						ConfigFilePath:          c.String(mimirConfigFilePath),
+						ConfigProfile:           c.String(mimirConfigProfile),
+						CustomS3Endpoint:        c.String(mimirCustomS3Endpoint),
+						S3ForcePathStyle:        c.Bool(mimirS3ForcePathStyle),
+						S3TLSInsecureSkipVerify: c.Bool(mimirS3TLSInsecureSkipVerify),
+						SSEKMSKeyID:             c.String(mimirSSEKMSKeyID),
+						SSEAlgorithm:            c.String(mimirSSEAlgorithm),
+					}
+					cl, err := mimir.NewClient(ctx, mCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create mimir client: %s", err)
+					}
+
+					pp := prometheusProcessor{
+						cl:        cl,
+						im:        importer,
+						cc:        c.Int(mimirConcurrency),
+						isVerbose: c.Bool(globalVerbose),
+					}
+					return pp.run(ctx)
+				},
+			},
+			{
+				Name:   "thanos",
+				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
+				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Before: beforeFn,
+				Action: func(c *cli.Context) error {
+					fmt.Println("Thanos import mode")
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
+
+					importer, err = vm.NewImporter(ctx, vmCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create VM importer: %s", err)
+					}
 					thanosCfg := thanos.Config{
 						Snapshot: c.String(thanosSnapshot),
 						Filter: thanos.Filter{

app/vmctl/mimir/lazyreader.go

@@ -0,0 +1,195 @@
+package mimir
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/tsdb/tombstones"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+var _ tsdb.BlockReader = (*lazyBlockReader)(nil)
+
+// lazyBlockReader is stores block id and segment num information.
+// It is used to lazily fetch and parse block data.
+// It implements tsdb.BlockReader interface.
+type lazyBlockReader struct {
+	// Block ID.
+	ID ulid.ULID
+	// SegmentsNum stores the number of chunks segments in the block.
+	SegmentsNum int
+
+	mu          sync.Mutex
+	reader      *tsdb.Block
+	tempDirPath string
+	fs          common.RemoteFS
+	err         error
+}
+
+// newLazyBlockReader returns a new LazyBlockReader for the given block.
+func newLazyBlockReader(block *Block, fs common.RemoteFS) (*lazyBlockReader, error) {
+	if block.SegmentsFormat != "1b6d" {
+		return nil, fmt.Errorf("unsupported segments format: %s", block.SegmentsFormat)
+	}
+
+	return &lazyBlockReader{
+		ID:          block.ID,
+		SegmentsNum: block.SegmentsNum,
+		fs:          fs,
+	}, nil
+}
+
+func (lbr *lazyBlockReader) initialize() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader != nil {
+		return nil
+	}
+	// fetching block and parse it and store it in lbr.reader
+	temp, err := lbr.mkTempDir()
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %s", err)
+	}
+
+	lbr.tempDirPath = temp
+
+	// TODO: replace fetchFile and writeFile with buffered IO if needed
+	meta, err := lbr.fetchFile(metaFilename)
+	if err != nil {
+		return err
+	}
+	if err := lbr.writeFile(temp, metaFilename, meta); err != nil {
+		return fmt.Errorf("failed to write meta file: %w", err)
+	}
+	idx, err := lbr.fetchFile(indexFilename)
+	if err != nil {
+		return fmt.Errorf("failed to fetch index file %q: %w", indexFilename, err)
+	}
+	if err := lbr.writeFile(temp, indexFilename, idx); err != nil {
+		return err
+	}
+
+	for i := 1; i <= lbr.SegmentsNum; i++ {
+		// segments formats has format 1b06d
+		// https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L32
+		chunkName := fmt.Sprintf("%06d", i)
+		blockChunkPath := filepath.Join("chunks", chunkName)
+		chunk, err := lbr.fetchFile(blockChunkPath)
+		if err != nil {
+			return fmt.Errorf("failed to fetch chunk file: %q: %w", chunkName, err)
+		}
+		if err := lbr.writeFile(temp, blockChunkPath, chunk); err != nil {
+			return fmt.Errorf("failed to write chunk file: %q: %s", chunkName, err)
+		}
+	}
+
+	// Set postingDecoder to nil because
+	// If it is nil then a default decoder is used, compatible with Prometheus v2.
+	pb, err := tsdb.OpenBlock(nil, temp, nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to open block %q: %w", lbr.ID, err)
+	}
+	lbr.reader = pb
+	return nil
+}
+
+// Index returns an IndexReader over the block's data.
+func (lbr *lazyBlockReader) Index() (tsdb.IndexReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Index()
+}
+
+// Chunks returns a ChunkReader over the block's data.
+func (lbr *lazyBlockReader) Chunks() (tsdb.ChunkReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Chunks()
+}
+
+// Tombstones returns a tombstones.Reader over the block's deleted data.
+func (lbr *lazyBlockReader) Tombstones() (tombstones.Reader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Tombstones()
+}
+
+// Meta provides meta information about the block reader.
+func (lbr *lazyBlockReader) Meta() tsdb.BlockMeta {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("cannot get BlockMeta: %w", err)
+		return tsdb.BlockMeta{}
+	}
+	return lbr.reader.Meta()
+}
+
+// Size returns the number of bytes that the block takes up on disk.
+func (lbr *lazyBlockReader) Size() int64 {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("error get Size of the block: %s, return zero size", err)
+		return 0
+	}
+	return lbr.reader.Size()
+}
+
+// Err returns the last error that occurred on the block reader.
+func (lbr *lazyBlockReader) Err() error {
+	return lbr.err
+}
+
+// Close closes block and releases all resources
+func (lbr *lazyBlockReader) Close() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader == nil {
+		return nil
+	}
+
+	err := lbr.reader.Close()
+	if err := os.RemoveAll(lbr.tempDirPath); err != nil {
+		log.Printf("failed to remove temp dir: %s", err)
+	}
+	lbr.reader = nil
+	lbr.tempDirPath = ""
+
+	return err
+}
+
+func (lbr *lazyBlockReader) mkTempDir() (string, error) {
+	temp, err := os.MkdirTemp("", lbr.ID.String())
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	err = os.Mkdir(filepath.Join(temp, "chunks"), os.ModePerm)
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	return temp, nil
+}
+
+func (lbr *lazyBlockReader) fetchFile(filePath string) ([]byte, error) {
+	blockID := lbr.ID.String()
+	blockPath := filepath.Join(blockID, filePath)
+	has, err := lbr.fs.HasFile(blockPath)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("block meta %s not found", blockID)
+	}
+	return lbr.fs.ReadFile(blockPath)
+}
+
+func (lbr *lazyBlockReader) writeFile(folder string, filename string, file []byte) error {
+	fileName := filepath.Join(folder, filename)
+	return os.WriteFile(fileName, file, os.ModePerm)
+}

app/vmctl/mimir/mimir.go

@@ -0,0 +1,238 @@
+package mimir
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
+	utils "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+const (
+	bucketIndex                   = "bucket-index.json"
+	bucketIndexCompressedFilename = bucketIndex + ".gz"
+	metaFilename                  = "meta.json"
+	indexFilename                 = "index"
+)
+
+// BlockDeletionMark holds the information about a block's deletion mark in the index.
+// This type was copied from the mimir repository https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L234.
+type BlockDeletionMark struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
+	DeletionTime int64 `json:"deletion_time"`
+}
+
+// Block holds the information about a block in the index.
+// This is a partial implementation of the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L73
+type Block struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
+	MinTime int64 `json:"min_time"`
+	MaxTime int64 `json:"max_time"`
+
+	// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
+	// in the block.
+	SegmentsFormat string `json:"segments_format,omitempty"`
+	SegmentsNum    int    `json:"segments_num,omitempty"`
+}
+
+// Index contains all known blocks and markers of a tenant.
+// This is a partial implementation pof the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L36
+type Index struct {
+	// Version of the index format.
+	Version int `json:"version"`
+
+	// List of complete blocks (partial blocks are excluded from the index).
+	Blocks []*Block `json:"blocks"`
+}
+
+// Config contains a list of params needed
+// for reading mimir snapshots
+type Config struct {
+	// Path to remote storage bucket
+	Path string
+	// TenantID is the tenant id for the storage
+	TenantID string
+
+	Filter Filter
+
+	CredsFilePath           string
+	ConfigFilePath          string
+	ConfigProfile           string
+	CustomS3Endpoint        string
+	S3ForcePathStyle        bool
+	S3TLSInsecureSkipVerify bool
+
+	SSEKMSKeyID  string
+	SSEAlgorithm string
+}
+
+// Filter contains configuration for filtering
+// the timeseries
+type Filter struct {
+	TimeMin    string
+	TimeMax    string
+	Label      string
+	LabelValue string
+}
+
+// Client is a wrapper over Prometheus tsdb.DBReader
+type Client struct {
+	common.RemoteFS
+	filter filter
+}
+
+type filter struct {
+	min, max   int64
+	label      string
+	labelValue string
+}
+
+func (f filter) inRange(minTime, maxTime int64) bool {
+	fmin, fmax := f.min, f.max
+	if minTime == 0 {
+		fmin = minTime
+	}
+	if fmax == 0 {
+		fmax = maxTime
+	}
+	return minTime <= fmax && fmin <= maxTime
+}
+
+// NewClient creates and validates new Client
+// with given Config
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Path == "" {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+
+	if cfg.TenantID != "" {
+		cfg.Path = fmt.Sprintf("%s/%s", cfg.Path, cfg.TenantID)
+	}
+
+	var c Client
+	rfs, err := newRemoteFS(ctx, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", cfg.Path, err)
+	}
+
+	c.RemoteFS = rfs
+	timeMin, err := utils.ParseTime(cfg.Filter.TimeMin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse min time in filter: %s", err)
+	}
+	timeMax, err := utils.ParseTime(cfg.Filter.TimeMax)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse max time in filter: %s", err)
+	}
+	c.filter = filter{
+		min:        timeMin.UnixMilli(),
+		max:        timeMax.UnixMilli(),
+		label:      cfg.Filter.Label,
+		labelValue: cfg.Filter.LabelValue,
+	}
+	return &c, nil
+}
+
+// Explore a fetches bucket-index.json file from a remote storage or local filesystem
+// and filter blocks via the defined time range, but does not take into account label filters.
+func (c *Client) Explore() ([]tsdb.BlockReader, error) {
+
+	log.Printf("Fetching blocks from remote storage")
+
+	indexFile, err := c.fetchIndexFile()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch index file: %s", err)
+	}
+
+	var blocksToImport []tsdb.BlockReader
+	for _, block := range indexFile.Blocks {
+		if !c.filter.inRange(block.MinTime, block.MaxTime) {
+			// Skipping block outside of time range
+			continue
+		}
+
+		if block.ID.String() == "" {
+			continue
+		}
+
+		lazyBlockReader, err := newLazyBlockReader(block, c.RemoteFS)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create lazy block reader: %s", err)
+		}
+		blocksToImport = append(blocksToImport, lazyBlockReader)
+	}
+
+	return blocksToImport, nil
+}
+
+// Read reads the given BlockReader according to configured
+// time and label filters.
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error) {
+	meta := block.Meta()
+	if b, ok := block.(*lazyBlockReader); ok && b.Err() != nil {
+		return nil, fmt.Errorf("failed to read block: %s", b.Err())
+	}
+
+	if meta.ULID.String() == "" {
+		return nil, fmt.Errorf("unexpected block without id")
+	}
+
+	minTime, maxTime := meta.MinTime, meta.MaxTime
+	if c.filter.min != 0 {
+		minTime = c.filter.min
+	}
+	if c.filter.max != 0 {
+		maxTime = c.filter.max
+	}
+	q, err := tsdb.NewBlockQuerier(block, minTime, maxTime)
+	if err != nil {
+		return nil, err
+	}
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &prometheus.CloseableSeriesSet{SeriesSet: ss, Close: q.Close}, nil
+}
+
+func (c *Client) fetchIndexFile() (*Index, error) {
+	has, err := c.HasFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("bucket-index.json.gz not found")
+	}
+
+	file, err := c.ReadFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read bucket index: %s", err)
+	}
+
+	r := bytes.NewReader(file)
+	// Read all the content.
+	gzipReader, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %s", err)
+	}
+
+	var indexFile Index
+	err = json.NewDecoder(gzipReader).Decode(&indexFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode bucket index: %s", err)
+	}
+
+	return &indexFile, nil
+}

app/vmctl/mimir/remotefs.go

@@ -0,0 +1,93 @@
+package mimir
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/azremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
+)
+
+// newRemoteFS returns new remote fs from the given Config.
+func newRemoteFS(ctx context.Context, cfg Config) (common.RemoteFS, error) {
+	if len(cfg.Path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	n := strings.Index(cfg.Path, "://")
+	if n < 0 {
+		return nil, fmt.Errorf("missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `azblob://`, `fs://`", cfg.Path)
+	}
+	scheme := cfg.Path[:n]
+	dir := cfg.Path[n+len("://"):]
+	switch scheme {
+	case "fs":
+		if !filepath.IsAbs(dir) {
+			return nil, fmt.Errorf("dir must be absolute; got %q", dir)
+		}
+		fsr := &fsremote.FS{
+			Dir: filepath.Clean(dir),
+		}
+		return fsr, nil
+	case "gcs", "gs":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &gcsremote.FS{
+			CredsFilePath: cfg.CredsFilePath,
+			Bucket:        bucket,
+			Dir:           dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to gcs: %w", err)
+		}
+		return fsr, nil
+	case "azblob":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the AZBlob container %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &azremote.FS{
+			Container: bucket,
+			Dir:       dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to AZBlob: %w", err)
+		}
+		return fsr, nil
+	case "s3":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &s3remote.FS{
+			CredsFilePath:         cfg.CredsFilePath,
+			ConfigFilePath:        cfg.ConfigFilePath,
+			CustomEndpoint:        cfg.CustomS3Endpoint,
+			TLSInsecureSkipVerify: cfg.S3TLSInsecureSkipVerify,
+			S3ForcePathStyle:      cfg.S3ForcePathStyle,
+			ProfileName:           cfg.ConfigProfile,
+			Bucket:                bucket,
+			Dir:                   dir,
+			SSEKMSKeyId:           cfg.SSEKMSKeyID,
+			SSEAlgorithm:          s3remote.StringToEncryptionAlgorithm(cfg.SSEAlgorithm),
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)
+		}
+		return fsr, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmctl/prometheus.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"strings"
 	"sync"
@@ -18,10 +19,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 )
 
+// Runner is an interface for fetching and reading
+// snapshot blocks
+type Runner interface {
+	Explore() ([]tsdb.BlockReader, error)
+	Read(context.Context, tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error)
+}
+
 type prometheusProcessor struct {
-	// prometheus client fetches and reads
+	// Runner fetches and reads
 	// snapshot blocks
-	cl *prometheus.Client
+	cl Runner
 	// importer performs import requests
 	// for timeseries data returned from
 	// snapshot blocks
@@ -48,7 +56,7 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	if err := pp.processBlocks(blocks); err != nil {
+	if err := pp.processBlocks(ctx, blocks); err != nil {
 		return fmt.Errorf("migration failed: %s", err)
 	}
 
@@ -57,11 +65,17 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 	return nil
 }
 
-func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
-	ss, err := pp.cl.Read(b)
+func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
+	css, err := pp.cl.Read(ctx, b)
 	if err != nil {
 		return fmt.Errorf("failed to read block: %s", err)
 	}
+	defer func() {
+		if err := css.Close(); err != nil {
+			log.Printf("cannot close SeriesSet for block: %q : %s\n", b.Meta().ULID, err)
+		}
+	}()
+	ss := css.SeriesSet
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
@@ -114,7 +128,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	return ss.Err()
 }
 
-func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
+func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.BlockReader) error {
 	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
@@ -130,11 +144,16 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	for range pp.cc {
 		wg.Go(func() {
 			for br := range blockReadersCh {
-				if err := pp.do(br); err != nil {
+				if err := pp.do(ctx, br); err != nil {
 					promErrorsTotal.Inc()
-					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
+					errCh <- fmt.Errorf("cannot read block %q: %s", br.Meta().ULID, err)
 					return
 				}
+				if cb, ok := br.(io.Closer); ok {
+					if err := cb.Close(); err != nil {
+						errCh <- fmt.Errorf("cannot close block: %q: %w", br.Meta().ULID, err)
+					}
+				}
 				promBlocksProcessed.Inc()
 				bar.Increment()
 			}

app/vmctl/prometheus/prometheus.go

@@ -8,6 +8,8 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
 )
 
 // Config contains a list of params needed
@@ -60,13 +62,13 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
 	}
 	c := &Client{DBReadOnly: db}
-	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	timeMin, timeMax, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
 	}
 	c.filter = filter{
-		min:        minTime,
-		max:        maxTime,
+		min:        timeMin,
+		max:        timeMax,
 		label:      cfg.Filter.Label,
 		labelValue: cfg.Filter.LabelValue,
 	}
@@ -83,7 +85,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch blocks: %s", err)
 	}
-	s := &Stats{
+	s := &vmctlutil.Stats{
 		Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
 		Blocks:   len(blocks),
 	}
@@ -108,9 +110,15 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	return blocksToImport, nil
 }
 
+// CloseableSeriesSet defines a SeriesSet with Close method
+type CloseableSeriesSet struct {
+	SeriesSet storage.SeriesSet
+	Close     func() error
+}
+
 // Read reads the given BlockReader according to configured
 // time and label filters.
-func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*CloseableSeriesSet, error) {
 	minTime, maxTime := block.Meta().MinTime, block.Meta().MaxTime
 	if c.filter.min != 0 {
 		minTime = c.filter.min
@@ -122,8 +130,8 @@ func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
 	if err != nil {
 		return nil, err
 	}
-	ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
-	return ss, nil
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &CloseableSeriesSet{ss, q.Close}, nil
 }
 
 func parseTime(start, end string) (int64, int64, error) {

app/vmctl/vmctlutil/stats.go

@@ -1,4 +1,4 @@
-package prometheus
+package vmctlutil
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ type Stats struct {
 
 // String returns string representation for s.
 func (s Stats) String() string {
-	str := fmt.Sprintf("Prometheus snapshot stats:\n"+
+	str := fmt.Sprintf("Snapshot stats:\n"+
 		"  blocks found: %d;\n"+
 		"  blocks skipped by time filter: %d;\n"+
 		"  min time: %d (%v);\n"+

app/vmctl/vmctlutil/time_test.go

@@ -20,6 +20,9 @@ func TestGetTime_Failure(t *testing.T) {
 
 	// negative time
 	f("-292273086-05-16T16:47:06Z")
+
+	// relative duration that resolves to a timestamp before 1970
+	f("-9223372036.855")
 }
 
 func TestGetTime_Success(t *testing.T) {
@@ -77,9 +80,6 @@ func TestGetTime_Success(t *testing.T) {
 	// float timestamp representation",
 	f("1562529662.324", time.Date(2019, 7, 7, 20, 01, 02, 324e6, time.UTC))
 
-	// negative timestamp
-	f("-9223372036.855", time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC))
-
 	// big timestamp
 	f("1223372036855", time.Date(2008, 10, 7, 9, 33, 56, 855e6, time.UTC))
 

app/vminsert/common/insert_ctx.go

@@ -184,7 +184,7 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 	}
 	ctx.mms = mms
 
-	err := vmstorage.AddMetadataRows(mms)
+	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{
 			Err:        fmt.Errorf("cannot store metrics metadata: %w", err),
@@ -209,7 +209,7 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 	}
 	ctx.mms = mms
 
-	err := vmstorage.AddMetadataRows(mms)
+	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{
 			Err:        fmt.Errorf("cannot store prometheus metrics metadata: %w", err),
@@ -278,7 +278,7 @@ func (ctx *InsertCtx) FlushBufs() error {
 	// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
 	// used at every stream.Parse() call under lib/protoparser/*
 
-	err := vmstorage.AddRows(ctx.mrs)
+	err := vmstorage.VMInsertAPI.WriteRows(ctx.mrs)
 	ctx.Reset(0)
 	if err == nil {
 		return nil

app/vminsert/common/streamaggr.go

@@ -283,7 +283,7 @@ func pushAggregateSeries(tss []prompb.TimeSeries) {
 	}
 	// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
 	// since the number of concurrent pushAggregateSeries() calls should be already limited by lib/streamaggr.
-	if err := vmstorage.AddRows(ctx.mrs); err != nil {
+	if err := vmstorage.VMInsertAPI.WriteRows(ctx.mrs); err != nil {
 		logger.Errorf("cannot flush aggregate series: %s", err)
 	}
 }

app/vminsert/main.go

@@ -89,6 +89,7 @@ var staticServer = http.FileServer(http.FS(staticFiles))
 func Init() {
 	relabel.Init()
 	common.InitStreamAggr()
+	opentelemetry.Init()
 	protoparserutil.StartUnmarshalWorkers()
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)

app/vminsert/opentelemetry/request_handler.go

@@ -20,6 +20,11 @@ var (
 	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(req *http.Request) error {
 	extraLabels, err := protoparserutil.GetExtraLabels(req)

app/vmselect/graphite/metrics_api.go

@@ -1,7 +1,6 @@
 package graphite
 
 import (
-	"flag"
 	"fmt"
 	"math"
 	"net/http"
@@ -21,8 +20,6 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-var maxTagValueSuffixes = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
-
 // MetricsFindHandler implements /metrics/find handler.
 //
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
@@ -222,10 +219,11 @@ func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 
 // metricsFind searches for label values that match the given qHead and qTail.
 func metricsFind(tr storage.TimeRange, label, qHead, qTail string, delimiter byte, isExpand bool, deadline searchutil.Deadline) ([]string, error) {
+	maxSuffixes := 0 // let vmstorage use its maxTagValueSuffixesPerSearch limit
 	n := strings.IndexAny(qTail, "*{[")
 	if n < 0 {
 		query := qHead + qTail
-		suffixes, err := netstorage.TagValueSuffixes(nil, tr, label, query, delimiter, *maxTagValueSuffixes, deadline)
+		suffixes, err := netstorage.TagValueSuffixes(nil, tr, label, query, delimiter, maxSuffixes, deadline)
 		if err != nil {
 			return nil, err
 		}
@@ -245,7 +243,7 @@ func metricsFind(tr storage.TimeRange, label, qHead, qTail string, delimiter byt
 	}
 	if n == len(qTail)-1 && strings.HasSuffix(qTail, "*") {
 		query := qHead + qTail[:len(qTail)-1]
-		suffixes, err := netstorage.TagValueSuffixes(nil, tr, label, query, delimiter, *maxTagValueSuffixes, deadline)
+		suffixes, err := netstorage.TagValueSuffixes(nil, tr, label, query, delimiter, maxSuffixes, deadline)
 		if err != nil {
 			return nil, err
 		}

app/vmselect/graphite/tags_api.go

@@ -138,7 +138,9 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 		mr.Timestamp = ct
 	}
-	vmstorage.RegisterMetricNames(nil, mrs)
+	if err := vmstorage.VMSelectAPI.RegisterMetricNames(nil, mrs, 0); err != nil {
+		return err
+	}
 
 	// Return response
 	contentType := "text/plain; charset=utf-8"

app/vmselect/main.go

@@ -21,7 +21,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/stats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -36,12 +35,6 @@ var (
 	deleteAuthKey                = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It could be passed via authKey query arg. It overrides -httpAuth.*")
 	metricNamesStatsResetAuthKey = flagutil.NewPassword("metricNamesStatsResetAuthKey", "authKey for resetting metric names usage cache via /api/v1/admin/status/metric_names_stats/reset. It overrides -httpAuth.*. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage")
-
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
 	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
@@ -50,25 +43,21 @@ var (
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
 
-func getDefaultMaxConcurrentRequests() int {
-	// A single request can saturate all the CPU cores, so there is no sense
-	// in allowing higher number of concurrent requests - they will just contend
-	// for unavailable CPU time.
-	n := min(cgroup.AvailableCPUs()*2, 16)
-	return n
-}
-
 // Init initializes vmselect
-func Init() {
-	tmpDirPath := *vmstorage.DataPath + "/tmp"
+func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration) {
+	tmpDirPath := vmstorage.DataPath() + "/tmp"
 	fs.MustRemoveDirContents(tmpDirPath)
 	netstorage.InitTmpBlocksDir(tmpDirPath)
-	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
-	prometheus.InitMaxUniqueTimeseries(*maxConcurrentRequests)
+	promql.InitRollupResultCache(vmstorage.DataPath() + "/cache/rollupResult")
+
+	maxConcurrentRequests = vmselectMaxConcurrentRequests
+	maxQueueDuration = vmselectMaxQueueDuration
+	concurrencyLimitCh = make(chan struct{}, maxConcurrentRequests)
 
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
 	initVMUIConfig()
 	initVMAlertProxy()
+
+	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 
 // Stop stops vmselect
@@ -76,7 +65,11 @@ func Stop() {
 	promql.StopRollupResultCache()
 }
 
-var concurrencyLimitCh chan struct{}
+var (
+	maxConcurrentRequests int
+	maxQueueDuration      time.Duration
+	concurrencyLimitCh    chan struct{}
+)
 
 var (
 	concurrencyLimitReached = metrics.NewCounter(`vm_concurrent_select_limit_reached_total`)
@@ -88,9 +81,6 @@ var (
 	_ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 {
 		return float64(len(concurrencyLimitCh))
 	})
-	_ = metrics.NewGauge(`vm_search_max_unique_timeseries`, func() float64 {
-		return float64(prometheus.GetMaxUniqueTimeSeries())
-	})
 )
 
 //go:embed vmui
@@ -129,12 +119,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		d := min(searchutil.GetMaxQueryDuration(r), *maxQueueDuration)
+		d := min(searchutil.GetMaxQueryDuration(r), maxQueueDuration)
 		t := timerpool.Get(d)
 		select {
 		case concurrencyLimitCh <- struct{}{}:
 			timerpool.Put(t)
-			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", *maxConcurrentRequests)
+			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", maxConcurrentRequests)
 			defer func() { <-concurrencyLimitCh }()
 		case <-r.Context().Done():
 			timerpool.Put(t)
@@ -150,7 +140,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
 					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
-					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
+					d.Seconds(), maxConcurrentRequests, maxQueueDuration),
 				StatusCode: http.StatusTooManyRequests,
 			}
 			w.Header().Add("Retry-After", "10")

app/vmselect/netstorage/netstorage.go

@@ -27,10 +27,6 @@ import (
 )
 
 var (
-	maxTagKeysPerSearch = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned from /api/v1/labels . "+
-		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
-	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned from /api/v1/label/<label_name>/values . "+
-		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
 	maxSamplesPerSeries = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
 	maxSamplesPerQuery  = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. "+
 		"This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
@@ -80,7 +76,7 @@ func (rss *Results) Cancel() {
 }
 
 func (rss *Results) mustClose() {
-	putStorageSearch(rss.sr)
+	vmstorage.PutSearch(rss.sr)
 	rss.sr = nil
 	putTmpBlocksFile(rss.tbf)
 	rss.tbf = nil
@@ -758,12 +754,7 @@ var sbhPool sync.Pool
 func DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline searchutil.Deadline) (int, error) {
 	qt = qt.NewChild("delete series: %s", sq)
 	defer qt.Done()
-	tr := sq.GetTimeRange()
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
-	if err != nil {
-		return 0, err
-	}
-	return vmstorage.DeleteSeries(qt, tfss, sq.MaxMetrics)
+	return vmstorage.VMSelectAPI.DeleteSeries(qt, sq, deadline.Deadline())
 }
 
 // LabelNames returns label names matching the given sq until the given deadline.
@@ -773,15 +764,7 @@ func LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames i
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	if maxLabelNames > *maxTagKeysPerSearch || maxLabelNames <= 0 {
-		maxLabelNames = *maxTagKeysPerSearch
-	}
-	tr := sq.GetTimeRange()
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	labels, err := vmstorage.SearchLabelNames(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
+	labels, err := vmstorage.VMSelectAPI.LabelNames(qt, sq, maxLabelNames, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search on time range: %w", err)
 	}
@@ -841,15 +824,7 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	if maxLabelValues > *maxTagValuesPerSearch || maxLabelValues <= 0 {
-		maxLabelValues = *maxTagValuesPerSearch
-	}
-	tr := sq.GetTimeRange()
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	labelValues, err := vmstorage.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
+	labelValues, err := vmstorage.VMSelectAPI.LabelValues(qt, sq, labelName, maxLabelValues, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
 	}
@@ -864,7 +839,10 @@ func GetMetricsMetadata(qt *querytracer.Tracer, limit int, metricName string) ([
 	qt = qt.NewChild("get metrics metadata: limit=%d, metric_name=%q", limit, metricName)
 	defer qt.Done()
 
-	metadata := vmstorage.Storage.GetMetadataRows(qt, limit, metricName)
+	metadata, err := vmstorage.VMSelectAPI.GetMetadataRecords(qt, nil, limit, metricName, 0)
+	if err != nil {
+		return nil, err
+	}
 
 	sort.Slice(metadata, func(i, j int) bool {
 		return string(metadata[i].MetricFamilyName) < string(metadata[j].MetricFamilyName)
@@ -912,16 +890,11 @@ func TagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagV
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	suffixes, err := vmstorage.SearchTagValueSuffixes(qt, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline.Deadline())
+	suffixes, err := vmstorage.VMSelectAPI.TagValueSuffixes(qt, 0, 0, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during search for suffixes for tagKey=%q, tagValuePrefix=%q, delimiter=%c on time range %s: %w",
 			tagKey, tagValuePrefix, delimiter, tr.String(), err)
 	}
-	if len(suffixes) >= maxSuffixes {
-		return nil, fmt.Errorf("more than -search.maxTagValueSuffixesPerSearch=%d tag value suffixes found for tagKey=%q, tagValuePrefix=%q, delimiter=%c on time range %s; "+
-			"either narrow down the query or increase -search.maxTagValueSuffixesPerSearch command-line flag value",
-			maxSuffixes, tagKey, tagValuePrefix, delimiter, tr.String())
-	}
 	return suffixes, nil
 }
 
@@ -934,13 +907,7 @@ func TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel stri
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	tr := sq.GetTimeRange()
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	date := uint64(tr.MinTimestamp) / (3600 * 24 * 1000)
-	status, err := vmstorage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, sq.MaxMetrics, deadline.Deadline())
+	status, err := vmstorage.VMSelectAPI.TSDBStatus(qt, sq, focusLabel, topN, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during tsdb status request: %w", err)
 	}
@@ -954,28 +921,13 @@ func SeriesCount(qt *querytracer.Tracer, deadline searchutil.Deadline) (uint64,
 	if deadline.Exceeded() {
 		return 0, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	n, err := vmstorage.GetSeriesCount(deadline.Deadline())
+	n, err := vmstorage.VMSelectAPI.SeriesCount(qt, 0, 0, deadline.Deadline())
 	if err != nil {
 		return 0, fmt.Errorf("error during series count request: %w", err)
 	}
 	return n, nil
 }
 
-func getStorageSearch() *storage.Search {
-	v := ssPool.Get()
-	if v == nil {
-		return &storage.Search{}
-	}
-	return v.(*storage.Search)
-}
-
-func putStorageSearch(sr *storage.Search) {
-	sr.MustClose()
-	ssPool.Put(sr)
-}
-
-var ssPool sync.Pool
-
 // ExportBlocks searches for time series matching sq and calls f for each found block.
 //
 // f is called in parallel from multiple goroutines.
@@ -989,21 +941,13 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 	if deadline.Exceeded() {
 		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
 	}
+
 	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return err
-	}
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
+	sr, _, err := vmstorage.GetSearch(qt, sq, deadline.Deadline())
 	if err != nil {
 		return err
 	}
-
-	vmstorage.WG.Add(1)
-	defer vmstorage.WG.Done()
-
-	sr := getStorageSearch()
-	defer putStorageSearch(sr)
-	sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
+	defer vmstorage.PutSearch(sr)
 
 	// Start workers that call f in parallel on available CPU cores.
 	workCh := make(chan *exportWork, gomaxprocs*8)
@@ -1096,17 +1040,7 @@ func SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline
 		return nil, fmt.Errorf("timeout exceeded before starting to search metric names: %s", deadline.String())
 	}
 
-	// Setup search.
-	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return nil, err
-	}
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-
-	metricNames, err := vmstorage.SearchMetricNames(qt, tfss, tr, sq.MaxMetrics, deadline.Deadline())
+	metricNames, err := vmstorage.VMSelectAPI.SearchMetricNames(qt, sq, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("cannot find metric names: %w", err)
 	}
@@ -1125,21 +1059,11 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
 
-	// Setup search.
-	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return nil, err
-	}
-	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
+	sr, maxSeriesCount, err := vmstorage.GetSearch(qt, sq, deadline.Deadline())
 	if err != nil {
 		return nil, err
 	}
 
-	vmstorage.WG.Add(1)
-	defer vmstorage.WG.Done()
-
-	sr := getStorageSearch()
-	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	type blockRefs struct {
 		brs []blockRef
 	}
@@ -1177,7 +1101,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		blocksRead++
 		if deadline.Exceeded() {
 			putTmpBlocksFile(tbf)
-			putStorageSearch(sr)
+			vmstorage.PutSearch(sr)
 			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
 		br := sr.MetricBlockRef.BlockRef
@@ -1189,7 +1113,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		samples += br.RowsCount()
 		if *maxSamplesPerQuery > 0 && samples > *maxSamplesPerQuery {
 			putTmpBlocksFile(tbf)
-			putStorageSearch(sr)
+			vmstorage.PutSearch(sr)
 			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: increase the -search.maxSamplesPerQuery; "+
 				"reduce time range for the query; use more specific label filters in order to select fewer series", *maxSamplesPerQuery)
 		}
@@ -1198,7 +1122,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		addr, err := tbf.WriteBlockRefData(buf)
 		if err != nil {
 			putTmpBlocksFile(tbf)
-			putStorageSearch(sr)
+			vmstorage.PutSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
 
@@ -1256,7 +1180,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 
 	if err := sr.Error(); err != nil {
 		putTmpBlocksFile(tbf)
-		putStorageSearch(sr)
+		vmstorage.PutSearch(sr)
 		if errors.Is(err, storage.ErrDeadlineExceeded) {
 			return nil, fmt.Errorf("timeout exceeded during the query: %s", deadline.String())
 		}
@@ -1264,13 +1188,13 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	}
 	if err := tbf.Finalize(); err != nil {
 		putTmpBlocksFile(tbf)
-		putStorageSearch(sr)
+		vmstorage.PutSearch(sr)
 		return nil, fmt.Errorf("cannot finalize temporary file: %w", err)
 	}
 	qt.Printf("fetch unique series=%d, blocks=%d, samples=%d, bytes=%d", len(m), blocksRead, samples, tbf.Len())
 
 	var rss Results
-	rss.tr = tr
+	rss.tr = sq.GetTimeRange()
 	rss.deadline = deadline
 	pts := make([]packedTimeseries, len(orderedMetricNames))
 	for i, metricName := range orderedMetricNames {
@@ -1311,35 +1235,6 @@ func getBlockRefsEnd(a []blockRef) uintptr {
 	return uintptr(unsafe.Pointer(unsafe.SliceData(a))) + uintptr(len(a))*unsafe.Sizeof(blockRef{})
 }
 
-func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]storage.TagFilter, maxMetrics int, deadline searchutil.Deadline) ([]*storage.TagFilters, error) {
-	tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
-	for _, tagFilters := range tagFilterss {
-		tfs := storage.NewTagFilters()
-		for i := range tagFilters {
-			tf := &tagFilters[i]
-			if string(tf.Key) == "__graphite__" {
-				query := tf.Value
-				paths, err := vmstorage.SearchGraphitePaths(qt, tr, query, maxMetrics, deadline.Deadline())
-				if err != nil {
-					return nil, fmt.Errorf("error when searching for Graphite paths for query %q: %w", query, err)
-				}
-				if len(paths) >= maxMetrics {
-					return nil, fmt.Errorf("more than %d time series match Graphite query %q; "+
-						"either narrow down the query or increase the corresponding -search.max* command-line flag value; "+
-						"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#resource-usage-limits", maxMetrics, query)
-				}
-				tfs.AddGraphiteQuery(query, paths, tf.IsNegative)
-				continue
-			}
-			if err := tfs.Add(tf.Key, tf.Value, tf.IsNegative, tf.IsRegexp); err != nil {
-				return nil, fmt.Errorf("cannot parse tag filter %s: %w", tf, err)
-			}
-		}
-		tfss = append(tfss, tfs)
-	}
-	return tfss, nil
-}
-
 func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 	// Anchor filter regexp to the beginning of the string as Graphite does.
 	// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
@@ -1366,13 +1261,12 @@ const maxFastAllocBlockSize = 32 * 1024
 func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (metricnamestats.StatsResult, error) {
 	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q", limit, le, matchPattern)
 	defer qt.Done()
-	return vmstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	return vmstorage.VMSelectAPI.GetMetricNamesUsageStats(qt, nil, limit, le, matchPattern, 0)
 }
 
 // ResetMetricNamesStats resets state of metric names usage
 func ResetMetricNamesStats(qt *querytracer.Tracer) error {
 	qt = qt.NewChild("reset metric names usage stats")
 	defer qt.Done()
-	vmstorage.ResetMetricNamesStats(qt)
-	return nil
+	return vmstorage.VMSelectAPI.ResetMetricNamesUsageStats(qt, 0)
 }

app/vmselect/prometheus/federate.qtpl

@@ -2,13 +2,16 @@
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
 {% stripspace %}
 
 // Federate writes rs in /federate format.
 // See https://prometheus.io/docs/prometheus/latest/federation/
-{% func Federate(rs *netstorage.Result) %}
+{% func Federate(rs *netstorage.Result, escapeScheme string) %}
 	{% code
 		values := rs.Values
 		timestamps := rs.Timestamps
@@ -24,10 +27,54 @@
 			See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185
 		{% endcomment %}
 		{% return %}
-	{% endif %}
-	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endif %}
+
+    {% switch escapeScheme %}
+       {% case federateEscapeSchemeUTF8 %}
+          {%= prometheusFederateMetricNameUTF8(&rs.MetricName) %}{% space %}
+
+       {% case federateEscapeSchemeUnderscore %}
+         {%= prometheusFederateMetricNameEscapeUnderscore(&rs.MetricName) %}{% space %}
+
+       {% case "" %}
+         {%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endswitch %}
+
 	{%f= lastValue %}{% space %}
 	{%dl= timestamps[len(timestamps)-1] %}{% newline %}
 {% endfunc %}
 
+{% func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) %}
+	{%s= promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)) %}
+	{% if len(mn.Tags) > 0 %}
+	{
+		{% code tags := mn.Tags %}
+		{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+	}
+	{% endif %}
+{% endfunc %}
+
+{% func prometheusFederateMetricNameUTF8(mn *storage.MetricName) %}
+	{
+        {%= escapePrometheusLabel(mn.MetricGroup) %}
+        {% if len(mn.Tags) > 0 %}
+        ,
+		{% code tags := mn.Tags %}
+		{%= escapePrometheusLabel(tags[0].Key) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%= escapePrometheusLabel(tag.Key) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+        {% endif %}
+	}
+{% endfunc %}
+
+
+
 {% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -9,82 +9,241 @@ import (
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 // Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
-func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:13
+//line app/vmselect/prometheus/federate.qtpl:14
+func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:16
 	values := rs.Values
 	timestamps := rs.Timestamps
 
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	if len(timestamps) == 0 || len(values) == 0 {
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 		return
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	}
-//line app/vmselect/prometheus/federate.qtpl:18
+//line app/vmselect/prometheus/federate.qtpl:21
 	lastValue := values[len(values)-1]
 
-//line app/vmselect/prometheus/federate.qtpl:20
+//line app/vmselect/prometheus/federate.qtpl:23
 	if math.IsNaN(lastValue) {
-//line app/vmselect/prometheus/federate.qtpl:26
+//line app/vmselect/prometheus/federate.qtpl:29
 		return
-//line app/vmselect/prometheus/federate.qtpl:27
+//line app/vmselect/prometheus/federate.qtpl:30
 	}
-//line app/vmselect/prometheus/federate.qtpl:28
-	streamprometheusMetricName(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/federate.qtpl:28
-	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:32
+	switch escapeScheme {
+//line app/vmselect/prometheus/federate.qtpl:33
+	case federateEscapeSchemeUTF8:
+//line app/vmselect/prometheus/federate.qtpl:34
+		streamprometheusFederateMetricNameUTF8(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:34
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:36
+	case federateEscapeSchemeUnderscore:
+//line app/vmselect/prometheus/federate.qtpl:37
+		streamprometheusFederateMetricNameEscapeUnderscore(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:37
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:39
+	case "":
+//line app/vmselect/prometheus/federate.qtpl:40
+		streamprometheusMetricName(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:40
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:41
+	}
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().F(lastValue)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().DL(timestamps[len(timestamps)-1])
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/federate.qtpl:31
-	StreamFederate(qw422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	StreamFederate(qw422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func Federate(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func Federate(rs *netstorage.Result, escapeScheme string) string {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/federate.qtpl:31
-	WriteFederate(qb422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	WriteFederate(qb422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	return qs422016
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+}
+
+//line app/vmselect/prometheus/federate.qtpl:47
+func streamprometheusFederateMetricNameEscapeUnderscore(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:48
+	qw422016.N().S(promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)))
+//line app/vmselect/prometheus/federate.qtpl:49
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:49
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:51
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)))
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:52
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:53
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:54
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:55
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:55
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)))
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:57
+		}
+//line app/vmselect/prometheus/federate.qtpl:57
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:59
+	}
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func writeprometheusFederateMetricNameEscapeUnderscore(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	streamprometheusFederateMetricNameEscapeUnderscore(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:60
+	writeprometheusFederateMetricNameEscapeUnderscore(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:62
+func streamprometheusFederateMetricNameUTF8(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:62
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:64
+	streamescapePrometheusLabel(qw422016, mn.MetricGroup)
+//line app/vmselect/prometheus/federate.qtpl:65
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:65
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:67
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Key)
+//line app/vmselect/prometheus/federate.qtpl:68
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:69
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:70
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:71
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:71
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Key)
+//line app/vmselect/prometheus/federate.qtpl:72
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:73
+		}
+//line app/vmselect/prometheus/federate.qtpl:74
+	}
+//line app/vmselect/prometheus/federate.qtpl:74
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func writeprometheusFederateMetricNameUTF8(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	streamprometheusFederateMetricNameUTF8(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func prometheusFederateMetricNameUTF8(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:76
+	writeprometheusFederateMetricNameUTF8(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:76
 }

app/vmselect/prometheus/federate_test.go

@@ -8,15 +8,15 @@ import (
 )
 
 func TestFederate(t *testing.T) {
-	f := func(rs *netstorage.Result, expectedResult string) {
+	f := func(rs *netstorage.Result, escapeScheme string, expectedResult string) {
 		t.Helper()
-		result := Federate(rs)
+		result := Federate(rs, escapeScheme)
 		if result != expectedResult {
 			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, expectedResult)
 		}
 	}
 
-	f(&netstorage.Result{}, ``)
+	f(&netstorage.Result{}, ``, ``)
 
 	f(&netstorage.Result{
 		MetricName: storage.MetricName{
@@ -39,5 +39,60 @@ func TestFederate(t *testing.T) {
 		},
 		Values:     []float64{1.23},
 		Timestamps: []int64{123},
-	}, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+	}, ``, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte("abc"),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUnderscore, `foo_bar{some__other="value.unchanged!.",qqq="\\",_key="value",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte(`ab"c`),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUTF8, `{"foo.bar","some.!other"="value.unchanged!.","qqq"="\\","!key"="value","ab\"c"="a<b\"\\c"} 1.23 123`+"\n")
+
 }

app/vmselect/prometheus/federate_timing_test.go

@@ -9,16 +9,17 @@ import (
 )
 
 func BenchmarkFederate(b *testing.B) {
+
 	rs := &netstorage.Result{
 		MetricName: storage.MetricName{
-			MetricGroup: []byte("foo_bar_bazaaaa_total"),
+			MetricGroup: []byte("foo_bar_?_._bazaaaa_total"),
 			Tags: []storage.Tag{
 				{
-					Key:   []byte("instance"),
+					Key:   []byte("instance:job"),
 					Value: []byte("foobarbaz:2344"),
 				},
 				{
-					Key:   []byte("job"),
+					Key:   []byte("job.name"),
 					Value: []byte("aaabbbccc"),
 				},
 			},
@@ -27,12 +28,22 @@ func BenchmarkFederate(b *testing.B) {
 		Timestamps: []int64{1234567890},
 	}
 
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var bb bytes.Buffer
-		for pb.Next() {
-			bb.Reset()
-			WriteFederate(&bb, rs)
-		}
-	})
+	f := func(name, escapeScheme string) {
+		b.Helper()
+
+		b.Run(name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				var bb bytes.Buffer
+				for pb.Next() {
+					bb.Reset()
+					WriteFederate(&bb, rs, escapeScheme)
+				}
+			})
+		})
+	}
+
+	f("without escape", "")
+	f("allow-utf-8", federateEscapeSchemeUTF8)
+	f("legacy-underscore", federateEscapeSchemeUnderscore)
 }

app/vmselect/prometheus/prometheus.go

@@ -28,8 +28,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -50,9 +48,6 @@ var (
 		"If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored")
 	maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
 		"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
-
-	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. "+
-		"When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional).")
 	maxFederateSeries       = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
 	maxExportSeries         = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
 	maxTSDBStatusSeries     = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
@@ -108,6 +103,11 @@ func PrettifyQuery(w http.ResponseWriter, r *http.Request) {
 	_ = bw.Flush()
 }
 
+const (
+	federateEscapeSchemeUnderscore = "underscore"
+	federateEscapeSchemeUTF8       = "utf-8"
+)
+
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer federateDuration.UpdateDuration(startTime)
@@ -132,6 +132,21 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
+	// add best-effort format negotiation
+	// modern version of Prometheus always set allow-utf-8 in order to properly parse utf-8 names and labels
+	// prometheus below v3 uses underscore escaping by default and it's the most common standard
+	var escapeScheme string
+	accept := r.Header.Get("Accept")
+	if len(accept) > 0 && strings.Contains(accept, "allow-utf-8") {
+		escapeScheme = federateEscapeSchemeUTF8
+	}
+	// try fallback to legacy underscore escaping if needed for Prometheus only,
+	// it's not widely used after Prometheus v3.0 release
+	// most of the Prometheus scrapers already use allow-utf-8 header
+	isPrometheus := strings.HasPrefix(r.UserAgent(), "Prometheus")
+	if len(escapeScheme) == 0 && isPrometheus {
+		escapeScheme = federateEscapeSchemeUnderscore
+	}
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
@@ -141,7 +156,7 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 			return err
 		}
 		bb := sw.getBuffer(workerID)
-		WriteFederate(bb, rs)
+		WriteFederate(bb, rs, escapeScheme)
 		return sw.maybeFlushBuffer(bb)
 	})
 	if err == nil {
@@ -853,7 +868,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		End:                 start,
 		Step:                step,
 		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           GetMaxUniqueTimeSeries(),
+		MaxSeries:           0, // let vmstorage use maxUniqueTimeseries by default
 		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
 		Deadline:            deadline,
 		MayCache:            mayCache,
@@ -964,7 +979,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		End:                 end,
 		Step:                step,
 		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           GetMaxUniqueTimeSeries(),
+		MaxSeries:           0, // let vmstorage use maxUniqueTimeseries by default
 		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
 		Deadline:            deadline,
 		MayCache:            mayCache,
@@ -1300,43 +1315,6 @@ func (sw *scalableWriter) flush() error {
 	return sw.bw.Flush()
 }
 
-var (
-	maxUniqueTimeseriesValueOnce sync.Once
-	maxUniqueTimeseriesValue     int
-)
-
-// InitMaxUniqueTimeseries init the max metrics limit calculated by available resources.
-// The calculation is split into calculateMaxUniqueTimeSeriesForResource for unit testing.
-func InitMaxUniqueTimeseries(maxConcurrentRequests int) {
-	maxUniqueTimeseriesValueOnce.Do(func() {
-		maxUniqueTimeseriesValue = *maxUniqueTimeseries
-		if maxUniqueTimeseriesValue <= 0 {
-			maxUniqueTimeseriesValue = calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, memory.Remaining())
-		}
-	})
-}
-
-// calculateMaxUniqueTimeSeriesForResource calculate the max metrics limit calculated by available resources.
-func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMemory int) int {
-	if maxConcurrentRequests <= 0 {
-		// This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`.
-		// In such cases, fallback to unlimited.
-		logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests)
-		return 2e9
-	}
-
-	// Calculate the max metrics limit for a single request in the worst-case concurrent scenario.
-	// The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes.
-	mts := remainingMemory / 200 / maxConcurrentRequests
-	logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory)
-	return mts
-}
-
-// GetMaxUniqueTimeSeries returns the max metrics limit calculated by available resources.
-func GetMaxUniqueTimeSeries() int {
-	return maxUniqueTimeseriesValue
-}
-
 // copied from https://github.com/prometheus/common/blob/adea6285c1c7447fcb7bfdeb6abfc6eff893e0a7/model/metric.go#L483
 // it's not possible to use direct import due to increased binary size
 func unescapePrometheusLabelName(name string) string {

app/vmselect/prometheus/prometheus_test.go

@@ -4,7 +4,6 @@ import (
 	"math"
 	"net/http"
 	"reflect"
-	"runtime"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
@@ -230,29 +229,3 @@ func TestGetLatencyOffsetMillisecondsFailure(t *testing.T) {
 	}
 	f("http://localhost?latency_offset=foobar")
 }
-
-func TestCalculateMaxMetricsLimitByResource(t *testing.T) {
-	f := func(maxConcurrentRequest, remainingMemory, expect int) {
-		t.Helper()
-		maxMetricsLimit := calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequest, remainingMemory)
-		if maxMetricsLimit != expect {
-			t.Fatalf("unexpected max metrics limit: got %d, want %d", maxMetricsLimit, expect)
-		}
-	}
-
-	// Skip when GOARCH=386
-	if runtime.GOARCH != "386" {
-		// 8 CPU & 32 GiB
-		f(16, int(math.Round(32*1024*1024*1024*0.4)), 4294967)
-		// 4 CPU & 32 GiB
-		f(8, int(math.Round(32*1024*1024*1024*0.4)), 8589934)
-	}
-
-	// 2 CPU & 4 GiB
-	f(4, int(math.Round(4*1024*1024*1024*0.4)), 2147483)
-
-	// other edge cases
-	f(0, int(math.Round(4*1024*1024*1024*0.4)), 2e9)
-	f(4, 0, 0)
-
-}

app/vmselect/promql/rollup.go

@@ -2439,8 +2439,15 @@ func rollupIntegrate(rfa *rollupFuncArg) float64 {
 		prevTimestamp = timestamp
 		prevValue = v
 	}
-	dt := float64(rfa.currTimestamp-prevTimestamp) / 1e3
-	sum += prevValue * dt
+	// Only extrapolate the last value through to currTimestamp when the time
+	// series has any sample after the lookbehind window. When realNextValue is
+	// NaN the series has effectively ended at prevTimestamp, so accruing area
+	// past it would overcount the integral.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+	if !math.IsNaN(rfa.realNextValue) {
+		dt := float64(rfa.currTimestamp-prevTimestamp) / 1e3
+		sum += prevValue * dt
+	}
 	return sum
 }
 

app/vmselect/promql/rollup_test.go

@@ -1385,10 +1385,65 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		if samplesScanned != 24 {
 			t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
-		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36}
+		// At tEnd=160 the series has no samples past the window (last sample is at
+		// ts=130), so integrate() must not extrapolate prevValue through tEnd.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 0.34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("integrate_past_series_end", func(t *testing.T) {
+		// Constant series of value 1.0 from t=0..3600s (1h) at 60s step.
+		// Query integrate(metric[1h]) across t=0..10800s with 600s step.
+		// For t=0..3600s the window overlap with the data is [0,t], so the integral grows from 0 to 3600 (seconds).
+		// After the series ends, integrate must NOT keep accruing 3600 — it
+		// should taper to 0 once the lookbehind window is entirely past the
+		// last sample.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+		var testValues []int64
+		var testTimestamps []float64
+		for t := int64(0); t <= 3600_000; t += 60_000 {
+			testValues = append(testValues, t)
+			testTimestamps = append(testTimestamps, 1.0)
+		}
+		rc := rollupConfig{
+			Func:               rollupIntegrate,
+			Start:              0,
+			End:                10800_000,
+			Step:               600_000,
+			Window:             3600_000,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		values, _ := rc.Do(nil, testTimestamps, testValues)
+		for i, ti := range rc.Timestamps {
+			v := values[i]
+
+			// For t<=3600s: window overlap is [0,ti], integral equals ti in seconds.
+			if ti <= 3600_000 {
+				expV := float64(ti / 1e3)
+				if v != expV {
+					t.Fatalf("unexpected integrate result at t=%ds, want=%.3f got=%.3f", ti/1e3, expV, v)
+				}
+				continue
+			}
+			// For 3600s<t<7200s: data is partially outside the window, so the
+			// integral shrinks linearly from 3600 to 0 as t approaches 7200s.
+			if ti > 3600_000 && ti < 7200_000 {
+				expV := float64((7200_000 - ti) / 1e3)
+				if v != expV {
+					t.Fatalf("unexpected integrate result at t=%ds, want=%.3f got=%.3f", ti/1e3, expV, v)
+				}
+				continue
+			}
+			if ti >= 7200_000 {
+				// Window entirely past data end: must be NaN.
+				if !math.IsNaN(v) {
+					t.Fatalf("unexpected integrate result at t=%ds, want=NaN got=%.3f", ti/1e3, v)
+				}
+			}
+		}
+	})
 	t.Run("distinct_over_time_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDistinct,

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C7gvW_Zn.js"></script>
+  <script type="module" crossorigin src="./assets/index-U3iNn2Tx.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -1,7 +1,6 @@
 package vmstorage
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -9,12 +8,10 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -23,14 +20,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vminsertapi"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
 )
 
 var (
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
@@ -38,14 +34,8 @@ var (
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
-	snapshotsMaxAge   = flagutil.NewRetentionDuration("snapshotsMaxAge", "3d", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
 	_                 = flag.Duration("snapshotCreateTimeout", 0, "Deprecated: this flag does nothing")
 
-	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
-
-	// DataPath is a path to storage data.
-	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
-
 	_ = flag.Duration("finalMergeDelay", 0, "Deprecated: this flag does nothing")
 	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing")
 	_ = flag.Int("smallMergeConcurrency", 0, "Deprecated: this flag does nothing")
@@ -53,11 +43,17 @@ var (
 	retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+
 		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
 		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
+		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
+	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
+		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
+		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
+		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
 
 	logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
 		"when big number of new series are ingested into VictoriaMetrics")
-	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod. "+
-		"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
+	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod and -futureRetention. "+
+		"When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. "+
 		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
 	maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
@@ -70,6 +66,11 @@ var (
 
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 100e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
 
+	finalDedupScheduleInterval = flag.Duration("storage.finalDedupScheduleCheckInterval", time.Hour, "The interval for checking when final deduplication process should be started."+
+		"Storage unconditionally adds 25% jitter to the interval value on each check evaluation."+
+		" Changing the interval to the bigger values may delay downsampling, deduplication for historical data."+
+		" See also https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
+
 	cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
 	cacheSizeStorageMetricName = flagutil.NewBytes("storage.cacheSizeStorageMetricName", 0, "Overrides max size for storage/metricName cache. "+
@@ -103,32 +104,22 @@ var (
 		"If set to 0 or a negative value, defaults to 1% of allowed memory.")
 )
 
-// CheckTimeRange returns true if the given tr is denied for querying.
-func CheckTimeRange(tr storage.TimeRange) error {
-	if !*denyQueriesOutsideRetention {
-		return nil
-	}
-	minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionPeriod.Milliseconds()
-	if tr.MinTimestamp > minAllowedTimestamp {
-		return nil
-	}
-	return &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention", &tr, retentionPeriod),
-		StatusCode: http.StatusServiceUnavailable,
-	}
+func DataPath() string {
+	return *storageDataPath
 }
 
 // Init initializes vmstorage.
-func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
-	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
-		logger.Fatalf("invalid `-precisionBits`: %s", err)
-	}
-
-	resetResponseCacheIfNeeded = resetCacheIfNeeded
+func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+	storage.SetDedupInterval(*minScrapeInterval)
+	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
+	if *finalDedupScheduleInterval < time.Hour {
+		logger.Fatalf("-storage.finalDedupScheduleCheckInterval cannot be smaller than 1 hour; got %s", *finalDedupScheduleInterval)
+	}
+	storage.SetFinalDedupScheduleInterval(*finalDedupScheduleInterval)
 	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
 	storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
 	storage.SetMetadataStorageSize(metadataStorageSize.IntN())
@@ -147,22 +138,22 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	if *idbPrefillStart > 23*time.Hour {
 		logger.Panicf("-storage.idbPrefillStart cannot exceed 23 hours; got %s", idbPrefillStart)
 	}
-	logger.Infof("opening storage at %q with -retentionPeriod=%s", *DataPath, retentionPeriod)
+	fs.RegisterPathFsMetrics(*storageDataPath)
+	logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
 	startTime := time.Now()
-	WG = syncwg.WaitGroup{}
 	opts := storage.OpenOptions{
-		Retention:             retentionPeriod.Duration(),
-		FutureRetention:       futureRetention.Duration(),
-		MaxHourlySeries:       getMaxHourlySeries(),
-		MaxDailySeries:        getMaxDailySeries(),
-		DisablePerDayIndex:    *disablePerDayIndex,
-		TrackMetricNamesStats: *trackMetricNamesStats,
-		IDBPrefillStart:       *idbPrefillStart,
-		LogNewSeries:          *logNewSeries,
+		Retention:                   retentionPeriod.Duration(),
+		FutureRetention:             futureRetention.Duration(),
+		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
+		MaxHourlySeries:             getMaxHourlySeries(),
+		MaxDailySeries:              getMaxDailySeries(),
+		DisablePerDayIndex:          *disablePerDayIndex,
+		TrackMetricNamesStats:       *trackMetricNamesStats,
+		IDBPrefillStart:             *idbPrefillStart,
+		LogNewSeries:                *logNewSeries,
 	}
-	strg := storage.MustOpenStorage(*DataPath, opts)
-	Storage = strg
-	initStaleSnapshotsRemover(strg)
+	strg := storage.MustOpenStorage(*storageDataPath, opts)
+	vmStorage = newVMStorageSingleNode(strg, vmselectMaxConcurrentRequests, resetCacheIfNeeded)
 
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
@@ -172,152 +163,36 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	rowsCount := tm.SmallRowsCount + tm.BigRowsCount
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
 	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		*DataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
+		*storageDataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
 
 	// register storage metrics
 	storageMetrics = metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
-	})
+	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
-	fs.RegisterPathFsMetrics(*DataPath)
+
+	VMInsertAPI = vmStorage
+	VMSelectAPI = vmStorage
+	GetSearch = vmStorage.GetSearch
+	PutSearch = vmStorage.PutSearch
+	RequestHandler = vmStorage.requestHandler
+	DebugFlush = vmStorage.vms.s.DebugFlush
 }
 
 var storageMetrics *metrics.Set
 
-// Storage is a storage.
-//
-// Every storage call must be wrapped into WG.Add(1) ... WG.Done()
-// for proper graceful shutdown when Stop is called.
-var Storage *storage.Storage
+var (
+	// vmStorageSingleNode is an instance of vmstorage used by vminsert and
+	// vmselect for writing and reading data.
+	vmStorage      *VMStorageSingleNode
+	VMInsertAPI    vminsertapi.API
+	VMSelectAPI    vmselectapi.API
+	GetSearch      func(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (*storage.Search, int, error)
+	PutSearch      func(sr *storage.Search)
+	RequestHandler func(w http.ResponseWriter, r *http.Request) bool
 
-// WG must be incremented before Storage call.
-//
-// Use syncwg instead of sync, since Add is called from concurrent goroutines.
-var WG syncwg.WaitGroup
-
-// resetResponseCacheIfNeeded is a callback for automatic resetting of response cache if needed.
-var resetResponseCacheIfNeeded func(mrs []storage.MetricRow)
-
-// AddRows adds mrs to the storage.
-//
-// The caller should limit the number of concurrent calls to AddRows() in order to limit memory usage.
-func AddRows(mrs []storage.MetricRow) error {
-	if Storage.IsReadOnly() {
-		return errReadOnly
-	}
-	resetResponseCacheIfNeeded(mrs)
-	WG.Add(1)
-	Storage.AddRows(mrs, uint8(*precisionBits))
-	WG.Done()
-	return nil
-}
-
-// AddMetadataRows adds mrs to the storage.
-//
-// The caller should limit the number of concurrent calls to AddMetadataRows() in order to limit memory usage.
-func AddMetadataRows(mms []metricsmetadata.Row) error {
-	if Storage.IsReadOnly() {
-		return errReadOnly
-	}
-	WG.Add(1)
-	Storage.AddMetadataRows(mms)
-	WG.Done()
-	return nil
-}
-
-var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
-
-// RegisterMetricNames registers all the metrics from mrs in the storage.
-func RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow) {
-	WG.Add(1)
-	Storage.RegisterMetricNames(qt, mrs)
-	WG.Done()
-}
-
-// DeleteSeries deletes series matching tfss.
-//
-// Returns the number of deleted series.
-func DeleteSeries(qt *querytracer.Tracer, tfss []*storage.TagFilters, maxMetrics int) (int, error) {
-	WG.Add(1)
-	n, err := Storage.DeleteSeries(qt, tfss, maxMetrics)
-	WG.Done()
-	return n, err
-}
-
-// GetMetricNamesStats returns metric names usage stats with give limit and lte predicate
-func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (metricnamestats.StatsResult, error) {
-	WG.Add(1)
-	r := Storage.GetMetricNamesStats(qt, limit, le, matchPattern)
-	WG.Done()
-	return r, nil
-}
-
-// ResetMetricNamesStats resets state for metric names usage tracker
-func ResetMetricNamesStats(qt *querytracer.Tracer) {
-	WG.Add(1)
-	Storage.ResetMetricNamesStats(qt)
-	WG.Done()
-}
-
-// SearchMetricNames returns metric names for the given tfss on the given tr.
-func SearchMetricNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	metricNames, err := Storage.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
-	WG.Done()
-	return metricNames, err
-}
-
-// SearchLabelNames searches for tag keys matching the given tfss on tr.
-func SearchLabelNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxTagKeys, maxMetrics int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	labelNames, err := Storage.SearchLabelNames(qt, tfss, tr, maxTagKeys, maxMetrics, deadline)
-	WG.Done()
-	return labelNames, err
-}
-
-// SearchLabelValues searches for label values for the given labelName, tfss and
-// tr.
-func SearchLabelValues(qt *querytracer.Tracer, labelName string, tfss []*storage.TagFilters, tr storage.TimeRange, maxLabelValues, maxMetrics int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	labelValues, err := Storage.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
-	WG.Done()
-	return labelValues, err
-}
-
-// SearchTagValueSuffixes returns all the tag value suffixes for the given tagKey and tagValuePrefix on the given tr.
-//
-// This allows implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or similar APIs.
-func SearchTagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxTagValueSuffixes int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	suffixes, err := Storage.SearchTagValueSuffixes(qt, tr, tagKey, tagValuePrefix, delimiter, maxTagValueSuffixes, deadline)
-	WG.Done()
-	return suffixes, err
-}
-
-// SearchGraphitePaths returns all the metric names matching the given Graphite query.
-func SearchGraphitePaths(qt *querytracer.Tracer, tr storage.TimeRange, query []byte, maxPaths int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	paths, err := Storage.SearchGraphitePaths(qt, tr, query, maxPaths, deadline)
-	WG.Done()
-	return paths, err
-}
-
-// GetTSDBStatus returns TSDB status for given filters on the given date.
-func GetTSDBStatus(qt *querytracer.Tracer, tfss []*storage.TagFilters, date uint64, focusLabel string, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) {
-	WG.Add(1)
-	status, err := Storage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics, deadline)
-	WG.Done()
-	return status, err
-}
-
-// GetSeriesCount returns the number of time series in the storage.
-func GetSeriesCount(deadline uint64) (uint64, error) {
-	WG.Add(1)
-	n, err := Storage.GetSeriesCount(deadline)
-	WG.Done()
-	return n, err
-}
+	// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
+	DebugFlush func()
+)
 
 // Stop stops the vmstorage
 func Stop() {
@@ -325,19 +200,24 @@ func Stop() {
 	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
-	logger.Infof("gracefully closing the storage at %s", *DataPath)
+	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime := time.Now()
-	WG.WaitAndBlock()
-	stopStaleSnapshotsRemover()
-	Storage.MustClose()
+	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
 	fs.MustStopDirRemover()
-	logger.Infof("the storage has been stopped")
+	logger.Infof("the vmstorage has been stopped")
 }
 
-// RequestHandler is a storage request handler.
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+func (vmssn *VMStorageSingleNode) requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.requestHandler(w, r)
+}
+
+// requestHandler is a storage request handler.
+// TODO(@rtm0): Move to a separate file, request_handler.go
+func (vms *VMStorage) requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
 		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
@@ -350,8 +230,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			defer activeForceMerges.Dec()
 			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
 			startTime := time.Now()
-			if err := Storage.ForceMergePartitions(partitionNamePrefix); err != nil {
+			if err := vms.s.ForceMergePartitions(partitionNamePrefix); err != nil {
 				logger.Errorf("error in forced merge for partition_prefix=%q: %s", partitionNamePrefix, err)
+				return
 			}
 			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
 		}()
@@ -362,9 +243,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		logger.Infof("flushing storage to make pending data available for reading")
-		Storage.DebugFlush()
+		vms.s.DebugFlush()
 		return true
 	}
+
 	if path == "/internal/log_new_series" {
 		if !httpserver.CheckAuthFlag(w, r, logNewSeriesAuthKey) {
 			return true
@@ -381,7 +263,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		logger.Infof("enabling logging of new series for the next %s. This may increase resource usage during this period.", time.Duration(dealine)*time.Second)
 		endTime := fasttime.UnixTimestamp() + uint64(dealine)
-		Storage.SetLogNewSeriesUntil(endTime)
+		vms.s.SetLogNewSeriesUntil(endTime)
 		fmt.Fprintf(w, `{"status":"success","data":{"logEndTime":%q}}`, time.Unix(int64(endTime), 0))
 		return true
 	}
@@ -403,13 +285,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotName := Storage.MustCreateSnapshot()
+		snapshotName := vms.s.MustCreateSnapshot()
 
 		// Verify whether the client already closed the connection.
 		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
 		if err := r.Context().Err(); err != nil {
 			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
-			if err := deleteSnapshot(snapshotName); err != nil {
+			if err := vms.deleteSnapshot(snapshotName); err != nil {
 				logger.Infof("cannot delete just created snapshot: %s", err)
 				return true
 			}
@@ -425,7 +307,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/list":
 		snapshotsListTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := Storage.MustListSnapshots()
+		snapshots := vms.s.MustListSnapshots()
 		fmt.Fprintf(w, `{"status":"ok","snapshots":[`)
 		if len(snapshots) > 0 {
 			for _, snapshot := range snapshots[:len(snapshots)-1] {
@@ -439,7 +321,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-		if err := deleteSnapshot(snapshotName); err != nil {
+		if err := vms.deleteSnapshot(snapshotName); err != nil {
 			jsonResponseError(w, err)
 			snapshotsDeleteErrorsTotal.Inc()
 			return true
@@ -449,9 +331,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := Storage.MustListSnapshots()
+		snapshots := vms.s.MustListSnapshots()
 		for _, snapshotName := range snapshots {
-			if err := Storage.DeleteSnapshot(snapshotName); err != nil {
+			if err := vms.s.DeleteSnapshot(snapshotName); err != nil {
 				err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err)
 				jsonResponseError(w, err)
 				snapshotsDeleteAllErrorsTotal.Inc()
@@ -465,50 +347,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
-func deleteSnapshot(snapshotName string) error {
-	snapshots := Storage.MustListSnapshots()
-	for _, snName := range snapshots {
-		if snName == snapshotName {
-			if err := Storage.DeleteSnapshot(snName); err != nil {
-				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-			}
-			return nil
-		}
-	}
-	return fmt.Errorf("cannot find snapshot %q", snapshotName)
-}
-
-func initStaleSnapshotsRemover(strg *storage.Storage) {
-	staleSnapshotsRemoverCh = make(chan struct{})
-	if snapshotsMaxAge.Duration() <= 0 {
-		return
-	}
-	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Go(func() {
-		d := timeutil.AddJitterToDuration(time.Second * 11)
-		t := time.NewTicker(d)
-		defer t.Stop()
-		for {
-			select {
-			case <-staleSnapshotsRemoverCh:
-				return
-			case <-t.C:
-			}
-			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
-		}
-	})
-}
-
-func stopStaleSnapshotsRemover() {
-	close(staleSnapshotsRemoverCh)
-	staleSnapshotsRemoverWG.Wait()
-}
-
-var (
-	staleSnapshotsRemoverCh chan struct{}
-	staleSnapshotsRemoverWG sync.WaitGroup
-)
-
 var (
 	activeForceMerges = metrics.NewCounter("vm_active_force_merges")
 
@@ -523,21 +361,30 @@ var (
 	snapshotsDeleteAllErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/delete_all"}`)
 )
 
-func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
+// TODO(@rtm0): Move to metrics.go.
+func (vmssn *VMStorageSingleNode) writeStorageMetrics(w io.Writer) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	vmssn.vms.writeStorageMetrics(w)
+}
+
+// TODO(@rtm0): Move to metrics.go.
+func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
+	strg := vms.s
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
 	idbm := &m.TableMetrics.IndexDBMetrics
 
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetFreeSpace(*DataPath))
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *DataPath), uint64(minFreeDiskSpaceBytes.N))
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_total_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetTotalSpace(*DataPath))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *storageDataPath), uint64(minFreeDiskSpaceBytes.N))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_total_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetTotalSpace(*storageDataPath))
 
 	isReadOnly := 0
 	if strg.IsReadOnly() {
 		isReadOnly = 1
 	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_storage_is_read_only{path=%q}`, *DataPath), uint64(isReadOnly))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_storage_is_read_only{path=%q}`, *storageDataPath), uint64(isReadOnly))
 
 	metrics.WriteGaugeUint64(w, `vm_active_merges{type="storage/inmemory"}`, tm.ActiveInmemoryMerges)
 	metrics.WriteGaugeUint64(w, `vm_active_merges{type="storage/small"}`, tm.ActiveSmallMerges)
@@ -747,6 +594,8 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled`, tm.ScheduledDownsamplingPartitions)
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled_size_bytes`, tm.ScheduledDownsamplingPartitionsSize)
 
+	metrics.WriteGaugeUint64(w, `vm_search_max_unique_timeseries`, uint64(vms.maxUniqueTimeSeriesCalculated))
+
 	metrics.WriteGaugeUint64(w, `vm_metrics_metadata_storage_items`, m.MetadataStorageItemsCurrent)
 	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_size_bytes`, m.MetadataStorageCurrentSizeBytes)
 	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_max_size_bytes`, m.MetadataStorageMaxSizeBytes)

app/vmstorage/vmstorage.go

@@ -0,0 +1,391 @@
+package vmstorage
+
+import (
+	"flag"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression "+
+		"at the cost of precision loss")
+	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be scanned during every query. "+
+		"This allows protecting against heavy queries, which select unexpectedly high number of series. When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional). See also -search.max* command-line flags at vmselect")
+	maxTagKeys = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search. "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxTagValues = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search. "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
+	snapshotsMaxAge              = flagutil.NewRetentionDuration("snapshotsMaxAge", "3d", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
+)
+
+// newVMStorage creates a new instance of of VMStorage.
+//
+// The created VMStorage instance takes ownership of s.
+func newVMStorage(s *storage.Storage, vmselectMaxConcurrentRequests int) *VMStorage {
+	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
+		logger.Fatalf("invalid -precisionBits=%d: %s", *precisionBits, err)
+	}
+
+	maxUniqueTimeseriesCalculated := *maxUniqueTimeseries
+	if maxUniqueTimeseriesCalculated <= 0 {
+		maxUniqueTimeseriesCalculated = calculateMaxUniqueTimeseries(vmselectMaxConcurrentRequests, memory.Remaining())
+	}
+
+	vms := &VMStorage{
+		s:                             s,
+		maxUniqueTimeseries:           *maxUniqueTimeseries,
+		maxUniqueTimeSeriesCalculated: maxUniqueTimeseriesCalculated,
+		staleSnapshotsRemoverCh:       make(chan struct{}),
+	}
+	vms.initStaleSnapshotsRemover()
+	return vms
+}
+
+// calculateMaxUniqueTimeseries calculates the maxUniqueTimeseries based on the
+// available system resources.
+func calculateMaxUniqueTimeseries(maxConcurrentRequests, remainingMemory int) int {
+	if maxConcurrentRequests <= 0 {
+		// This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`.
+		// In such cases, fallback to unlimited.
+		logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests)
+		return 2e9
+	}
+
+	// Calculate the max metrics limit for a single request in the worst-case concurrent scenario.
+	// The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes.
+	mts := remainingMemory / 200 / maxConcurrentRequests
+	logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory)
+	return mts
+}
+
+// VMStorage impelements vmselectapi.API and vminsertapi.API.
+type VMStorage struct {
+	s                             *storage.Storage
+	maxUniqueTimeseries           int
+	maxUniqueTimeSeriesCalculated int
+	staleSnapshotsRemoverCh       chan struct{}
+	staleSnapshotsRemoverWG       sync.WaitGroup
+}
+
+func (vms *VMStorage) initStaleSnapshotsRemover() {
+	if snapshotsMaxAge.Duration() <= 0 {
+		return
+	}
+	snapshotsMaxAgeDuration := snapshotsMaxAge.Duration()
+	vms.staleSnapshotsRemoverWG.Go(func() {
+		d := timeutil.AddJitterToDuration(time.Second * 11)
+		t := time.NewTicker(d)
+		defer t.Stop()
+		for {
+			select {
+			case <-vms.staleSnapshotsRemoverCh:
+				return
+			case <-t.C:
+			}
+			vms.s.MustDeleteStaleSnapshots(snapshotsMaxAgeDuration)
+		}
+	})
+}
+
+func (vms *VMStorage) Stop() {
+	close(vms.staleSnapshotsRemoverCh)
+	vms.staleSnapshotsRemoverWG.Wait()
+	vms.s.MustClose()
+}
+
+// WriteRows writes metric rows to the storage.
+//
+// The caller should limit the number of concurrent calls to WriteRows() in
+// order to limit memory usage.
+func (vms *VMStorage) WriteRows(rows []storage.MetricRow) error {
+	vms.s.AddRows(rows, uint8(*precisionBits))
+	return nil
+}
+
+// WriteMetadata writes metrics metadata to storage.
+//
+// The caller should limit the number of concurrent calls to WriteMetadata() in
+// order to limit memory usage.
+func (vms *VMStorage) WriteMetadata(rows []metricsmetadata.Row) error {
+	vms.s.AddMetadataRows(rows)
+	return nil
+}
+
+// IsReadOnly returns true is the storage is in read-only mode.
+func (vms *VMStorage) IsReadOnly() bool {
+	return vms.s.IsReadOnly()
+}
+
+func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := vms.getMaxMetrics(sq.MaxMetrics)
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(tfss) == 0 {
+		return nil, fmt.Errorf("missing tag filters")
+	}
+	bi := getBlockIterator()
+	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
+	if err := bi.sr.Error(); err != nil {
+		bi.MustClose()
+		return nil, err
+	}
+	return bi, nil
+}
+
+func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
+	if searchQueryLimit <= 0 {
+		return vms.maxUniqueTimeSeriesCalculated
+	}
+	// searchQueryLimit cannot exceed `-search.maxUniqueTimeseries`
+	if vms.maxUniqueTimeseries != 0 && searchQueryLimit > vms.maxUniqueTimeseries {
+		searchQueryLimit = vms.maxUniqueTimeseries
+	}
+	return searchQueryLimit
+}
+
+// blockIterator implements vmselectapi.BlockIterator
+type blockIterator struct {
+	sr storage.Search
+	mb storage.MetricBlock
+}
+
+var blockIteratorsPool sync.Pool
+
+func (bi *blockIterator) MustClose() {
+	bi.sr.MustClose()
+	bi.mb.MetricName = nil
+	bi.mb.Block.Reset()
+	blockIteratorsPool.Put(bi)
+}
+
+func getBlockIterator() *blockIterator {
+	v := blockIteratorsPool.Get()
+	if v == nil {
+		v = &blockIterator{}
+	}
+	return v.(*blockIterator)
+}
+
+func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	if !bi.sr.NextMetricBlock() {
+		return dst, false
+	}
+	mb := bi.mb
+	mb.MetricName = bi.sr.MetricBlockRef.MetricName
+	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
+	dst = mb.Marshal(dst[:0])
+	return dst, true
+}
+
+func (bi *blockIterator) Error() error {
+	return bi.sr.Error()
+}
+
+// SearchMetricNames returns metric names for the given tfss on the given tr.
+func (vms *VMStorage) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(tfss) == 0 {
+		return nil, fmt.Errorf("missing tag filters")
+	}
+	return vms.s.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
+}
+
+// SearchLabelValues searches for label values for the given labelName, tfss and
+// tr.
+func (vms *VMStorage) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	if maxLabelValues <= 0 || maxLabelValues > *maxTagValues {
+		maxLabelValues = *maxTagValues
+	}
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	return vms.s.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
+}
+
+// TagValueSuffixes returns all the tag value suffixes for the given tagKey and
+// tagValuePrefix on the given tr.
+//
+// This allows implementing
+// https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or
+// similar APIs.
+func (vms *VMStorage) TagValueSuffixes(qt *querytracer.Tracer, _, _ uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte,
+	maxSuffixes int, deadline uint64) ([]string, error) {
+	if maxSuffixes <= 0 || maxSuffixes > *maxTagValueSuffixesPerSearch {
+		maxSuffixes = *maxTagValueSuffixesPerSearch
+	}
+	suffixes, err := vms.s.SearchTagValueSuffixes(qt, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(suffixes) >= maxSuffixes {
+		return nil, fmt.Errorf("more than -search.maxTagValueSuffixesPerSearch=%d suffixes returned; "+
+			"either narrow down the search or increase -search.maxTagValueSuffixesPerSearch command-line flag value", maxSuffixes)
+	}
+	return suffixes, nil
+}
+
+// SearchLabelNames searches for tag keys matching the given tfss on tr.
+func (vms *VMStorage) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	if maxLabelNames <= 0 || maxLabelNames > *maxTagKeys {
+		maxLabelNames = *maxTagKeys
+	}
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	return vms.s.SearchLabelNames(qt, tfss, tr, maxLabelNames, maxMetrics, deadline)
+}
+
+func (vms *VMStorage) SeriesCount(_ *querytracer.Tracer, _, _ uint32, deadline uint64) (uint64, error) {
+	return vms.s.GetSeriesCount(deadline)
+}
+
+func (vms *VMStorage) Tenants(_ *querytracer.Tracer, _ storage.TimeRange, _ uint64) ([]string, error) {
+	return nil, nil
+}
+
+// GetTSDBStatus returns TSDB status for given filters on the given date.
+func (vms *VMStorage) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	date := uint64(sq.MinTimestamp) / (24 * 3600 * 1000)
+	return vms.s.GetTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics, deadline)
+}
+
+// DeleteSeries deletes series matching tfss.
+//
+// Returns the number of deleted series.
+func (vms *VMStorage) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return 0, err
+	}
+	if len(tfss) == 0 {
+		return 0, fmt.Errorf("missing tag filters")
+	}
+	return vms.s.DeleteSeries(qt, tfss, maxMetrics)
+}
+
+func (vms *VMStorage) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, _ uint64) error {
+	vms.s.RegisterMetricNames(qt, mrs)
+	return nil
+}
+
+// GetMetricNamesUsageStats returns metric name usage stats.
+func (vms *VMStorage) GetMetricNamesUsageStats(qt *querytracer.Tracer, _ *storage.TenantToken, limit, le int, matchPattern string, _ uint64) (metricnamestats.StatsResult, error) {
+	return vms.s.GetMetricNamesStats(qt, limit, le, matchPattern), nil
+}
+
+// ResetMetricNamesStats resets state for metric names usage tracker
+func (vms *VMStorage) ResetMetricNamesUsageStats(qt *querytracer.Tracer, _ uint64) error {
+	vms.s.ResetMetricNamesStats(qt)
+	return nil
+}
+
+func (vms *VMStorage) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQuery, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]*storage.TagFilters, error) {
+	tfss := make([]*storage.TagFilters, 0, len(sq.TagFilterss))
+	for _, tagFilters := range sq.TagFilterss {
+		tfs := storage.NewTagFilters()
+		for i := range tagFilters {
+			tf := &tagFilters[i]
+			if string(tf.Key) == "__graphite__" {
+				query := tf.Value
+				qtChild := qt.NewChild("searching for series matching __graphite__=%q", query)
+				paths, err := vms.s.SearchGraphitePaths(qtChild, tr, query, maxMetrics, deadline)
+				qtChild.Donef("found %d series", len(paths))
+				if err != nil {
+					return nil, fmt.Errorf("error when searching for Graphite paths for query %q: %w", query, err)
+				}
+				if len(paths) >= maxMetrics {
+					return nil, fmt.Errorf("more than %d time series match Graphite query %q; "+
+						"either narrow down the query or increase the corresponding -search.max* command-line flag value at vmselect nodes; "+
+						"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#resource-usage-limits", maxMetrics, query)
+				}
+				tfs.AddGraphiteQuery(query, paths, tf.IsNegative)
+				continue
+			}
+			if err := tfs.Add(tf.Key, tf.Value, tf.IsNegative, tf.IsRegexp); err != nil {
+				return nil, fmt.Errorf("cannot parse tag filter %s: %w", tf, err)
+			}
+		}
+		tfss = append(tfss, tfs)
+	}
+	return tfss, nil
+}
+
+func (vms *VMStorage) GetMetadataRecords(qt *querytracer.Tracer, _ *storage.TenantToken, limit int, metricName string, _ uint64) ([]*metricsmetadata.Row, error) {
+	return vms.s.GetMetadataRows(qt, limit, metricName), nil
+}
+
+// deleteSnapshot deletes a snapshot by its name.
+//
+// Callers must wrap the call with wg.Add(1)...wg.Done().
+func (vms *VMStorage) deleteSnapshot(snapshotName string) error {
+	snapshots := vms.s.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := vms.s.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}

app/vmstorage/vmstorage_single_node.go

@@ -0,0 +1,213 @@
+package vmstorage
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+// newVMStorageSingleNode creates a new instance of of VMStorage for vmsingle.
+func newVMStorageSingleNode(s *storage.Storage, maxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) *VMStorageSingleNode {
+	vms := newVMStorage(s, maxConcurrentRequests)
+	return &VMStorageSingleNode{
+		vms:                vms,
+		wg:                 syncwg.WaitGroup{},
+		resetCacheIfNeeded: resetCacheIfNeeded,
+	}
+}
+
+type VMStorageSingleNode struct {
+	vms *VMStorage
+
+	// wg is used to wrap every storage call into wg.Add(1) ... wg.Done()
+	// for proper graceful shutdown when Stop is called.
+	//
+	// Use syncwg instead of sync, since Add is called from concurrent
+	// goroutines.
+	wg syncwg.WaitGroup
+
+	// resetCacheIfNeeded is a callback for automatic resetting of response
+	// cache if needed.
+	resetCacheIfNeeded func(mrs []storage.MetricRow)
+}
+
+func (vmssn *VMStorageSingleNode) Stop() {
+	vmssn.wg.WaitAndBlock()
+	vmssn.vms.Stop()
+}
+
+// WriteRows writes metric rows to the storage.
+//
+// Returns an error if the storage is in read-only mode.
+//
+// The caller should limit the number of concurrent calls to WriteRows() in
+// order to limit memory usage.
+func (vmssn *VMStorageSingleNode) WriteRows(rows []storage.MetricRow) error {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+
+	if vmssn.vms.IsReadOnly() {
+		return errReadOnly
+	}
+	vmssn.resetCacheIfNeeded(rows)
+	return vmssn.vms.WriteRows(rows)
+}
+
+// WriteMetadata writes metrics metadata to storage.
+//
+// Returns an error if the storage is in read-only mode.
+//
+// The caller should limit the number of concurrent calls to WriteMetadata() in
+// order to limit memory usage.
+func (vmssn *VMStorageSingleNode) WriteMetadata(rows []metricsmetadata.Row) error {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+
+	if vmssn.vms.IsReadOnly() {
+		return errReadOnly
+	}
+	return vmssn.vms.WriteMetadata(rows)
+}
+
+var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
+
+func (vmssn *VMStorageSingleNode) IsReadOnly() bool {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.IsReadOnly()
+}
+
+func (vmssn *VMStorageSingleNode) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	return nil, fmt.Errorf("not implemented in vmsingle")
+}
+
+// GetSearch sets up an instance of storage search and returns it to the caller
+// along with the max series count that the search can return.
+//
+// This method is not part of the vmselectapi.API and must only be used by
+// vmsingle HTTP handlers.
+//
+// Callers of this method must call PutSearch() once the search instance is not
+// needed anymore.
+func (vmssn *VMStorageSingleNode) GetSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (*storage.Search, int, error) {
+	vmssn.wg.Add(1)
+
+	tr := sq.GetTimeRange()
+	maxMetrics := vmssn.vms.getMaxMetrics(sq.MaxMetrics)
+	tfss, err := vmssn.vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		vmssn.wg.Done()
+		return nil, 0, err
+	}
+
+	sr := getSearch()
+	maxSeriesCount := sr.Init(qt, vmssn.vms.s, tfss, tr, sq.MaxMetrics, deadline)
+	return sr, maxSeriesCount, nil
+}
+
+// PutSearch resets the search once it is not needed anymore and puts it aside
+// for future reuse.
+//
+// This method is not part of the vmselectapi.API and must only be used by
+// vmsingle HTTP handlers.
+//
+// The method must only be used on search instances that have been created with
+// GetSearch().
+func (vmssn *VMStorageSingleNode) PutSearch(sr *storage.Search) {
+	putSearch(sr)
+	vmssn.wg.Done()
+}
+
+func getSearch() *storage.Search {
+	v := ssPool.Get()
+	if v == nil {
+		return &storage.Search{}
+	}
+	return v.(*storage.Search)
+}
+
+func putSearch(sr *storage.Search) {
+	sr.MustClose()
+	ssPool.Put(sr)
+}
+
+var ssPool sync.Pool
+
+func (vmssn *VMStorageSingleNode) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.SearchMetricNames(qt, sq, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.LabelNames(qt, sq, maxLabelNames, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.SeriesCount(qt, accountID, projectID, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.Tenants(qt, tr, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.DeleteSeries(qt, sq, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.RegisterMetricNames(qt, mrs, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.ResetMetricNamesUsageStats(qt, deadline)
+}
+
+func (vmssn *VMStorageSingleNode) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	vmssn.wg.Add(1)
+	defer vmssn.wg.Done()
+	return vmssn.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
+}

app/vmstorage/vmstorage_test.go

@@ -0,0 +1,62 @@
+package vmstorage
+
+import (
+	"math"
+	"strconv"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+func TestCalculateMaxMetricsLimitByResource(t *testing.T) {
+	f := func(maxConcurrentRequest, remainingMemory, expect int) {
+		t.Helper()
+		maxMetricsLimit := calculateMaxUniqueTimeseries(maxConcurrentRequest, remainingMemory)
+		if maxMetricsLimit != expect {
+			t.Fatalf("unexpected max metrics limit: got %d, want %d", maxMetricsLimit, expect)
+		}
+	}
+
+	// 64-bit architectures support memory sizes > 4GB.
+	if strconv.IntSize == 64 {
+		// 8 CPU & 32 GiB
+		f(16, int(math.Round(32*1024*1024*1024*0.4)), 4294967)
+		// 4 CPU & 32 GiB
+		f(8, int(math.Round(32*1024*1024*1024*0.4)), 8589934)
+	}
+
+	// 2 CPU & 4 GiB
+	f(4, int(math.Round(4*1024*1024*1024*0.4)), 2147483)
+
+	// other edge cases
+	f(0, int(math.Round(4*1024*1024*1024*0.4)), 2e9)
+	f(4, 0, 0)
+
+}
+
+func TestGetMaxMetrics(t *testing.T) {
+	originalMaxUniqueTimeSeries := *maxUniqueTimeseries
+	defer func() {
+		*maxUniqueTimeseries = originalMaxUniqueTimeSeries
+		fs.MustRemoveDir(t.Name())
+	}()
+
+	maxConcurrentRequests := 2 * cgroup.AvailableCPUs()
+	f := func(searchQueryLimit, storageMaxUniqueTimeseries, expect int) {
+		t.Helper()
+		*maxUniqueTimeseries = storageMaxUniqueTimeseries
+		s := storage.MustOpenStorage(t.Name(), storage.OpenOptions{})
+		vms := newVMStorage(s, maxConcurrentRequests)
+		defer vms.Stop()
+		maxMetrics := vms.getMaxMetrics(searchQueryLimit)
+		if maxMetrics != expect {
+			t.Fatalf("unexpected max metrics: got %d, want %d", maxMetrics, expect)
+		}
+	}
+
+	f(0, 1e6, 1e6)
+	f(2e6, 0, 2e6)
+	f(2e6, 1e6, 1e6)
+}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.3 AS build-web-stage
+FROM golang:1.26.4 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/api/types.ts

@@ -3,6 +3,7 @@ export interface MetricBase {
   metric: {
     [key: string]: string;
   };
+  nullTimestamps?: number[];
 }
 
 export interface MetricResult extends MetricBase {

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -16,6 +16,7 @@ export interface ChartTooltipProps {
   point: { top: number, left: number };
   unit?: string;
   statsFormatted?: SeriesItemStatsFormatted;
+  description?: ReactNode;
   isSticky?: boolean;
   info?: ReactNode;
   marker?: string;
@@ -34,6 +35,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   unit = "",
   info,
   statsFormatted,
+  description,
   isSticky,
   marker,
   duplicateCount = 0,
@@ -173,6 +175,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           ))}
         </table>
       )}
+      {description && <p className="vm-chart-tooltip__description">{description}</p>}
       {info && <p className="vm-chart-tooltip__info">{info}</p>}
     </div>
   ), u.root);

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -143,4 +143,10 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     word-break: break-all;
     white-space: pre-wrap;
   }
+
+  &__description {
+    word-break: break-word;
+    white-space: pre-wrap;
+    opacity: 0.85;
+  }
 }

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts

@@ -1,7 +1,7 @@
-export const seriesFetchedWarning = `No match! 
+export const seriesFetchedWarning = `No match!
 This query hasn't selected any time series from database.
 Either the requested metrics are missing in the database,
 or there is a typo in series selector.`;
 
 export const partialWarning = `The shown results are marked as PARTIAL.
-The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`;
+The result is marked as partial if one or more storage nodes failed to respond to the query.`;

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -71,7 +71,7 @@ const RulesHeader = ({
           <TextField
             label="Search"
             value={search}
-            placeholder="Filter by rule, name or labels"
+            placeholder="Filter by group or rule name"
             startIcon={<SearchIcon />}
             onChange={onChangeSearch}
           />

app/vmui/packages/vmui/src/components/Main/Tabs/TabItem.tsx

@@ -1,6 +1,5 @@
 import { Component, FC, Ref } from "preact/compat";
 import classNames from "classnames";
-import { getCssVariable } from "../../../utils/theme";
 import { TabItemType } from "./Tabs";
 import TabItemWrapper from "./TabItemWrapper";
 import "./style.scss";
@@ -8,7 +7,6 @@ import "./style.scss";
 interface TabItemProps {
   activeItem: string
   item: TabItemType
-  color?: string
   onChange?: (value: string) => void
   activeNavRef: Ref<Component>
   isNavLink?: boolean
@@ -17,7 +15,6 @@ interface TabItemProps {
 const TabItem: FC<TabItemProps> = ({
   activeItem,
   item,
-  color = getCssVariable("color-primary"),
   activeNavRef,
   onChange,
   isNavLink
@@ -35,7 +32,6 @@ const TabItem: FC<TabItemProps> = ({
       })}
       isNavLink={isNavLink}
       to={item.value}
-      style={{ color: color }}
       onClick={createHandlerClickTab(item.value)}
       ref={activeItem === item.value ? activeNavRef : undefined}
     >

app/vmui/packages/vmui/src/components/Main/Tabs/TabItemWrapper.tsx

@@ -6,7 +6,6 @@ interface TabItemWrapperProps {
   to: string
   isNavLink?: boolean
   className: string
-  style: { color: string }
   children: ReactNode
   onClick: () => void
 }

app/vmui/packages/vmui/src/components/Main/Tabs/Tabs.tsx

@@ -1,6 +1,5 @@
 import { Component, FC, useRef, useState } from "preact/compat";
 import { ReactNode, useEffect } from "react";
-import { getCssVariable } from "../../../utils/theme";
 import TabItem from "./TabItem";
 import "./style.scss";
 import useWindowSize from "../../../hooks/useWindowSize";
@@ -15,7 +14,6 @@ export interface TabItemType {
 interface TabsProps {
   activeItem: string
   items: TabItemType[]
-  color?: string
   onChange?: (value: string) => void
   indicatorPlacement?: "bottom" | "top"
   isNavLink?: boolean
@@ -24,7 +22,6 @@ interface TabsProps {
 const Tabs: FC<TabsProps> = ({
   activeItem,
   items,
-  color = getCssVariable("color-primary"),
   onChange,
   indicatorPlacement = "bottom",
   isNavLink,
@@ -48,14 +45,13 @@ const Tabs: FC<TabsProps> = ({
         activeItem={activeItem}
         item={item}
         onChange={onChange}
-        color={color}
         activeNavRef={activeNavRef}
         isNavLink={isNavLink}
       />
     ))}
     <div
       className="vm-tabs__indicator"
-      style={{ ...indicatorPosition, borderColor: color }}
+      style={{ ...indicatorPosition }}
     />
   </div>;
 };

app/vmui/packages/vmui/src/components/Main/Tabs/style.scss

@@ -14,7 +14,7 @@
     align-items: center;
     justify-content: center;
     padding: $padding-global $padding-small;
-    color: inherit;
+    color: $color-primary;
     text-decoration: none;
     text-transform: capitalize;
     font-size: inherit;
@@ -46,5 +46,6 @@
     position: absolute;
     border-bottom: 2px solid;
     transition: width 200ms ease, left 300ms cubic-bezier(0.280, 0.840, 0.420, 1);
+    border-color: $color-primary;
   }
 }

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -4,7 +4,7 @@ import { ChartTooltipProps } from "../../components/Chart/ChartTooltip/ChartTool
 import { SeriesItem } from "../../types";
 import dayjs from "dayjs";
 import { DATE_FULL_TIMEZONE_FORMAT } from "../../constants/date";
-import { formatPrettyNumber, getMetricName } from "../../utils/uplot";
+import { getMetricName } from "../../utils/uplot";
 import { MetricResult } from "../../api/types";
 import useEventListener from "../useEventListener";
 
@@ -15,19 +15,65 @@ interface LineTooltipHook {
   unit?: string;
 }
 
+// Pixel proximity for detecting hover over null-timestamp X markers drawn at chart bottom.
+const NULL_HOVER_PROX = 8;
+// Half the visual marker height in CSS px (BASE_POINT_SIZE * 1.4 / 2 from scatter.ts).
+// scatter.ts lifts the marker center by this amount above yMin so the icon sits inside
+// the plot area; the hover y-anchor must match that offset.
+const NULL_MARKER_HALF_CSS = 2.8;
+
+interface NullHover {
+  seriesIdx: number;
+  timestamp: number;
+}
+
+const findNullHover = (u: uPlot): NullHover | null => {
+  const cursorLeft = u.cursor.left ?? -1;
+  const cursorTop = u.cursor.top ?? -1;
+  if (cursorLeft < 0 || cursorTop < 0) return null;
+
+  const scaleY = u.scales["1"];
+  if (!scaleY || scaleY.min == null) return null;
+  const yPos = u.valToPos(scaleY.min, "1") - NULL_MARKER_HALF_CSS;
+  if (Math.abs(cursorTop - yPos) > NULL_HOVER_PROX) return null;
+
+  let best: { seriesIdx: number; timestamp: number; dist: number } | null = null;
+  for (let s = 1; s < u.series.length; s++) {
+    const seriesItem = u.series[s] as SeriesItem;
+    if (!seriesItem.show) continue;
+    const nullTs = seriesItem.nullTimestamps;
+    if (!nullTs || !nullTs.length) continue;
+
+    for (let i = 0; i < nullTs.length; i++) {
+      const t = nullTs[i];
+      const xPos = u.valToPos(t, "x");
+      const dist = Math.abs(cursorLeft - xPos);
+      if (dist < NULL_HOVER_PROX && (best === null || dist < best.dist)) {
+        best = { seriesIdx: s, timestamp: t, dist };
+      }
+    }
+  }
+  return best ? { seriesIdx: best.seriesIdx, timestamp: best.timestamp } : null;
+};
+
+const NULL_DESCRIPTION = "\"null\" can be a staleness marker or an actual NaN/null value produced by exporter.";
+
 const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
+  const [nullTooltip, setNullTooltip] = useState<NullHover | null>(null);
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
 
   const resetTooltips = () => {
     setStickyToolTips([]);
     setTooltipIdx({ seriesIdx: -1, dataIdx: -1 });
+    setNullTooltip(null);
   };
 
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setTooltipIdx(prev => ({ ...prev, dataIdx }));
+    setNullTooltip(findNullHover(u));
   };
 
   const seriesFocus = (u: uPlot, sidx: (number | null)) => {
@@ -35,7 +81,36 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     setTooltipIdx(prev => ({ ...prev, seriesIdx }));
   };
 
+  const getNullTooltipProps = (hit: NullHover): ChartTooltipProps => {
+    const { seriesIdx, timestamp } = hit;
+    const metricItem = metrics[seriesIdx - 1];
+    const seriesItem = series[seriesIdx] as SeriesItem;
+
+    const groups = new Set(metrics.map(m => m.group));
+    const group = metricItem?.group || 0;
+
+    const yMin = u?.scales?.[1]?.min ?? 0;
+    const point = {
+      top: u ? u.valToPos(yMin, seriesItem?.scale || "1") - NULL_MARKER_HALF_CSS : 0,
+      left: u ? u.valToPos(timestamp, "x") : 0,
+    };
+
+    return {
+      u,
+      id: `null_${seriesIdx}_${timestamp}`,
+      title: groups.size > 1 ? `Query ${group}` : "",
+      dates: [dayjs(timestamp * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT)],
+      value: "null",
+      info: getMetricName(metricItem, seriesItem),
+      description: NULL_DESCRIPTION,
+      marker: `${seriesItem?.stroke}`,
+      point,
+    };
+  };
+
   const getTooltipProps = useCallback((): ChartTooltipProps => {
+    if (nullTooltip) return getNullTooltipProps(nullTooltip);
+
     const { seriesIdx, dataIdx } = tooltipIdx;
     const metricItem = metrics[seriesIdx - 1];
     const seriesItem = series[seriesIdx] as SeriesItem;
@@ -44,8 +119,6 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     const group = metricItem?.group || 0;
 
     const value = u?.data?.[seriesIdx]?.[dataIdx] || 0;
-    const min = u?.scales?.[1]?.min || 0;
-    const max = u?.scales?.[1]?.max || 1;
     const date = u?.data?.[0]?.[dataIdx] || 0;
 
     let duplicateCount = 1;
@@ -80,13 +153,13 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       id: `${seriesIdx}_${dataIdx}`,
       title: groups.size > 1 ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
-      value: formatPrettyNumber(value, min, max),
+      value: value.toLocaleString("en-US", { maximumFractionDigits: 20 }),
       info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit]);
+  }, [u, tooltipIdx, metrics, series, unit, nullTooltip]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;
@@ -101,8 +174,9 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   };
 
   useEffect(() => {
-    setShowTooltip(tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1);
-  }, [tooltipIdx]);
+    const normalHit = tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1;
+    setShowTooltip(normalHit || nullTooltip !== null);
+  }, [tooltipIdx, nullTooltip]);
 
   useEventListener("click", handleClick);
 

app/vmui/packages/vmui/src/pages/CustomPanel/utils.test.ts

@@ -20,7 +20,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123");
   });
 
   it("should return a valid CSV string for multiple metric entries with values", () => {
@@ -43,7 +43,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2\n456,value4");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123\n456,value4,1623949200,456");
   });
 
   it("should handle metric entries with multiple values field", () => {
@@ -58,7 +58,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123-456,values");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123-456,values,-,-");
   });
 
   it("should handle a combination of metric entries with value and values", () => {
@@ -81,6 +81,19 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,first\n456-789,second");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,first,1623945600,123\n456-789,second,-,-");
   });
+
+  it("should return value and timestamp if metric field is empty", () => {
+    const data: InstantMetricResult[] = [
+      {
+        value: [1623945600, "123"],
+        group: 0,
+        metric: {}
+      },
+    ];
+    const result = convertMetricsDataToCSV(data);
+    expect(result).toBe("__timestamp__,__value__\n1623945600,123");
+  });
+
 });

app/vmui/packages/vmui/src/pages/CustomPanel/utils.ts

@@ -3,16 +3,22 @@ import { getColumns, MetricCategory } from "../../hooks/useSortedCategories";
 import { formatValueToCSV } from "../../utils/csv";
 
 const getHeaders = (data: InstantMetricResult[]): string => {
-  return getColumns(data).map(({ key }) => key).join(",");
+  const metricHeaders = getColumns(data).map(({ key }) => key);
+  return [...metricHeaders, "__timestamp__", "__value__"].join(",");
 };
 
 const getRows = (data: InstantMetricResult[], headers: MetricCategory[]) => {
-  return data?.map(d => headers.map(c => formatValueToCSV(d.metric[c.key] || "-")).join(","));
+  return data?.map(d => {
+    const metricPart = headers.map(c => formatValueToCSV(d.metric[c.key] || "-"));
+    const timestamp = d.value ? formatValueToCSV(String(d.value[0])) : "-";
+    const value = d.value ? formatValueToCSV(d.value[1]) : "-";
+    return [...metricPart, timestamp, value].join(",");
+  });
 };
 
 export const convertMetricsDataToCSV = (data: InstantMetricResult[]): string => {
+  if (!data.length) return "";
   const headers = getHeaders(data);
-  if (!headers.length) return "";
   const rows = getRows(data, getColumns(data));
   return [headers, ...rows].join("\n");
 };

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -149,15 +149,21 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
             const pointsToTake = shouldDownsample ? maxPointsPerSeries : totalPoints;
             const step = shouldDownsample ? totalPoints / maxPointsPerSeries : 1;
 
-            const values: [number, number][] = Array.from({ length: pointsToTake }, (_, i) => {
+            const values: [number, number][] = new Array(pointsToTake);
+            const nullTimestamps: number[] = [];
+            for (let i = 0; i < pointsToTake; i++) {
               const idx = shouldDownsample ? Math.floor(i * step) : i;
-              return [rawTimestamps[idx] / 1000, rawValues[idx]];
-            });
+              const ts = rawTimestamps[idx] / 1000;
+              const raw = rawValues[idx];
+              if (raw === null) nullTimestamps.push(ts);
+              values[i] = [ts, raw as number];
+            }
 
             tempData.push({
               group: counter,
               metric: jsonLine.metric,
               values,
+              nullTimestamps,
             } as MetricBase);
           }
 

app/vmui/packages/vmui/src/state/customPanel/reducer.test.ts

@@ -0,0 +1,35 @@
+import { afterEach, describe, expect, it, vi, type Mock } from "vitest";
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+
+vi.mock("../../utils/storage", () => ({
+  getFromStorage: vi.fn(),
+  saveToStorage: vi.fn(),
+}));
+
+describe("customPanel reducer", () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+    vi.resetModules();
+  });
+
+  it("persists reduceMemUsage under its own storage key", async () => {
+    const { reducer, initialCustomPanelState } = await import("./reducer");
+
+    reducer(initialCustomPanelState, { type: "TOGGLE_REDUCE_MEM_USAGE" });
+
+    expect(saveToStorage).toHaveBeenCalledWith("REDUCE_MEM_USAGE", true);
+    expect(saveToStorage).not.toHaveBeenCalledWith("TABLE_COMPACT", true);
+  });
+
+  it("hydrates reduceMemUsage from storage", async () => {
+    const getFromStorageMock = getFromStorage as Mock;
+    getFromStorageMock.mockImplementation((key: string) => {
+      if (key === "REDUCE_MEM_USAGE") return true;
+      return undefined;
+    });
+
+    const { initialCustomPanelState } = await import("./reducer");
+
+    expect(initialCustomPanelState.reduceMemUsage).toBe(true);
+  });
+});

app/vmui/packages/vmui/src/state/customPanel/reducer.ts

@@ -35,7 +35,7 @@ export const initialCustomPanelState: CustomPanelState = {
   isTracingEnabled: false,
   seriesLimits: limitsStorage ? JSON.parse(limitsStorage) : DEFAULT_MAX_SERIES,
   tableCompact: getFromStorage("TABLE_COMPACT") as boolean || false,
-  reduceMemUsage: false
+  reduceMemUsage: getFromStorage("REDUCE_MEM_USAGE") as boolean || false
 };
 
 export function reducer(state: CustomPanelState, action: CustomPanelAction): CustomPanelState {
@@ -69,7 +69,7 @@ export function reducer(state: CustomPanelState, action: CustomPanelAction): Cus
         tableCompact: !state.tableCompact
       };
     case "TOGGLE_REDUCE_MEM_USAGE":
-      saveToStorage("TABLE_COMPACT", !state.reduceMemUsage);
+      saveToStorage("REDUCE_MEM_USAGE", !state.reduceMemUsage);
       return {
         ...state,
         reduceMemUsage: !state.reduceMemUsage

app/vmui/packages/vmui/src/types/uplot.ts

@@ -11,6 +11,7 @@ export interface SeriesItem extends Series {
     statsFormatted: SeriesItemStatsFormatted;
     median: number;
     hasAlias?: boolean;
+    nullTimestamps?: number[];
 }
 
 export interface HideSeriesArgs {

app/vmui/packages/vmui/src/utils/storage.ts

@@ -7,6 +7,7 @@ export const ALL_STORAGE_KEYS = [
   "SERIES_LIMITS",
   "LEGEND_AUTO_COLLAPSE",
   "TABLE_COMPACT",
+  "REDUCE_MEM_USAGE",
   "TIMEZONE",
   "DISABLED_DEFAULT_TIMEZONE",
   "THEME",

app/vmui/packages/vmui/src/utils/uplot/scatter.ts

@@ -103,6 +103,28 @@ export const drawPoints = (u: uPlot, seriesIdx: number) => {
     u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
     u.ctx.strokeStyle = u.ctx.fillStyle;
     u.ctx.stroke(squaresPath);
+
+    const nullTs = (series as unknown as { nullTimestamps?: number[] }).nullTimestamps;
+    if (nullTs && nullTs.length) {
+      const xSize = BASE_POINT_SIZE * 1.4 * uPlot.pxRatio;
+      const xHalf = xSize / 2;
+      // Lift the marker by half its size so the entire icon sits inside the plot area
+      // (yMin maps to the plot's bottom edge, so centering on it would clip the lower half).
+      const cy = valToPosY(yMin, scaleY, yDim, yOff) - xHalf;
+      const xPath = new Path2D();
+      for (let i = 0; i < nullTs.length; i++) {
+        const t = nullTs[i];
+        if (t < xMin || t > xMax) continue;
+        const cx = valToPosX(t, scaleX, xDim, xOff);
+        xPath.moveTo(cx - xHalf, cy - xHalf);
+        xPath.lineTo(cx + xHalf, cy + xHalf);
+        xPath.moveTo(cx + xHalf, cy - xHalf);
+        xPath.lineTo(cx - xHalf, cy + xHalf);
+      }
+      u.ctx.lineWidth = 1.6 * uPlot.pxRatio;
+      u.ctx.strokeStyle = u.ctx.fillStyle;
+      u.ctx.stroke(xPath);
+    }
   };
 
   uPlot.orient(u, seriesIdx, orientCallback);

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -38,6 +38,7 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
       paths: isRawQuery ? drawPoints : undefined,
+      nullTimestamps: d.nullTimestamps,
       ...getSeriesStatistics(d),
     };
   };

apptest/app.go

@@ -22,6 +22,7 @@ var (
 	vminsertAddrRE              = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
 	vminsertClusterNativeAddrRE = regexp.MustCompile(`started TCP clusternative server at "(.*:\d{1,5})"`)
 	vmselectAddrRE              = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
+	vmauthHttpListenAddrRE      = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
 )
 
 // app represents an instance of some VictoriaMetrics server (such as vmstorage,

apptest/model.go

@@ -79,24 +79,25 @@ type PrometheusWriteQuerier interface {
 
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
-	Tenant         string
-	Timeout        string
-	Start          string
-	End            string
-	Time           string
-	Step           string
-	ExtraFilters   []string
-	ExtraLabels    []string
-	Trace          string
-	ReduceMemUsage string
-	MaxLookback    string
-	LatencyOffset  string
-	Format         string
-	NoCache        string
-	Headers        http.Header
-	From           string
-	Until          string
-	StorageStep    string
+	Tenant              string
+	Timeout             string
+	Start               string
+	End                 string
+	Time                string
+	Step                string
+	ExtraFilters        []string
+	ExtraLabels         []string
+	Trace               string
+	ReduceMemUsage      string
+	MaxLookback         string
+	LatencyOffset       string
+	Format              string
+	NoCache             string
+	Headers             http.Header
+	From                string
+	Until               string
+	StorageStep         string
+	DenyPartialResponse string
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -132,6 +133,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("from", qos.From)
 	addNonEmpty("until", qos.Until)
 	addNonEmpty("storage_step", qos.StorageStep)
+	addNonEmpty("deny_partial_response", qos.DenyPartialResponse)
 
 	return uv
 }

apptest/testcase.go

@@ -88,19 +88,11 @@ func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
 }
 
 // MustStartVmsingle is a test helper function that starts an instance of
-// vmsingle located at ../../bin/victoria-metrics-race and fails the test if the app
-// fails to start.
+// vmsingle (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
 	tc.t.Helper()
-	return tc.MustStartVmsingleAt(instance, "../../bin/victoria-metrics-race", flags)
-}
 
-// MustStartVmsingleAt is a test helper function that starts an instance of
-// vmsingle and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string) *Vmsingle {
-	tc.t.Helper()
-
-	app, err := StartVmsingleAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmsingle(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -109,19 +101,11 @@ func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string)
 }
 
 // MustStartVmstorage is a test helper function that starts an instance of
-// vmstorage located at ../../bin/vmstorage-race and fails the test if the app fails
-// to start.
+// vmstorage (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
 	tc.t.Helper()
-	return tc.MustStartVmstorageAt(instance, "../../bin/vmstorage-race", flags)
-}
 
-// MustStartVmstorageAt is a test helper function that starts an instance of
-// vmstorage and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags []string) *Vmstorage {
-	tc.t.Helper()
-
-	app, err := StartVmstorageAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmstorage(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -130,7 +114,7 @@ func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags [
 }
 
 // MustStartVmselect is a test helper function that starts an instance of
-// vmselect and fails the test if the app fails to start.
+// vmselect (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
 	tc.t.Helper()
 
@@ -290,10 +274,8 @@ func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
 // tests usually come paired with corresponding vmsingle tests.
 type ClusterOptions struct {
 	Vmstorage1Instance string
-	Vmstorage1Binary   string
 	Vmstorage1Flags    []string
 	Vmstorage2Instance string
-	Vmstorage2Binary   string
 	Vmstorage2Flags    []string
 	VminsertInstance   string
 	VminsertFlags      []string
@@ -305,15 +287,8 @@ type ClusterOptions struct {
 func (tc *TestCase) MustStartCluster(opts *ClusterOptions) *Vmcluster {
 	tc.t.Helper()
 
-	if opts.Vmstorage1Binary == "" {
-		opts.Vmstorage1Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage1 := tc.MustStartVmstorageAt(opts.Vmstorage1Instance, opts.Vmstorage1Binary, opts.Vmstorage1Flags)
-
-	if opts.Vmstorage2Binary == "" {
-		opts.Vmstorage2Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage2 := tc.MustStartVmstorageAt(opts.Vmstorage2Instance, opts.Vmstorage2Binary, opts.Vmstorage2Flags)
+	vmstorage1 := tc.MustStartVmstorage(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
 
 	opts.VminsertFlags = append(opts.VminsertFlags, []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),

apptest/testcase_legacy.go

@@ -0,0 +1,50 @@
+package apptest
+
+// MustStartVmsingle_v1_132_0 is a test helper function that starts an instance
+// of vmsingle-v1.132.0 (last version that uses legacy index) and fails the test
+// if the app fails to start.
+func (tc *TestCase) MustStartVmsingle_v1_132_0(instance string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingle_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartVmstorage_v1_132_0 is a test helper function that starts an instance
+// of vmstorage-v1.132.0 (last version that uses legacy index)  and fails the
+// test if the app fails to start.
+func (tc *TestCase) MustStartVmstorage_v1_132_0(instance string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorage_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartCluster_v1_132_0 starts a cluster with vmstorage-v1.132.0 with
+// custom flags.
+func (tc *TestCase) MustStartCluster_v1_132_0(opts *ClusterOptions) *Vmcluster {
+	tc.t.Helper()
+
+	vmstorage1 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
+
+	opts.VminsertFlags = append(opts.VminsertFlags, []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	}...)
+	vminsert := tc.MustStartVminsert(opts.VminsertInstance, opts.VminsertFlags)
+
+	opts.VmselectFlags = append(opts.VmselectFlags, []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	}...)
+	vmselect := tc.MustStartVmselect(opts.VmselectInstance, opts.VmselectFlags)
+
+	return &Vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
+}

apptest/tests/legacy_indexdb_test.go

@@ -2,7 +2,6 @@ package tests
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"slices"
 	"testing"
@@ -11,11 +10,6 @@ import (
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
-var (
-	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
-	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
-)
-
 type testLegacyDeleteSeriesOpts struct {
 	startLegacySUT func() at.PrometheusWriteQuerier
 	startNewSUT    func() at.PrometheusWriteQuerier
@@ -31,7 +25,7 @@ func TestLegacySingleDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.maxStalenessInterval=1m",
@@ -64,15 +58,13 @@ func TestLegacyClusterDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -191,7 +183,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - start legacy vmsingle
 	// - insert data1
-	// - confirm that metric names and samples are searcheable
+	// - confirm that metric names and samples are searchable
 	// - stop legacy vmsingle
 	const step = 24 * 3600 * 1000 // 24h
 	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
@@ -204,12 +196,12 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 	opts.stopLegacySUT()
 
 	// - start new vmsingle
-	// - confirm that data1 metric names and samples are searcheable
+	// - confirm that data1 metric names and samples are searchable
 	// - delete data1
-	// - confirm that data1 metric names and samples are not searcheable anymore
+	// - confirm that data1 metric names and samples are not searchable anymore
 	// - insert data2 (same metric names, different dates)
-	// - confirm that metric names become searcheable again
-	// - confirm that data1 samples are not searchable and data2 samples are searcheable
+	// - confirm that metric names become searchable again
+	// - confirm that data1 samples are not searchable and data2 samples are searchable
 
 	newSUT := opts.startNewSUT()
 	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
@@ -230,7 +222,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - restart new vmsingle
 	// - confirm that metric names still searchable, data1 samples are not
-	//   searchable, and data2 samples are searcheable
+	//   searchable, and data2 samples are searchable
 
 	opts.stopNewSUT()
 	newSUT = opts.startNewSUT()
@@ -255,7 +247,7 @@ func TestLegacySingleBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -298,15 +290,13 @@ func TestLegacyClusterBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -583,7 +573,7 @@ func TestLegacySingleDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -618,15 +608,13 @@ func TestLegacyClusterDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",

apptest/tests/replication_test.go

@@ -1015,35 +1015,42 @@ func testGroupSkipSlowReplicas(tc *apptest.TestCase, opts *testGroupReplicationO
 func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOpts) {
 	t := tc.T()
 
-	assertSeries := func(app *apptest.Vmselect, wantPartial bool) {
+	assertSeries := func(app *apptest.Vmselect, denyPartialResponse string, want *apptest.PrometheusAPIV1SeriesResponse) {
 		t.Helper()
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
 				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{
-					Start: "2024-01-01T00:00:00Z",
-					End:   "2024-01-31T00:00:00Z",
+					Start:               "2024-01-01T00:00:00Z",
+					End:                 "2024-01-31T00:00:00Z",
+					DenyPartialResponse: denyPartialResponse,
 				}).Sort()
 			},
-			Want: &apptest.PrometheusAPIV1SeriesResponse{
-				Status:    "success",
-				IsPartial: wantPartial,
-			},
+			Want: want,
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data", "Error"),
 			},
 		})
 	}
 
-	mustReturnPartialResponse := true
-	mustReturnFullResponse := false
+	allowPartialResponse := ""
+	denyPartialResponse := "1"
+
+	mustReturnPartialResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: true,
+	}
+	mustReturnFullResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+	}
 
 	// All vmstorage replicas are available so both vmselects must return full
 	// response.
-	assertSeries(opts.c.vmselect, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorage nodes in first group.
 	//
@@ -1053,10 +1060,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// about the replication factor and therefore they must still be able to
 	// return full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, opts.groupRF-1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorages in the remaining groups.
 	//
@@ -1066,10 +1073,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < len(opts.c.storageGroups); g++ {
 		opts.c.storageGroups[g].stopNodes(tc, opts.groupRF-1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstorage in the first group.
 	//
@@ -1077,10 +1084,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// because it is unaware of replication across groups. vmselectGroupGlobalRF
 	// will continue retuning full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in remaining globarRF-1 groups.
 	//
@@ -1089,19 +1096,56 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < opts.globalRF-1; g++ {
 		opts.c.storageGroups[g].stopNodes(tc, 1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in one more group.
 	//
 	// vmselectGroupGlobalRF must now return partial dataset.
 	opts.c.storageGroups[opts.globalRF].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	// Stop all the remaining vmstorage nodes except a single node.
+	//
+	// At this point vmselects still must be able to return partial response
+	// because at least one vmstorage node has successfully returned results.
+	n := len(opts.c.storageGroups[0].vmstorages)
+	opts.c.storageGroups[0].stopNodes(tc, n-1)
+	for g := 1; g < len(opts.c.storageGroups); g++ {
+		n := len(opts.c.storageGroups[g].vmstorages)
+		opts.c.storageGroups[g].stopNodes(tc, n)
+	}
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	mustReturnUnavailableError := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "error",
+		ErrorType: "503",
+	}
+
+	// vmselects must return an error for the same request when partial
+	// responses are denied explicitly.
+	assertSeries(opts.c.vmselect, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+
+	// Stop the last remaining vmstorage node.
+	//
+	// vmselects must return an error when there are no successful vmstorage
+	// responses.
+	opts.c.storageGroups[0].stopNodes(tc, 1)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnUnavailableError)
 }
 
 // TestClusterReplication_PartialResponseMultitenant checks how vmselect handles some

apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/meta.json

@@ -0,0 +1,51 @@
+{
+	"ulid": "01JFJBS3YP1SHZ3PJQ6HK76EC3",
+	"minTime": 1734709200000,
+	"maxTime": 1734709320000,
+	"stats": {
+		"numSamples": 400,
+		"numSeries": 100,
+		"numChunks": 100
+	},
+	"compaction": {
+		"level": 1,
+		"sources": [
+			"01JFJBS3YP1SHZ3PJQ6HK76EC3"
+		],
+		"parents": [
+			{
+				"ulid": "00000000000000000000000000",
+				"minTime": 0,
+				"maxTime": 0
+			}
+		],
+		"hints": [
+			"from-out-of-order"
+		]
+	},
+	"version": 1,
+	"out_of_order": false,
+	"thanos": {
+		"labels": {},
+		"downsample": {
+			"resolution": 0
+		},
+		"source": "receive",
+		"segment_files": [
+			"000001"
+		],
+		"files": [
+			{
+				"rel_path": "chunks/000001",
+				"size_bytes": 4808
+			},
+			{
+				"rel_path": "index",
+				"size_bytes": 55021
+			},
+			{
+				"rel_path": "meta.json"
+			}
+		]
+	}
+}