docs/changelog: cut v1.136.0

Exploit uint64set data structure peculiarities (adjacent elements are
stored in
64KiB buckets) to optimize metricIDCache memory footprint.

As the result the cache utilizes 87% less memory and is up to 90%
faster. See
[benchstat.txt](https://github.com/user-attachments/files/25294076/benchstat.txt).

Follow-up for #10388 and #10346.

Thanks to @valyala for the optimization idea.

---------

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -71,7 +71,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
         run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/check-licenses.yml

@@ -21,9 +21,11 @@ jobs:
         id: go
         uses: actions/setup-go@v6
         with:
-          go-version: stable
+          go-version-file: 'go.mod'
           cache: false
 
+      - run: go version
+
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
@@ -32,7 +34,7 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
+          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-
 
       - name: Check License
         run: make check-licenses

.github/workflows/codeql-analysis-go.yml

@@ -36,7 +36,8 @@ jobs:
         uses: actions/setup-go@v6
         with:
           cache: false
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
@@ -46,7 +47,7 @@ jobs:
             ~/go/bin
             ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v4

.github/workflows/test.yml

@@ -42,8 +42,9 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
 
+      - run: go version
 
       - name: Cache golangci-lint
         uses: actions/cache@v4
@@ -51,7 +52,7 @@ jobs:
           path: |
             ~/.cache/golangci-lint
             ~/go/bin
-          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
+          key: golangci-lint-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('.golangci.yml') }}
 
       - name: Run check-all
         run: |
@@ -81,7 +82,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Run tests
         run: GOGC=10 make ${{ matrix.scenario}}
@@ -91,8 +93,8 @@ jobs:
         with:
           files: ./coverage.txt
 
-  integration:
-    name: integration
+  apptest:
+    name: apptest
     runs-on: ubuntu-latest
 
     steps:
@@ -107,7 +109,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
-      - name: Run integration tests
-        run: make integration-test
+      - name: Run app tests
+        run: make apptest

.github/workflows/vmui.yml

@@ -34,33 +34,39 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v6
 
-      - name: Setup Node
-        uses: actions/setup-node@v6
+      - name: Cache node_modules
+        id: cache
+        uses: actions/cache@v5
         with:
-          node-version: '24.x'
+          path: app/vmui/packages/vmui/node_modules
+          key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
+          restore-keys: |
+            vmui-deps-${{ runner.os }}-
 
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make vmui-install
 
       - name: Run lint
         id: lint
         run: make vmui-lint
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run tests
         id: test
         run: make vmui-test
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run typecheck
         id: typecheck
         run: make vmui-typecheck
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
         uses: ataylorme/eslint-annotate-action@v3

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2026 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.7.2
+GOLANGCI_LINT_VERSION := 2.9.0
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -443,7 +443,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet -tags 'synctest' ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -452,28 +452,25 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test -tags 'synctest' ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -tags 'synctest' -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test -tags 'synctest' ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
-
-integration-test:
-	$(MAKE) apptest
+	GOARCH=386 go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 apptest:
 	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
 	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
 
-integration-test-legacy: victoria-metrics vmbackup vmrestore
+apptest-legacy: victoria-metrics vmbackup vmrestore
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
 	VERSION=v1.132.0; \
@@ -490,17 +487,17 @@ integration-test-legacy: victoria-metrics vmbackup vmrestore
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
-	go test -bench=. ./app/...
+	go test -run=NO_TESTS -bench=. ./lib/...
+	go test -run=NO_TESTS -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
-	CGO_ENABLED=0 go test -bench=. ./app/...
+	CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./lib/...
+	CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./app/...
 
 vendor-update:
 	go get -u ./lib/...
 	go get -u ./app/...
-	go mod tidy -compat=1.24
+	go mod tidy -compat=1.26
 	go mod vendor
 
 app-local:
@@ -524,7 +521,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run --build-tags 'synctest'
 
 install-golangci-lint:
 	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)

README.md

@@ -16,16 +16,21 @@
   <img src="docs/victoriametrics/logo.webp" width="300" alt="VictoriaMetrics logo">
 </picture>
 
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
+VictoriaMetrics is a fast, cost-effective, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
 
 Here are some resources and information about VictoriaMetrics:
 
-- Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
-- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
-- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
-- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/)
-- Changelog: [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics)
-- Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)
+- **Case studies**: [Grammarly, Roblox, Wix, Spotify,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
+- **Available**: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), Docker images on [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- **Deployment types**: [Single-node version](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) under [Apache License 2.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
+- **Getting started:** Read [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/) and follow the
+  [quick start guide](https://docs.victoriametrics.com/victoriametrics/quick-start/).
+- **Community**: [Slack](https://slack.victoriametrics.com/) (join via [Slack Inviter](https://slack.victoriametrics.com/)), [X (Twitter)](https://x.com/VictoriaMetrics), [YouTube](https://www.youtube.com/@VictoriaMetrics). See full list [here](https://docs.victoriametrics.com/victoriametrics/#community-and-contributions).
+- **Changelog**: Project evolves fast - check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
+- **Enterprise support:** [Contact us](mailto:info@victoriametrics.com) for commercial support with additional [enterprise features](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+- **Enterprise releases:** Enterprise and [long-term support releases (LTS)](https://docs.victoriametrics.com/victoriametrics/lts-releases/) are publicly available and can be evaluated for free
+  using a [free trial license](https://victoriametrics.com/products/enterprise/trial/).
+- **Security:** we achieved [security certifications](https://victoriametrics.com/security/) for Database Software Development and Software-Based Monitoring Services.
 
 Yes, we open-source both the single-node VictoriaMetrics and the cluster version.
 

app/victoria-metrics/main.go

@@ -134,6 +134,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/victoria-metrics/self_scraper.go

@@ -29,11 +29,9 @@ var selfScraperWG sync.WaitGroup
 
 func startSelfScraper() {
 	selfScraperStopCh = make(chan struct{})
-	selfScraperWG.Add(1)
-	go func() {
-		defer selfScraperWG.Done()
+	selfScraperWG.Go(func() {
 		selfScraper(*selfScrapeInterval)
-	}()
+	})
 }
 
 func stopSelfScraper() {

app/vmagent/main.go

@@ -245,6 +245,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/vmagent/remotewrite/client.go

@@ -202,14 +202,10 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
+		return float64(concurrency)
 	})
-	for i := 0; i < concurrency; i++ {
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			c.runWorker()
-		}()
+	for range concurrency {
+		c.wg.Go(c.runWorker)
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }

app/vmagent/remotewrite/pendingseries.go

@@ -48,11 +48,7 @@ func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Boo
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
-	ps.periodicFlusherWG.Add(1)
-	go func() {
-		defer ps.periodicFlusherWG.Done()
-		ps.periodicFlusher()
-	}()
+	ps.periodicFlusherWG.Go(ps.periodicFlusher)
 	return &ps
 }
 

app/vmagent/remotewrite/relabel.go

@@ -9,19 +9,18 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"go.yaml.in/yaml/v3"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
-		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
+	unparsedLabelsGlobal    = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
@@ -139,6 +138,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
+
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
@@ -176,19 +176,9 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+// isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
+	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.

app/vmagent/remotewrite/remotewrite.go

@@ -59,7 +59,7 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -176,13 +176,6 @@ func Init() {
 		})
 	}
 
-	if *queues > maxQueues {
-		*queues = maxQueues
-	}
-	if *queues <= 0 {
-		*queues = 1
-	}
-
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
@@ -215,9 +208,7 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
-	configReloaderWG.Add(1)
-	go func() {
-		defer configReloaderWG.Done()
+	configReloaderWG.Go(func() {
 		for {
 			select {
 			case <-configReloaderStopCh:
@@ -227,7 +218,7 @@ func Init() {
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
 		}
-	}()
+	})
 }
 
 func dropDanglingQueues() {
@@ -267,17 +258,6 @@ func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
-	if maxInmemoryBlocks / *queues > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * *queues
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
 	rwctxIdx := make([]int, len(urls))
 	if retryMaxTime.String() != "" {
@@ -292,7 +272,7 @@ func initRemoteWriteCtxs(urls []string) {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
 		rwctxIdx[i] = i
 	}
 
@@ -558,11 +538,9 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	// Push metadata to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
 				if forceDropSamplesOnFailure {
@@ -571,7 +549,7 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 				}
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -603,15 +581,13 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -633,13 +609,11 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 		if len(shard) == 0 {
 			continue
 		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
+		wg.Go(func() {
+			if !rwctx.TryPushTimeSeries(shard, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx, shard)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -848,7 +822,7 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
@@ -863,6 +837,23 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
+	queuesSize := queues.GetOptionalArg(argIdx)
+	if queuesSize > maxQueues {
+		queuesSize = maxQueues
+	} else if queuesSize <= 0 {
+		queuesSize = 1
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks/queuesSize > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * queuesSize
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
@@ -880,16 +871,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, *queues, sanitizedURL)
+	c.init(argIdx, queuesSize, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
 	rd := roundDigits.GetOptionalArg(argIdx)
-	pssLen := *queues
+	pssLen := queuesSize
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
 		// since every pendingSeries can saturate up to a single CPU.
@@ -1089,7 +1080,7 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
 	}()
 
 	if len(labelsGlobal) > 0 {
-		// Make a copy of tss before adding extra labels in order to prevent
+		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
 		v = tssPool.Get().(*[]prompb.TimeSeries)

app/vmalert/config/types.go

@@ -76,11 +76,14 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
-		fields, _ := q.GetStatsByFields()
-		for i := range fields {
+		labels, err := q.GetStatsLabels()
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
+		}
+		for i := range labels {
 			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
 			// making the result meaningless and may lead to cardinality issues.
-			if fields[i] == "_time" {
+			if labels[i] == "_time" {
 				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 			}
 		}

app/vmalert/main.go

@@ -81,9 +81,7 @@ absolute path to all .tpl files in root.
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
-var (
-	extURL *url.URL
-)
+var extURL *url.URL
 
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
@@ -161,7 +159,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("failed to init: %s", err)
+		logger.Fatalf("failed to create manager: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)

app/vmalert/manager_test.go

@@ -65,11 +65,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
@@ -79,7 +77,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }
@@ -261,7 +259,7 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	for i, r := range a.Rules {
 		got, want := r, b.Rules[i]
 		if a.CreateID() != b.CreateID() {
-			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+			t.Fatalf("expected to have rule %d; got %d", want.ID(), got.ID())
 		}
 		if err := rule.CompareRules(t, want, got); err != nil {
 			t.Fatalf("comparison error: %s", err)

app/vmalert/notifier/alert.go

@@ -80,14 +80,15 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Type     string
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+	Type      string
+	Labels    map[string]string
+	Value     float64
+	Expr      string
+	AlertID   uint64
+	GroupID   uint64
+	ActiveAt  time.Time
+	For       time.Duration
+	IsPartial bool
 }
 
 var tplHeaders = []string{
@@ -101,6 +102,7 @@ var tplHeaders = []string{
 	"{{ $groupID := .GroupID }}",
 	"{{ $activeAt := .ActiveAt }}",
 	"{{ $for := .For }}",
+	"{{ $isPartial := .IsPartial }}",
 }
 
 // ExecTemplate executes the Alert template for given

app/vmalert/notifier/alertmanager.go

@@ -14,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -172,11 +171,6 @@ const alertManagerPath = "/api/v2/alerts"
 func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
 	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration,
 ) (*AlertManager, error) {
-
-	if err := httputil.CheckURL(alertManagerURL); err != nil {
-		return nil, fmt.Errorf("invalid alertmanager URL: %w", err)
-	}
-
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig

app/vmalert/notifier/config_watcher_test.go

@@ -212,18 +212,16 @@ consul_sd_configs:
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/init.go

@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -229,6 +229,9 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
+		if err := httputil.CheckURL(addr); err != nil {
+			return nil, fmt.Errorf("invalid notifier.url %q: %w", addr, err)
+		}
 		addr = strings.TrimSuffix(addr, "/")
 		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
@@ -266,7 +269,7 @@ func GetTargets() map[TargetType][]Target {
 	if getActiveNotifiers == nil {
 		return nil
 	}
-	var targets = make(map[TargetType][]Target)
+	targets := make(map[TargetType][]Target)
 	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()
@@ -287,7 +290,7 @@ func GetTargets() map[TargetType][]Target {
 }
 
 // Send sends alerts to all active notifiers
-func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
+func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) chan error {
 	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompb.Label, 0, len(alerts))
 	// apply global relabel config first without modifying original alerts in alerts
@@ -300,17 +303,18 @@ func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string
 		lblss = append(lblss, lbls)
 	}
 
-	errGr := new(vmalertutil.ErrGroup)
 	wg := sync.WaitGroup{}
 	activeNotifiers := getActiveNotifiers()
+	errCh := make(chan error, len(activeNotifiers))
+	defer close(errCh)
 	for i := range activeNotifiers {
 		nt := activeNotifiers[i]
 		wg.Go(func() {
 			if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
-				errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
+				errCh <- fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err)
 			}
 		})
 	}
 	wg.Wait()
-	return errGr
+	return errCh
 }

app/vmalert/notifier/init_test.go

@@ -55,9 +55,9 @@ func TestInitNegative(t *testing.T) {
 		*blackHole = oldBlackHole
 	}()
 
-	f := func(path, addr string, bh bool) {
+	f := func(path string, addr []string, bh bool) {
 		*configPath = path
-		*addrs = flagutil.ArrayString{addr}
+		*addrs = flagutil.ArrayString(addr)
 		*blackHole = bh
 		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
@@ -65,9 +65,12 @@ func TestInitNegative(t *testing.T) {
 	}
 
 	// *configPath, *addrs and *blackhole are mutually exclusive
-	f("/dummy/path", "127.0.0.1", false)
-	f("/dummy/path", "", true)
-	f("", "127.0.0.1", true)
+	f("/dummy/path", []string{"127.0.0.1"}, false)
+	f("/dummy/path", []string{}, true)
+	f("", []string{"127.0.0.1"}, true)
+	// addr cannot be ""
+	f("", []string{""}, false)
+	f("", []string{"127.0.0.1", ""}, false)
 }
 
 func TestBlackHole(t *testing.T) {
@@ -202,7 +205,9 @@ alert_relabel_configs:
 		},
 	}
 	errG := Send(context.Background(), firingAlerts, nil)
-	if errG.Err() != nil {
-		t.Fatalf("unexpected error when sending alerts: %s", err)
+	for err := range errG {
+		if err != nil {
+			t.Errorf("unexpected error when sending alerts: %s", err)
+		}
 	}
 }

app/vmalert/rule/alerting.go

@@ -346,6 +346,8 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		ls.processed[l.Name] = l.Value
 	}
 
+	// labels only support limited templating variables,
+	// including `labels`, `value` and `expr`, to avoid breaking alert states or causing cardinality issue with results
 	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
@@ -387,11 +389,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, err
 		}
 		alertID := hash(ls.processed)
-		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
-		if err != nil {
-			return nil, err
-		}
-		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
+		a := ar.newAlert(s, time.Time{}, ls.processed, nil) // initial alert
 
 		prevT := time.Time{}
 		for i := range s.Values {
@@ -407,8 +405,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
-				// re-template the annotations as active timestamp is changed
-				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -463,7 +459,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
+	isPartial := isPartialResponse(res)
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartial)
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err
@@ -489,7 +486,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				at = a.ActiveAt
 			}
 		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls, isPartial)
 		if err != nil {
 			// only set error in current state, but do not break alert processing
 			curState.Err = err
@@ -607,16 +604,17 @@ func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.
 	return ls, nil
 }
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet, isPartial bool) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
-		Value:    m.Values[0],
-		Type:     ar.Type.String(),
-		Labels:   ls.origin,
-		Expr:     ar.Expr,
-		AlertID:  hash(ls.processed),
-		GroupID:  ar.GroupID,
-		ActiveAt: activeAt,
-		For:      ar.For,
+		Value:     m.Values[0],
+		Type:      ar.Type.String(),
+		Labels:    ls.origin,
+		Expr:      ar.Expr,
+		AlertID:   hash(ls.processed),
+		GroupID:   ar.GroupID,
+		ActiveAt:  activeAt,
+		For:       ar.For,
+		IsPartial: isPartial,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
@@ -820,7 +818,9 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 	expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
 		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
 
-	res, _, err := q.Query(ctx, expr, ts)
+	// query ALERTS_FOR_STATE at `ts-1s` instead `ts` to avoid retrieving data written in the current run,
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335
+	res, _, err := q.Query(ctx, expr, ts.Add(-1*time.Second))
 	if err != nil {
 		return fmt.Errorf("failed to execute restore query %q: %w ", expr, err)
 	}

app/vmalert/rule/alerting_synctest_test.go

@@ -0,0 +1,106 @@
+//go:build synctest
+
+package rule
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
+// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
+// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
+func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
+	// wrap into synctest because of time manipulations
+	synctest.Test(t, func(t *testing.T) {
+		fq := &datasource.FakeQuerier{}
+
+		ar := &AlertingRule{
+			Name: "TestActiveAtPreservation",
+			Labels: map[string]string{
+				"test_query_in_label": `{{ "static_value" }}`,
+			},
+			Annotations: map[string]string{
+				"description": "Alert active since {{ $activeAt }}",
+			},
+			alerts: make(map[uint64]*notifier.Alert),
+			q:      fq,
+			state: &ruleState{
+				entries: make([]StateEntry, 10),
+			},
+		}
+
+		// Mock query result - return empty result to make suppress_for_mass_alert = false
+		// (no need to add anything to fq for empty result)
+
+		// Add a metric that should trigger the alert
+		fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
+
+		// First execution - creates new alert
+		ts1 := time.Now()
+		_, err := ar.exec(context.TODO(), ts1, 0)
+		if err != nil {
+			t.Fatalf("unexpected error on first exec: %s", err)
+		}
+
+		if len(ar.alerts) != 1 {
+			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
+		}
+
+		firstAlert := ar.GetAlerts()[0]
+		// Verify first execution: activeAt should be ts1 and annotation should reflect it
+		if !firstAlert.ActiveAt.Equal(ts1) {
+			t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
+		}
+
+		// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
+		expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
+		if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
+			t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
+		}
+
+		// Second execution - should preserve activeAt in annotation
+
+		// Ensure different timestamp with different seconds
+		// sleep is non-blocking thanks to synctest
+		time.Sleep(2 * time.Second)
+		ts2 := time.Now()
+		_, err = ar.exec(context.TODO(), ts2, 0)
+		if err != nil {
+			t.Fatalf("unexpected error on second exec: %s", err)
+		}
+
+		// Get the alert again (should be the same alert)
+		if len(ar.alerts) != 1 {
+			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
+		}
+		secondAlert := ar.GetAlerts()[0]
+
+		// Critical test: activeAt should still be ts1, not ts2
+		if !secondAlert.ActiveAt.Equal(ts1) {
+			t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
+		}
+
+		// Critical test: annotation should still contain ts1 time, not ts2
+		if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
+			t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
+		}
+
+		// Additional verification: annotation should NOT contain ts2 time
+		ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
+		if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
+			t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
+		}
+
+		// Verify query template in labels still works (this would fail if query templates were broken)
+		if firstAlert.Labels["test_query_in_label"] != "static_value" {
+			t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
+		}
+	})
+}

app/vmalert/rule/alerting_test.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"testing/synctest"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -664,7 +663,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -684,7 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
-			Annotations: map[string]string{"activeAt": "1000"},
+			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
 			ActiveAt:    time.Unix(1, 0),
 			Start:       time.Unix(5, 0),
@@ -705,7 +704,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -1120,7 +1119,7 @@ func TestAlertingRuleLimit_Success(t *testing.T) {
 }
 
 func TestAlertingRule_Template(t *testing.T) {
-	f := func(rule *AlertingRule, metrics []datasource.Metric, alertsExpected map[uint64]*notifier.Alert) {
+	f := func(rule *AlertingRule, metrics []datasource.Metric, isResponsePartial bool, alertsExpected map[uint64]*notifier.Alert) {
 		t.Helper()
 
 		fakeGroup := Group{
@@ -1133,6 +1132,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			entries: make([]StateEntry, 10),
 		}
 		fq.Add(metrics...)
+		fq.SetPartialResponse(isResponsePartial)
 
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -1163,7 +1163,7 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 1, "instance", "foo"),
 		metricWithValueAndLabels(t, 1, "instance", "bar"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
 				"summary": `common: Too high connection number for "foo"`,
@@ -1192,14 +1192,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}".{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 			"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
 		metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 			Labels: map[string]string{
 				alertNameLabel:       "override label",
@@ -1207,7 +1207,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "foo",
 			},
 			Annotations: map[string]string{
-				"summary":     `first: Too high connection number for "foo"`,
+				"summary":     `first: Too high connection number for "foo".`,
 				"description": `override: It is 2 connections for "foo"`,
 			},
 		},
@@ -1218,7 +1218,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "bar",
 			},
 			Annotations: map[string]string{
-				"summary":     `second: Too high connection number for "bar"`,
+				"summary":     `second: Too high connection number for "bar".`,
 				"description": `override: It is 10 connections for "bar"`,
 			},
 		},
@@ -1231,7 +1231,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}.{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1239,7 +1239,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			alertNameLabel, "originAlertname",
 			alertGroupNameLabel, "originGroupname",
 			"instance", "foo"),
-	}, map[uint64]*notifier.Alert{
+	}, true, map[uint64]*notifier.Alert{
 		hash(map[string]string{
 			alertNameLabel:        "OriginLabels",
 			"exported_alertname":  "originAlertname",
@@ -1255,7 +1255,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":            "foo",
 			},
 			Annotations: map[string]string{
-				"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+				"summary": `Alert "originAlertname(originGroupname)" for instance foo. WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.`,
 			},
 		},
 	})
@@ -1385,7 +1385,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"group":         "vmalert",
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
-		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1395,7 +1395,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)
@@ -1478,95 +1478,3 @@ func TestAlertingRule_QueryTemplateInLabels(t *testing.T) {
 		t.Fatalf("expected 'suppress_for_mass_alert' label to be 'true' or 'false', got '%s'", suppressLabel)
 	}
 }
-
-// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
-// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
-// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
-func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
-	// wrap into synctest because of time manipulations
-	synctest.Test(t, func(t *testing.T) {
-		fq := &datasource.FakeQuerier{}
-
-		ar := &AlertingRule{
-			Name: "TestActiveAtPreservation",
-			Labels: map[string]string{
-				"test_query_in_label": `{{ "static_value" }}`,
-			},
-			Annotations: map[string]string{
-				"description": "Alert active since {{ $activeAt }}",
-			},
-			alerts: make(map[uint64]*notifier.Alert),
-			q:      fq,
-			state: &ruleState{
-				entries: make([]StateEntry, 10),
-			},
-		}
-
-		// Mock query result - return empty result to make suppress_for_mass_alert = false
-		// (no need to add anything to fq for empty result)
-
-		// Add a metric that should trigger the alert
-		fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
-
-		// First execution - creates new alert
-		ts1 := time.Now()
-		_, err := ar.exec(context.TODO(), ts1, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on first exec: %s", err)
-		}
-
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-
-		firstAlert := ar.GetAlerts()[0]
-		// Verify first execution: activeAt should be ts1 and annotation should reflect it
-		if !firstAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
-		}
-
-		// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
-		expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
-		if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
-		}
-
-		// Second execution - should preserve activeAt in annotation
-
-		// Ensure different timestamp with different seconds
-		// sleep is non-blocking thanks to synctest
-		time.Sleep(2 * time.Second)
-		ts2 := time.Now()
-		_, err = ar.exec(context.TODO(), ts2, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on second exec: %s", err)
-		}
-
-		// Get the alert again (should be the same alert)
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-		secondAlert := ar.GetAlerts()[0]
-
-		// Critical test: activeAt should still be ts1, not ts2
-		if !secondAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
-		}
-
-		// Critical test: annotation should still contain ts1 time, not ts2
-		if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Additional verification: annotation should NOT contain ts2 time
-		ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
-		if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
-			t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Verify query template in labels still works (this would fail if query templates were broken)
-		if firstAlert.Labels["test_query_in_label"] != "static_value" {
-			t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
-		}
-	})
-}

app/vmalert/rule/group.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -374,7 +375,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 	g.infof("started")
 
-	eval := func(ctx context.Context, ts time.Time) {
+	eval := func(ctx context.Context, ts time.Time) time.Time {
 		g.metrics.iterationTotal.Inc()
 
 		start := time.Now()
@@ -382,7 +383,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
 			g.LastEvaluation = start
-			return
+			return ts
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
@@ -396,6 +397,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
 		g.LastEvaluation = start
+		return ts
 	}
 
 	evalCtx, cancel := context.WithCancel(ctx)
@@ -404,7 +406,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	eval(evalCtx, evalTS)
+	realEvalTS := eval(evalCtx, evalTS)
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
@@ -412,7 +414,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {
-		err := g.restore(ctx, rr, evalTS, *remoteReadLookBack)
+		err := g.restore(ctx, rr, realEvalTS, *remoteReadLookBack)
 		if err != nil {
 			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
 		}
@@ -755,6 +757,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return fmt.Errorf("rule %q: failed to execute: %w", r, err)
 	}
 
+	var errG vmalertutil.ErrGroup
 	if e.Rw != nil {
 		pushToRW := func(tss []prompb.TimeSeries) error {
 			var lastErr error
@@ -766,20 +769,26 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 			return lastErr
 		}
 		if err := pushToRW(tss); err != nil {
-			return err
+			errG.Add(err)
 		}
 	}
 
 	ar, ok := r.(*AlertingRule)
 	if !ok {
-		return nil
+		return errG.Err()
 	}
 
 	alerts := ar.alertsToSend(resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
-		return nil
+		return errG.Err()
 	}
 
-	errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
-	return errGr.Err()
+	notifierErr := notifier.Send(ctx, alerts, e.notifierHeaders)
+	for err := range notifierErr {
+		if err != nil {
+			errG.Add(fmt.Errorf("rule %q: notifier failure: %w", r, err))
+		}
+	}
+
+	return errG.Err()
 }

app/vmalert/rule/rule_test.go

@@ -65,17 +65,15 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	r := &AlertingRule{state: &ruleState{entries: make([]StateEntry, 20)}}
 	const workers = 50
 	const iterations = 100
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func() {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Go(func() {
 			for i := 0; i < iterations; i++ {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()
 			}
-		}()
+		})
 	}
 	wg.Wait()
 }

app/vmalert/rule/test_helpers.go

@@ -19,13 +19,13 @@ func CompareRules(t *testing.T, a, b Rule) error {
 	case *AlertingRule:
 		br, ok := b.(*AlertingRule)
 		if !ok {
-			return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
+			return fmt.Errorf("rule %d supposed to be of type AlertingRule", b.ID())
 		}
 		return compareAlertingRules(t, v, br)
 	case *RecordingRule:
 		br, ok := b.(*RecordingRule)
 		if !ok {
-			return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
+			return fmt.Errorf("rule %d supposed to be of type RecordingRule", b.ID())
 		}
 		return compareRecordingRules(t, v, br)
 	default:

app/vmalert/vmalertutil/err_group.go

@@ -45,7 +45,7 @@ func (eg *ErrGroup) Error() string {
 		return ""
 	}
 	var b strings.Builder
-	fmt.Fprintf(&b, "errors(%d): ", len(eg.errs))
+	fmt.Fprintf(&b, "errors(%d): \n", len(eg.errs))
 	for i, err := range eg.errs {
 		b.WriteString(err.Error())
 		if i != len(eg.errs)-1 {

app/vmalert/vmalertutil/err_group_test.go

@@ -30,8 +30,8 @@ func TestErrGroup(t *testing.T) {
 	}
 
 	f(nil, "")
-	f([]error{errors.New("timeout")}, "errors(1): timeout")
-	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
+	f([]error{errors.New("timeout")}, "errors(1): \ntimeout")
+	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): \ntimeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent

app/vmalert/web.qtpl

@@ -9,6 +9,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -78,6 +79,8 @@
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
+        Version {%s buildinfo.Version %} <br>
+
         API:<br>
         {% for _, p := range apiLinks  %}
             {%code p, doc := p[0], p[1] %}

app/vmauth/auth_config.go

@@ -113,10 +113,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		ui.concurrencyLimitReached.Inc()
-
-		// The per-user limit for the number of concurrent requests is reached.
-		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// The number of concurrently executed requests for the given user equals the limt.
+		// Wait until some of the currently executed requests are finished, so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case ui.concurrencyLimitCh <- struct{}{}:
@@ -124,6 +122,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 		case <-ctx.Done():
 			err := ctx.Err()
 			if errors.Is(err, context.DeadlineExceeded) {
+				// The current request couldn't be executed until the request timeout.
+				ui.concurrencyLimitReached.Inc()
 				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
 					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
 			}
@@ -150,12 +150,22 @@ func (ui *UserInfo) stopHealthChecks() {
 	if ui == nil {
 		return
 	}
-	if ui.URLPrefix == nil {
-		return
-	}
 
-	bus := ui.URLPrefix.bus.Load()
-	bus.stopHealthChecks()
+	if ui.URLPrefix != nil {
+		bus := ui.URLPrefix.bus.Load()
+		bus.stopHealthChecks()
+	}
+	if ui.DefaultURL != nil {
+		bus := ui.DefaultURL.bus.Load()
+		bus.stopHealthChecks()
+	}
+	for i := range ui.URLMaps {
+		um := &ui.URLMaps[i]
+		if um.URLPrefix != nil {
+			bus := um.URLPrefix.bus.Load()
+			bus.stopHealthChecks()
+		}
+	}
 }
 
 // Header is `Name: Value` http header, which must be added to the proxied request.
@@ -363,12 +373,10 @@ func (bu *backendURL) isBroken() bool {
 
 func (bu *backendURL) setBroken() {
 	if bu.broken.CompareAndSwap(false, true) {
-		bu.healthCheckWG.Add(1)
-		go func() {
-			defer bu.healthCheckWG.Done()
+		bu.healthCheckWG.Go(func() {
 			bu.runHealthCheck()
 			bu.broken.Store(false)
-		}()
+		})
 	}
 }
 
@@ -394,7 +402,7 @@ func (bu *backendURL) runHealthCheck() {
 				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
 					return
 				}
-				logger.Warnf("ignoring the backend at %s for %s becasue of dial error: %s", addr, *failTimeout, err)
+				logger.Warnf("ignoring the backend at %s for %s because of dial error: %s", addr, *failTimeout, err)
 				continue
 			}
 
@@ -733,11 +741,9 @@ func initAuthConfig() {
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	stopCh = make(chan struct{})
-	authConfigWG.Add(1)
-	go func() {
-		defer authConfigWG.Done()
+	authConfigWG.Go(func() {
 		authConfigReloader(sighupCh)
-	}()
+	})
 }
 
 func stopAuthConfig() {
@@ -809,7 +815,7 @@ func reloadAuthConfig() (bool, error) {
 
 	ok, err := reloadAuthConfigData(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 	if !ok {
 		return false, nil

app/vmauth/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ioutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -40,27 +41,38 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
-		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
+	idleConnTimeout        = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
-	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
-	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
-		"and max_concurrent_requests option in per-user config")
-	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
-		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
-		"This allows graceful handling of short spikes in the number of concurrent requests")
+	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
+		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
+		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
+		"See also -requestBufferSize")
+
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests "+
+		"and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 100, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This provides fairness and isolation between users, preventing a single user from consuming all the available resources. "+
+		"It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. "+
+		"This default can be overridden for individual users via max_concurrent_requests option in per-user config. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration to wait before rejecting incoming requests if concurrency limit "+
+		"specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. "+
+		"Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. "+
+		"This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
 
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
+	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -215,48 +227,121 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
-	// Limit the concurrency of requests to backends
+	// Acquire global concurrency limit.
+	if err := beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer endConcurrencyLimit()
+
+	// Set read deadline for reading the initial chunk for the request body.
+	rc := http.NewResponseController(w)
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		logger.Panicf("BUG: expecting valid deadline for the context")
+	}
+	if err := rc.SetReadDeadline(deadline); err != nil {
+		logger.Panicf("BUG: cannot set read deadline: %s", err)
+	}
+
+	// Read the initial chunk for the request body.
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+	bb, err := bufferRequestBody(ctx, r.Body, userName)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	r.Body = bb
+
+	// Disable the read deadline for the rest of the request body.
+	if err := rc.SetReadDeadline(time.Time{}); err != nil {
+		logger.Panicf("BUG: cannot reset read deadline: %s", err)
+	}
+
+	// Acquire concurrency limit for the given user.
+	if err := ui.beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer ui.endConcurrencyLimit()
+
+	// Process the request.
+	processRequest(w, r, ui)
+}
+
+func beginConcurrencyLimit(ctx context.Context) error {
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(ctx); err != nil {
-			handleConcurrencyLimitError(w, r, err)
-			<-concurrencyLimitCh
-			return
-		}
+		return nil
 	default:
 		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
 		// so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			if err := ui.beginConcurrencyLimit(ctx); err != nil {
-				handleConcurrencyLimitError(w, r, err)
-				<-concurrencyLimitCh
-				return
-			}
+			return nil
 		case <-ctx.Done():
 			err := ctx.Err()
-
-			concurrentRequestsLimitReached.Inc()
-
 			if errors.Is(err, context.DeadlineExceeded) {
-				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+				// The current request couldn't be executed until the request timeout.
+				concurrentRequestsLimitReached.Inc()
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
 					*maxQueueDuration, cap(concurrencyLimitCh))
-				handleConcurrencyLimitError(w, r, err)
-				return
 			}
-
-			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
-			handleConcurrencyLimitError(w, r, err)
-			return
+			return fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
 		}
 	}
-	processRequest(w, r, ui)
-	ui.endConcurrencyLimit()
+}
+
+func endConcurrencyLimit() {
 	<-concurrencyLimitCh
 }
 
+func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
+	if r == nil {
+		// This is a GET request with nil reader.
+		return nil, nil
+	}
+
+	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
+	if maxBufSize <= 0 {
+		return r, nil
+	}
+
+	lr := ioutil.GetLimitedReader(r, int64(maxBufSize))
+	defer ioutil.PutLimitedReader(lr)
+
+	start := time.Now()
+	buf, err := io.ReadAll(lr)
+	bufferRequestBodyDuration.UpdateDuration(start)
+
+	if err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			rejectSlowClientRequests.Inc()
+
+			d := time.Since(start)
+
+			return nil, &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
+					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
+				StatusCode: http.StatusBadRequest,
+			}
+		}
+
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot read request body: %w", err),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+
+	bb := newBufferedBody(r, buf, maxBufSize)
+	return bb, nil
+}
+
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
@@ -282,9 +367,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	r.Body = rtb
-
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
@@ -292,18 +374,19 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			break
 		}
 		targetURL := bu.url
-		// Don't change path and add request_path query param for default route.
 		if isDefault {
+			// Don't change path and add request_path query param for default route.
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
-		} else { // Update path for regular routes.
+		} else {
+			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
 	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
 		if needLocalRetry && !wasLocalRetry {
 			wasLocalRetry = true
 			goto again
@@ -313,6 +396,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if ok {
 			return
 		}
+
 		bu.setBroken()
 		ui.backendErrors.Inc()
 	}
@@ -324,7 +408,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ui.requestErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo, bu *backendURL) (bool, bool) {
 	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
@@ -339,27 +423,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		}
 	}
 
-	rtb, rtbOK := req.Body.(*readTrackingBody)
+	bb, bbOK := req.Body.(*bufferedBody)
+	canRetry := !bbOK || bb.canRetry()
+
 	res, err := ui.rt.RoundTrip(req)
 
-	if ctxErr := r.Context().Err(); ctxErr != nil {
-		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
-		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
-		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
-		err = ctxErr
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
 	}
+
 	if err != nil {
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			// Do not retry canceled or timed out requests
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			if errors.Is(err, context.DeadlineExceeded) {
-				// Timed out request must be counted as errors, since this usually means that the backend is slow.
-				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-			}
-			return false, false
-		}
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
@@ -368,27 +444,32 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
 			ui.requestErrors.Inc()
+			bu.setBroken()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
 			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			if bbOK {
+				bb.resetReader()
+			}
 			return false, true
 		}
 
-		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
+		// Retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, requestURI, targetURL, err)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
-		_ = res.Body.Close()
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request body has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
@@ -397,13 +478,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			ui.requestErrors.Inc()
 			return true, false
 		}
+
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
+		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
+			remoteAddr, requestURI, targetURL, res.StatusCode, retryStatusCodes)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	removeHopHeaders(res.Header)
@@ -413,10 +497,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
+
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
+	}
+
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
-
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 		ui.requestErrors.Inc()
 		return true, false
@@ -546,6 +636,10 @@ var (
 	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
+	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
+
+	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -629,10 +723,10 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
-	ctx := r.Context()
-	if errors.Is(ctx.Err(), context.Canceled) {
+	if errors.Is(r.Context().Err(), context.Canceled) {
 		// Do not return any response for the request canceled by the client,
 		// since the connection to the client is already closed.
+		clientCanceledRequests.Inc()
 		return
 	}
 
@@ -644,123 +738,78 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// readTrackingBody must be obtained via getReadTrackingBody()
-type readTrackingBody struct {
-	// maxBodySize is the maximum body size to cache in buf.
+// bufferedBody serves two purposes:
+//  1. Enables request retries when the body size does not exceed maxBodySize
+//     by fully buffering the body in memory.
+//  2. Prevents slow clients from reducing effective server capacity by
+//     buffering the request body before acquiring a per-user concurrency slot.
+//
+// See bufferRequestBody for details on how bufferedBody is used.
+type bufferedBody struct {
+	// r contains reader for reading the data after buf is read.
 	//
-	// Bigger bodies cannot be retried.
-	maxBodySize int
-
-	// r contains reader for initial data reading
+	// r is nil if buf contains all the data.
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
-	// If more than maxBodySize is read from r, then cannotRetry is set to true.
+	// buf contains the initial buffer read from r.
 	buf []byte
 
-	// readBuf points to the cached data at buf, which must be read in the next call to Read().
-	readBuf []byte
+	// bufOffset is the offset at buf for already read bytes.
+	bufOffset int
 
-	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
-	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
+	// cannotRetry is set to true after Close() call on non-nil r.
 	cannotRetry bool
-
-	// bufComplete is set to true when buf contains complete request body read from r.
-	bufComplete bool
 }
 
-func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	// do not use sync.Pool there
-	// since http.RoundTrip may still use request body after return
-	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
-	rtb := &readTrackingBody{}
-	if maxBodySize < 0 {
-		maxBodySize = 0
+func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
+	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+
+	if len(buf) < maxBufSize {
+		// Read the full request body into buf.
+		r = nil
 	}
-	rtb.maxBodySize = maxBodySize
 
-	if r == nil {
-		// This is GET request without request body
-		r = (*zeroReader)(nil)
+	return &bufferedBody{
+		r:   r,
+		buf: buf,
 	}
-	rtb.r = r
-	return rtb
-}
-
-type zeroReader struct{}
-
-func (r *zeroReader) Read(_ []byte) (int, error) {
-	return 0, io.EOF
-}
-
-func (r *zeroReader) Close() error {
-	return nil
 }
 
 // Read implements io.Reader interface.
-func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(rtb.readBuf) > 0 {
-		n := copy(p, rtb.readBuf)
-		rtb.readBuf = rtb.readBuf[n:]
+func (bb *bufferedBody) Read(p []byte) (int, error) {
+	if bb.cannotRetry {
+		return 0, fmt.Errorf("cannot read already closed body")
+	}
+	if bb.bufOffset < len(bb.buf) {
+		n := copy(p, bb.buf[bb.bufOffset:])
+		bb.bufOffset += n
 		return n, nil
 	}
-
-	if rtb.r == nil {
-		if rtb.bufComplete {
-			return 0, io.EOF
-		}
-		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+	if bb.r == nil {
+		return 0, io.EOF
 	}
-
-	n, err := rtb.r.Read(p)
-	if rtb.cannotRetry {
-		return n, err
-	}
-
-	if len(rtb.buf)+n > rtb.maxBodySize {
-		rtb.cannotRetry = true
-		return n, err
-	}
-	rtb.buf = append(rtb.buf, p[:n]...)
-	if err == io.EOF {
-		rtb.bufComplete = true
-	}
-	return n, err
+	return bb.r.Read(p)
 }
 
-func (rtb *readTrackingBody) canRetry() bool {
-	if rtb.cannotRetry {
-		return false
-	}
-	if rtb.bufComplete {
-		return true
-	}
-	return rtb.r != nil
+func (bb *bufferedBody) canRetry() bool {
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
-func (rtb *readTrackingBody) Close() error {
-	if !rtb.cannotRetry {
-		rtb.readBuf = rtb.buf
-	} else {
-		rtb.readBuf = nil
+func (bb *bufferedBody) Close() error {
+	bb.resetReader()
+	if bb.r != nil {
+		bb.cannotRetry = true
+		return bb.r.Close()
 	}
-
-	// Close rtb.r only if the request body is completely read or if it is too big.
-	// http.Roundtrip performs body.Close call even without any Read calls,
-	// so this hack allows us to reuse request body.
-	if rtb.bufComplete || rtb.cannotRetry {
-		if rtb.r == nil {
-			return nil
-		}
-		err := rtb.r.Close()
-		rtb.r = nil
-		return err
-	}
-
 	return nil
 }
 
+func (bb *bufferedBody) resetReader() {
+	bb.bufOffset = 0
+}
+
 func debugInfo(u *url.URL, r *http.Request) string {
 	s := &strings.Builder{}
 	fmt.Fprintf(s, " (host: %q; ", r.Host)

app/vmauth/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
@@ -546,28 +548,300 @@ func (w *fakeResponseWriter) WriteHeader(statusCode int) {
 	}
 }
 
-func TestReadTrackingBody_RetrySuccess(t *testing.T) {
+// This is needed for net/http.ResponseController
+func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
+	return nil
+}
+
+func TestBufferRequestBody_Success(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	f := func(body *bytes.Buffer, requestBufferSizeFlag, maxRequestBodySizeToRetryFlag string) {
+		t.Helper()
+
+		expectedResponse := "statusCode=200"
+		if body.Len() > 0 {
+			expectedResponse += "\n" + body.String()
+		}
+
+		if err := requestBufferSize.Set(requestBufferSizeFlag); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		if err := maxRequestBodySizeToRetry.Set(maxRequestBodySizeToRetryFlag); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if !backendCalled {
+			t.Fatalf("backend is not called")
+		}
+	}
+
+	// no body, no buffering, no retry
+	f(bytes.NewBuffer(nil), "0", "0")
+
+	// no body, buffering on, no retry
+	f(bytes.NewBuffer(nil), "100", "0")
+
+	// no body, no buffering, retry on
+	f(bytes.NewBuffer(nil), "0", "100")
+
+	// no body, buffering on, retry on
+	f(bytes.NewBuffer(nil), "100", "100")
+
+	// body smaller than buffer, retry max on
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "101", "101")
+
+	// body smaller than buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "501", "0")
+
+	// body same size as buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "500", "0")
+
+	// body bigger than a buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "499", "0")
+
+	// body bigger than tmpBuf 8KiB used in buffering
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16384", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16385", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16383", "")
+}
+
+func TestBufferRequestBody_Failure(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	defaultMaxQueueDuration := *maxQueueDuration
+	defer func() {
+		*maxQueueDuration = defaultMaxQueueDuration
+	}()
+
+	f := func(body *mockBody, expectedResponse string) {
+		t.Helper()
+
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+		if err := requestBufferSize.Set("2048"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		*maxQueueDuration = 100 * time.Millisecond
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if backendCalled {
+			t.Fatalf("backend is called")
+		}
+	}
+
+	// an error at the beginning of reading
+	f(&mockBody{err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+
+	// an error after reading 1024 bytes, buffer size is 2048 bytes
+	f(&mockBody{head: make([]byte, 1024), err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+}
+
+type mockBody struct {
+	head []byte
+	err  error
+	tail []byte
+}
+
+func (r *mockBody) Read(p []byte) (n int, err error) {
+	if len(r.head) > 0 {
+		n = copy(p, r.head)
+		r.head = r.head[n:]
+		return n, nil
+	}
+
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	if len(r.tail) > 0 {
+		n = copy(p, r.tail)
+		r.tail = r.tail[n:]
+		return n, nil
+	}
+
+	return 0, io.EOF
+}
+
+func TestBufferedBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
-			data, err := io.ReadAll(rtb)
+			data, err := io.ReadAll(rb)
 			if err != nil {
 				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
 			}
 			if string(data) != s {
 				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
 			}
-			if err := rtb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
-			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true at iteration %d", i)
+			if err := rb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing bufferedBody at iteration %d: %s", i, err)
 			}
 		}
 	}
@@ -577,19 +851,48 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
+func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
+			t.Fatalf("canRetry must return true")
+		}
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
-			n, err := io.ReadFull(rtb, buf)
+			n, err := io.ReadFull(rb, buf)
 			if err != nil {
 				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
 			}
@@ -599,26 +902,20 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 			if string(buf) != s[:i] {
 				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
 			}
-			if err := rtb.Close(); err != nil {
+			if err := rb.Close(); err != nil {
 				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
 			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
-			}
 		}
 
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 	}
 
@@ -627,30 +924,53 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
+func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set("0"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if canRetry {
+			t.Fatalf("canRetry() must return false because of too big request body")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rtb, buf)
+		n, err := io.ReadFull(rb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after reading one byte")
-		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
@@ -658,14 +978,11 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 		if dataRead != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		data, err = io.ReadAll(rtb)
+		data, err = io.ReadAll(rb)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -679,35 +996,48 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		data, err = io.ReadAll(rb)
+		if err != nil {
+			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		data, err = io.ReadAll(rtb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 

app/vmctl/backoff/backoff.go

@@ -7,6 +7,8 @@ import (
 	"math"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -55,6 +57,7 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 			return attempt, err // fail fast if not recoverable
 		}
 		attempt++
+		retriesTotal.Inc()
 		backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
 		dur := time.Duration(backoff)
 		logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
@@ -74,3 +77,7 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 	}
 	return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
 }
+
+var (
+	retriesTotal = metrics.NewCounter(`vmctl_backoff_retries_total`)
+)

app/vmctl/flags.go

@@ -14,6 +14,12 @@ const (
 	globalSilent             = "s"
 	globalVerbose            = "verbose"
 	globalDisableProgressBar = "disable-progress-bar"
+
+	globalPushMetricsURL         = "pushmetrics.url"
+	globalPushMetricsInterval    = "pushmetrics.interval"
+	globalPushExtraLabels        = "pushmetrics.extraLabel"
+	globalPushHeaders            = "pushmetrics.header"
+	globalPushDisableCompression = "pushmetrics.disableCompression"
 )
 
 var (
@@ -33,6 +39,29 @@ var (
 			Value: false,
 			Usage: "Whether to disable progress bar during the import.",
 		},
+		&cli.StringSliceFlag{
+			Name:  globalPushMetricsURL,
+			Usage: "Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics",
+		},
+		&cli.DurationFlag{
+			Name:  globalPushMetricsInterval,
+			Value: 10 * time.Second,
+			Usage: "Interval for pushing metrics to every -pushmetrics.url",
+		},
+		&cli.StringSliceFlag{
+			Name: globalPushExtraLabels,
+			Usage: "Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. " +
+				"Flag can be set multiple times, to add few additional labels. " +
+				"For example, -pushmetrics.extraLabel='instance=\"foo\"' adds instance=\"foo\" label to all the metrics pushed to every -pushmetrics.url",
+		},
+		&cli.StringSliceFlag{
+			Name:  globalPushHeaders,
+			Usage: "Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.",
+		},
+		&cli.BoolFlag{
+			Name:  globalPushDisableCompression,
+			Usage: "Whether to disable compression when pushing metrics.",
+		},
 	}
 )
 
@@ -123,32 +152,32 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination.",
 		},
 		&cli.StringFlag{
 			Name:  vmCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmServerName,
-			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
+			Usage: "Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
+			Usage: "Whether to skip tls verification when connecting to '--vm-addr'",
 			Value: false,
 		},
 		&cli.IntFlag{
@@ -468,7 +497,7 @@ var (
 			Name: vmNativeFilterMatch,
 			Usage: "Time series selector to match series for export. For example, select {instance!=\"localhost\"} will " +
 				"match all series with \"instance\" label different to \"localhost\".\n" +
-				" See more details here https://github.com/VictoriaMetrics/VictoriaMetrics#how-to-export-data-in-native-format",
+				" See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format",
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{
@@ -598,7 +627,7 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
@@ -625,8 +654,8 @@ var (
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
 			Usage: "Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format " +
-				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API." +
-				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
+				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. " +
+				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. " +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},

app/vmctl/influx.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"sync"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -52,6 +54,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
+	influxSeriesTotal.Add(len(series))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -63,18 +66,18 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	ip.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(ip.cc)
-	for i := 0; i < ip.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range ip.cc {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if err := ip.do(s); err != nil {
+					influxErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
 					return
 				}
+				influxSeriesProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import
@@ -83,6 +86,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("influx error: %s", infErr)
 		case vmErr := <-ip.im.Errors():
+			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		case seriesCh <- s:
 		}
@@ -95,6 +99,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
 		if vmErr.Err != nil {
+			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		}
 	}
@@ -169,3 +174,9 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 		}
 	}
 }
+
+var (
+	influxSeriesTotal     = metrics.NewCounter(`vmctl_influx_migration_series_total`)
+	influxSeriesProcessed = metrics.NewCounter(`vmctl_influx_migration_series_processed`)
+	influxErrorsTotal     = metrics.NewCounter(`vmctl_influx_migration_errors_total`)
+)

app/vmctl/limiter/limiter.go

@@ -4,6 +4,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
@@ -45,9 +47,16 @@ func (l *Limiter) Register(dataLen int) {
 			t := timerpool.Get(d)
 			<-t.C
 			timerpool.Put(t)
+			limiterThrottleEventsTotal.Inc()
 		}
 		l.budget += limit
 		l.deadline = time.Now().Add(time.Second)
 	}
 	l.budget -= int64(dataLen)
+	limiterBytesProcessed.Add(dataLen)
 }
+
+var (
+	limiterBytesProcessed      = metrics.NewCounter(`vmctl_limiter_bytes_processed_total`)
+	limiterThrottleEventsTotal = metrics.NewCounter(`vmctl_limiter_throttle_events_total`)
+)

app/vmctl/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"log"
 	"net/http"
@@ -19,7 +20,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
@@ -41,11 +44,20 @@ func main() {
 	ctx, cancelCtx := context.WithCancel(context.Background())
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
+		flag.Parse()
+		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
 			barpool.Disable(true)
 		}
 		netutil.EnableIPv6()
+		pushmetrics.InitWith(&pushmetrics.Config{
+			URLs:               c.StringSlice(globalPushMetricsURL),
+			Interval:           c.Duration(globalPushMetricsInterval),
+			ExtraLabels:        c.StringSlice(globalPushExtraLabels),
+			DisableCompression: c.Bool(globalPushDisableCompression),
+			Headers:            c.StringSlice(globalPushHeaders),
+		})
 		return nil
 	}
 	app := &cli.App{
@@ -451,6 +463,7 @@ func main() {
 		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
+	pushmetrics.StopAndPush()
 }
 
 func initConfigVM(c *cli.Context) (vm.Config, error) {

app/vmctl/native/client.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 )
 
@@ -36,12 +38,15 @@ type Response struct {
 
 // Explore finds metric names by provided filter from api/v1/label/__name__/values
 func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start, end time.Time) ([]string, error) {
+	startTime := time.Now()
+	exploreRequestsTotal.Inc()
 	url := fmt.Sprintf("%s/%s", c.Addr, nativeMetricNamesAddr)
 	if tenantID != "" {
 		url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeMetricNamesAddr)
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
+		exploreRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
 	}
 
@@ -53,37 +58,53 @@ func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start,
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
+		exploreRequestsErrorsTotal.Inc()
+		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("series request failed: %s", err)
 	}
 
 	var response Response
 	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		exploreRequestsErrorsTotal.Inc()
+		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("cannot decode series response: %s", err)
 	}
+	exploreDuration.UpdateDuration(startTime)
 	return response.MetricNames, resp.Body.Close()
 }
 
 // ImportPipe uses pipe reader in request to process data
 func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
+	startTime := time.Now()
+	importRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
 	if err != nil {
+		importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
 	}
 
 	importResp, err := c.do(req, http.StatusNoContent)
 	if err != nil {
+		importRequestsErrorsTotal.Inc()
+		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request failed: %s", err)
 	}
 	if err := importResp.Body.Close(); err != nil {
+		importRequestsErrorsTotal.Inc()
+		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("cannot close import response body: %s", err)
 	}
+	importDuration.UpdateDuration(startTime)
 	return nil
 }
 
 // ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
 func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
+	startTime := time.Now()
+	exportRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
+		exportRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
 	}
 
@@ -102,8 +123,11 @@ func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadC
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
+		exportRequestsErrorsTotal.Inc()
+		exportDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("export request failed: %w", err)
 	}
+	exportDuration.UpdateDuration(startTime)
 	return resp.Body, nil
 }
 
@@ -162,3 +186,16 @@ func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
 	}
 	return resp, err
 }
+
+var (
+	importRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="import"}`)
+	exportRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="export"}`)
+	exploreRequestsTotal       = metrics.NewCounter(`vmctl_vm_native_requests_total{type="explore"}`)
+	importRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="import"}`)
+	exportRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="export"}`)
+	exploreRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="explore"}`)
+
+	importDuration  = metrics.NewHistogram(`vmctl_vm_native_import_duration_seconds`)
+	exportDuration  = metrics.NewHistogram(`vmctl_vm_native_export_duration_seconds`)
+	exploreDuration = metrics.NewHistogram(`vmctl_vm_native_explore_duration_seconds`)
+)

app/vmctl/opentsdb.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	vmetrics "github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/cheggaaa/pb/v3"
@@ -57,6 +59,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	if !prompt(ctx, question) {
 		return nil
 	}
+
 	op.im.ResetStats()
 	var startTime int64
 	if op.oc.HardTS != 0 {
@@ -84,23 +87,24 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 		seriesCh := make(chan queryObj, op.otsdbcc)
 		errCh := make(chan error)
 		// we're going to make serieslist * queryRanges queries, so we should represent that in the progress bar
+		otsdbSeriesTotal.Add(len(serieslist) * queryRanges)
 		bar := pb.StartNew(len(serieslist) * queryRanges)
 		defer func(bar *pb.ProgressBar) {
 			bar.Finish()
 		}(bar)
 		var wg sync.WaitGroup
-		wg.Add(op.otsdbcc)
-		for i := 0; i < op.otsdbcc; i++ {
-			go func() {
-				defer wg.Done()
+		for range op.otsdbcc {
+			wg.Go(func() {
 				for s := range seriesCh {
 					if err := op.do(s); err != nil {
+						otsdbErrorsTotal.Inc()
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
 						return
 					}
+					otsdbSeriesProcessed.Inc()
 					bar.Increment()
 				}
-			}()
+			})
 		}
 		/*
 			Loop through all series for this metric, processing all retentions and time ranges
@@ -117,6 +121,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 					case otsdbErr := <-errCh:
 						return fmt.Errorf("opentsdb error: %s", otsdbErr)
 					case vmErr := <-op.im.Errors():
+						otsdbErrorsTotal.Inc()
 						return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 					case seriesCh <- queryObj{
 						Tr: tr, StartTime: startTime,
@@ -141,6 +146,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	op.im.Close()
 	for vmErr := range op.im.Errors() {
 		if vmErr.Err != nil {
+			otsdbErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 		}
 	}
@@ -171,3 +177,9 @@ func (op *otsdbProcessor) do(s queryObj) error {
 	}
 	return op.im.Input(&ts)
 }
+
+var (
+	otsdbSeriesTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_total`)
+	otsdbSeriesProcessed = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_processed`)
+	otsdbErrorsTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_errors_total`)
+)

app/vmctl/opentsdb/opentsdb.go

@@ -109,7 +109,7 @@ func (c Client) FindMetrics(q string) ([]string, error) {
 		return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(resp.Body)
@@ -133,7 +133,7 @@ func (c Client) FindSeries(metric string) ([]Meta, error) {
 		return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err)
 	}
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(resp.Body)

app/vmctl/prometheus.go

@@ -4,11 +4,15 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -61,19 +65,19 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
-		var labels []vm.LabelPair
+		var labelPairs []vm.LabelPair
 		series := ss.At()
 
-		for _, label := range series.Labels() {
+		series.Labels().Range(func(label labels.Label) {
 			if label.Name == "__name__" {
 				name = label.Value
-				continue
+				return
 			}
-			labels = append(labels, vm.LabelPair{
-				Name:  label.Name,
-				Value: label.Value,
+			labelPairs = append(labelPairs, vm.LabelPair{
+				Name:  strings.Clone(label.Name),
+				Value: strings.Clone(label.Value),
 			})
-		}
+		})
 		if name == "" {
 			return fmt.Errorf("failed to find `__name__` label in labelset for block %v", b.Meta().ULID)
 		}
@@ -99,7 +103,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 		}
 		ts := vm.TimeSeries{
 			Name:       name,
-			LabelPairs: labels,
+			LabelPairs: labelPairs,
 			Timestamps: timestamps,
 			Values:     values,
 		}
@@ -111,6 +115,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 }
 
 func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
+	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -122,18 +127,18 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	pp.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(pp.cc)
-	for i := 0; i < pp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range pp.cc {
+		wg.Go(func() {
 			for br := range blockReadersCh {
 				if err := pp.do(br); err != nil {
+					promErrorsTotal.Inc()
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
 					return
 				}
+				promBlocksProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 	// any error breaks the import
 	for _, br := range blocks {
@@ -143,6 +148,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 			return fmt.Errorf("prometheus error: %s", promErr)
 		case vmErr := <-pp.im.Errors():
 			close(blockReadersCh)
+			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		case blockReadersCh <- br:
 		}
@@ -156,6 +162,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	// drain import errors channel
 	for vmErr := range pp.im.Errors() {
 		if vmErr.Err != nil {
+			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		}
 	}
@@ -165,3 +172,9 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 
 	return nil
 }
+
+var (
+	promBlocksTotal     = metrics.NewCounter(`vmctl_prometheus_migration_blocks_total`)
+	promBlocksProcessed = metrics.NewCounter(`vmctl_prometheus_migration_blocks_processed`)
+	promErrorsTotal     = metrics.NewCounter(`vmctl_prometheus_migration_errors_total`)
+)

app/vmctl/remoteread.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
@@ -51,6 +53,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
+	remoteReadRangesTotal.Add(len(ranges))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -66,18 +69,18 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	errCh := make(chan error)
 
 	var wg sync.WaitGroup
-	wg.Add(rrp.cc)
-	for i := 0; i < rrp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range rrp.cc {
+		wg.Go(func() {
 			for r := range rangeC {
 				if err := rrp.do(ctx, r); err != nil {
+					remoteReadErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for: %s", err)
 					return
 				}
+				remoteReadRangesProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	for _, r := range ranges {
@@ -85,6 +88,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("remote read error: %s", infErr)
 		case vmErr := <-rrp.dst.Errors():
+			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		case rangeC <- &remoteread.Filter{
 			StartTimestampMs: r[0].UnixMilli(),
@@ -100,6 +104,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range rrp.dst.Errors() {
 		if vmErr.Err != nil {
+			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		}
 	}
@@ -120,3 +125,9 @@ func (rrp *remoteReadProcessor) do(ctx context.Context, filter *remoteread.Filte
 		return nil
 	})
 }
+
+var (
+	remoteReadRangesTotal     = metrics.NewCounter(`vmctl_remote_read_migration_ranges_total`)
+	remoteReadRangesProcessed = metrics.NewCounter(`vmctl_remote_read_migration_ranges_processed`)
+	remoteReadErrorsTotal     = metrics.NewCounter(`vmctl_remote_read_migration_errors_total`)
+)

app/vmctl/vm/vm.go

@@ -12,6 +12,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -80,6 +82,12 @@ type Importer struct {
 
 	s       *stats
 	backoff *backoff.Backoff
+
+	importRequestsTotal       *metrics.Counter
+	importRequestsErrorsTotal *metrics.Counter
+	importSamplesTotal        *metrics.Counter
+	importBytesTotal          *metrics.Counter
+	importDuration            *metrics.Histogram
 }
 
 // ResetStats resets im stats.
@@ -147,6 +155,12 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
 		backoff:    cfg.Backoff,
+
+		importRequestsTotal:       metrics.GetOrCreateCounter(`vmctl_importer_requests_total`),
+		importRequestsErrorsTotal: metrics.GetOrCreateCounter(`vmctl_importer_request_errors_total`),
+		importSamplesTotal:        metrics.GetOrCreateCounter(`vmctl_importer_samples_total`),
+		importBytesTotal:          metrics.GetOrCreateCounter(`vmctl_importer_bytes_total`),
+		importDuration:            metrics.GetOrCreateHistogram(`vmctl_importer_request_duration_seconds`),
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
@@ -156,15 +170,13 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		cfg.BatchSize = 1e5
 	}
 
-	im.wg.Add(int(cfg.Concurrency))
-	for i := 0; i < int(cfg.Concurrency); i++ {
+	for i := range int(cfg.Concurrency) {
 		pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i)
 		bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0)
 
-		go func(bar barpool.Bar) {
-			defer im.wg.Done()
+		im.wg.Go(func() {
 			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
-		}(bar)
+		})
 	}
 	im.ResetStats()
 	return im, nil
@@ -313,9 +325,13 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 		return nil
 	}
 
+	startTime := time.Now()
+	im.importRequestsTotal.Inc()
+
 	pr, pw := io.Pipe()
 	req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
 	if err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}
 	if im.user != "" {
@@ -335,6 +351,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	if im.compress {
 		zw, err := gzip.NewWriterLevel(w, 1)
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
 		}
 		w = zw
@@ -346,29 +363,39 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	for _, ts := range tsBatch {
 		n, err := ts.write(bw)
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("write err: %w", err)
 		}
 		totalBytes += n
 		totalSamples += len(ts.Values)
 	}
 	if err := bw.Flush(); err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 	if closer, ok := w.(io.Closer); ok {
 		err := closer.Close()
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return err
 		}
 	}
 	if err := pw.Close(); err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 
 	requestErr := <-errCh
 	if requestErr != nil {
+		im.importRequestsErrorsTotal.Inc()
+		im.importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request error for %q: %w", im.addr, requestErr)
 	}
 
+	im.importSamplesTotal.Add(totalSamples)
+	im.importBytesTotal.Add(totalBytes)
+	im.importDuration.UpdateDuration(startTime)
+
 	im.s.Lock()
 	im.s.bytes += uint64(totalBytes)
 	im.s.samples += uint64(totalSamples)

app/vmctl/vm_native.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -82,13 +84,19 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 		if !prompt(ctx, question) {
 			return nil
 		}
+		migrationTenantsTotal.Set(uint64(len(tenants)))
 	}
 
 	for _, tenantID := range tenants {
 		err := p.runBackfilling(ctx, tenantID, ranges)
 		if err != nil {
+			migrationErrorsTotal.Inc()
 			return fmt.Errorf("migration failed: %s", err)
 		}
+
+		if p.interCluster {
+			migrationTenantsProcessed.Inc()
+		}
 	}
 
 	log.Println("Import finished!")
@@ -156,6 +164,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
 	p.s.bytes += uint64(written)
 	p.s.requests++
 	p.s.Unlock()
+	migrationBytesTransferredTotal.AddInt64(written)
 
 	if err := pw.Close(); err != nil {
 		return err
@@ -199,7 +208,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var foundSeriesMsg string
 	var requestsToMake int
-	var metrics = map[string][][]time.Time{
+	var metricsMap = map[string][][]time.Time{
 		"": ranges,
 	}
 
@@ -211,11 +220,11 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	if !p.disablePerMetricRequests {
 		format = fmt.Sprintf(nativeWithBackoffTpl, barPrefix)
-		metrics, err = p.explore(ctx, p.src, tenantID, ranges)
+		metricsMap, err = p.explore(ctx, p.src, tenantID, ranges)
 		if err != nil {
 			return fmt.Errorf("failed to explore metric names: %s", err)
 		}
-		if len(metrics) == 0 {
+		if len(metricsMap) == 0 {
 			errMsg := "no metrics found"
 			if tenantID != "" {
 				errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
@@ -223,10 +232,14 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			log.Println(errMsg)
 			return nil
 		}
-		for _, m := range metrics {
+		for _, m := range metricsMap {
 			requestsToMake += len(m)
 		}
-		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metrics), requestsToMake)
+		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metricsMap), requestsToMake)
+
+		migrationMetricsTotal.Add(len(metricsMap))
+	} else {
+		requestsToMake = len(ranges)
 	}
 
 	if !p.interCluster {
@@ -240,6 +253,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		log.Print(foundSeriesMsg)
 	}
 
+	migrationRequestsPlanned.Add(requestsToMake)
 	bar := barpool.NewSingleProgress(format, requestsToMake)
 	bar.Start()
 	defer bar.Finish()
@@ -249,9 +263,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var wg sync.WaitGroup
 	for i := 0; i < p.cc; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
@@ -265,12 +277,13 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 						return
 					}
 				}
+				migrationRequestsCompleted.Inc()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import
-	for mName, mRanges := range metrics {
+	for mName, mRanges := range metricsMap {
 		match, err := buildMatchWithFilter(p.filter.Match, mName)
 		if err != nil {
 			logger.Errorf("failed to build filter %q for metric name %q: %s", p.filter.Match, mName, err)
@@ -290,6 +303,9 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			}:
 			}
 		}
+		if !p.disablePerMetricRequests {
+			migrationMetricsProcessed.Inc()
+		}
 	}
 
 	close(filterCh)
@@ -398,3 +414,18 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
 	match := "{" + strings.Join(filters, " or ") + "}"
 	return match, nil
 }
+
+var (
+	migrationMetricsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_metrics_total`)
+	migrationMetricsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_metrics_processed`)
+
+	migrationRequestsPlanned   = metrics.NewCounter(`vmctl_vm_native_migration_requests_planned`)
+	migrationRequestsCompleted = metrics.NewCounter(`vmctl_vm_native_migration_requests_completed`)
+
+	migrationErrorsTotal = metrics.NewCounter(`vmctl_vm_native_migration_errors_total`)
+
+	migrationTenantsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_tenants_total`)
+	migrationTenantsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_tenants_processed`)
+
+	migrationBytesTransferredTotal = metrics.NewCounter(`vmctl_vm_native_migration_bytes_transferred_total`)
+)

app/vminsert/common/insert_ctx.go

@@ -182,6 +182,7 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
 	}
+	ctx.mms = mms
 
 	err := vmstorage.AddMetadataRows(mms)
 	if err != nil {
@@ -206,6 +207,7 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 	}
+	ctx.mms = mms
 
 	err := vmstorage.AddMetadataRows(mms)
 	if err != nil {

app/vminsert/common/streamaggr.go

@@ -111,9 +111,7 @@ func InitStreamAggr() {
 	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 
 	// Start config reloader.
-	saCfgReloaderWG.Add(1)
-	go func() {
-		defer saCfgReloaderWG.Done()
+	saCfgReloaderWG.Go(func() {
 		for {
 			select {
 			case <-sighupCh:
@@ -122,7 +120,7 @@ func InitStreamAggr() {
 			}
 			reloadStreamAggrConfig()
 		}
-	}()
+	})
 }
 
 func reloadStreamAggrConfig() {

app/vminsert/main.go

@@ -232,7 +232,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
-	case "zabbixconnector/api/v1/history":
+	case "/zabbixconnector/api/v1/history":
 		zabbixconnectorHistoryRequests.Inc()
 		if err := zabbixconnector.InsertHandlerForHTTP(r); err != nil {
 			zabbixconnectorHistoryErrors.Inc()
@@ -241,7 +241,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			fmt.Fprintf(w, `{"error":%q}`, err.Error())
 			return true
 		}
-		w.WriteHeader(http.StatusAccepted)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()

app/vmselect/graphite/eval.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -196,12 +197,17 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
 				pathExpression: safePathExpression(expr),
 			}
 			s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
-			t := timerpool.Get(30 * time.Second)
+
+			// A negative or zero duration will cause timer.C to return immediately
+			remainingTimeout := ec.deadline.Deadline() - fasttime.UnixTimestamp()
+			t := timerpool.Get(time.Duration(remainingTimeout) * time.Second)
 			defer timerpool.Put(t)
+
 			select {
 			case seriesCh <- s:
 			case <-t.C:
-				logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
+				logger.Errorf("reached timeout when processing the %s (full query: %s), it can be due to the amount of storageNodes configured in vmselect is more than vmselect’s available CPU count "+
+					"or vmselect is heavy loaded. Consider adding resources or increasing `-search.maxQueryDuration` or `timeout` parameter in the query.",
 					expr.AppendString(nil), ec.originalQuery)
 			}
 			return nil

app/vmselect/graphite/render_api.go

@@ -82,7 +82,7 @@ func RenderHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if s := r.FormValue("maxDataPoints"); len(s) > 0 {
 		n, err := strconv.ParseFloat(s, 64)
 		if err != nil {
-			return fmt.Errorf("cannot parse maxDataPoints=%q: %w", maxDataPoints, err)
+			return fmt.Errorf("cannot parse maxDataPoints=%d: %w", maxDataPoints, err)
 		}
 		if n <= 0 {
 			return fmt.Errorf("maxDataPoints must be greater than 0; got %f", n)

app/vmselect/graphite/transform.go

@@ -3896,27 +3896,9 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 	seriesCh := make(chan *series, goroutines)
 	errCh := make(chan error, 1)
 	var wg sync.WaitGroup
-	wg.Add(goroutines)
-	go func() {
-		var err error
-		for {
-			s, e := nextSeries()
-			if e != nil || s == nil {
-				err = e
-				break
-			}
-			seriesCh <- s
-		}
-		close(seriesCh)
-		wg.Wait()
-		close(resultCh)
-		errCh <- err
-		close(errCh)
-	}()
 	var skipProcessing atomic.Bool
-	for i := 0; i < goroutines; i++ {
-		go func() {
-			defer wg.Done()
+	for range goroutines {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if skipProcessing.Load() {
 					continue
@@ -3934,8 +3916,24 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 					}
 				}
 			}
-		}()
+		})
 	}
+	go func() {
+		var err error
+		for {
+			s, e := nextSeries()
+			if e != nil || s == nil {
+				err = e
+				break
+			}
+			seriesCh <- s
+		}
+		close(seriesCh)
+		wg.Wait()
+		close(resultCh)
+		errCh <- err
+		close(errCh)
+	}()
 	wrapper := func() (*series, error) {
 		r := <-resultCh
 		if r == nil {

app/vmselect/main.go

@@ -520,7 +520,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"for accessing vmalert flag '-vmalert.proxyURL' must be configured"}`)
 			return true
 		}
-		proxyVMAlertRequests(w, r)
+		proxyVMAlertRequests(w, r, path)
 		return true
 	}
 
@@ -558,7 +558,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
@@ -568,7 +568,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
@@ -578,7 +578,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -725,7 +725,7 @@ var (
 	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
-func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
+func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 	defer func() {
 		err := recover()
 		if err == nil || err == http.ErrAbortHandler {
@@ -736,8 +736,10 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	r.Host = vmalertProxyHost
-	vmalertProxy.ServeHTTP(w, r)
+	req := r.Clone(r.Context())
+	req.URL.Path = strings.TrimPrefix(path, "prometheus")
+	req.Host = vmalertProxyHost
+	vmalertProxy.ServeHTTP(w, req)
 }
 
 var (

app/vmselect/netstorage/netstorage.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -296,14 +297,12 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := range workChs {
-		wg.Add(1)
-		qtChild := qt.NewChild("worker #%d", i)
-		go func(workerID uint) {
-			timeseriesWorker(qtChild, workChs, workerID)
+	for workerID := range workChs {
+		qtChild := qt.NewChild("worker #%d", workerID)
+		wg.Go(func() {
+			timeseriesWorker(qtChild, workChs, uint(workerID))
 			qtChild.Done()
-			wg.Done()
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 
@@ -514,12 +513,10 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go func(workerID uint) {
-			unpackWorker(workChs, workerID)
-			wg.Done()
-		}(uint(i))
+	for workerID := range workers {
+		wg.Go(func() {
+			unpackWorker(workChs, uint(workerID))
+		})
 	}
 	wg.Wait()
 
@@ -582,6 +579,7 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		return
 	}
 	heap.Init(sbh)
+	var dedupSamples int
 	for {
 		sbs := sbh.sbs
 		top := sbs[0]
@@ -597,6 +595,7 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		if n := equalSamplesPrefix(top, sbNext); n > 0 && dedupInterval > 0 {
 			// Skip n replicated samples at top if deduplication is enabled.
 			top.NextIdx = topNextIdx + n
+			dedupSamples += n
 		} else {
 			// Copy samples from top to dst with timestamps not exceeding tsNext.
 			top.NextIdx = topNextIdx + binarySearchTimestamps(top.Timestamps[topNextIdx:], tsNext)
@@ -611,8 +610,8 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		}
 	}
 	timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values, dedupInterval)
-	dedups := len(dst.Timestamps) - len(timestamps)
-	dedupsDuringSelect.Add(dedups)
+	dedupSamples += len(dst.Timestamps) - len(timestamps)
+	dedupsDuringSelect.Add(dedupSamples)
 	dst.Timestamps = timestamps
 	dst.Values = values
 }
@@ -638,7 +637,7 @@ func equalTimestampsPrefix(a, b []int64) int {
 
 func equalValuesPrefix(a, b []float64) int {
 	for i, v := range a {
-		if i >= len(b) || v != b[i] {
+		if i >= len(b) || math.Float64bits(v) != math.Float64bits(b[i]) {
 			return i
 		}
 	}
@@ -1020,12 +1019,10 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		mustStop      atomic.Bool
 	)
 	var wg sync.WaitGroup
-	wg.Add(gomaxprocs)
-	for i := 0; i < gomaxprocs; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range gomaxprocs {
+		wg.Go(func() {
 			for xw := range workCh {
-				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
+				if err := f(&xw.mn, &xw.b, tr, uint(workerID)); err != nil {
 					errGlobalLock.Lock()
 					if errGlobal == nil {
 						errGlobal = err
@@ -1036,7 +1033,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 				xw.reset()
 				exportWorkPool.Put(xw)
 			}
-		}(uint(i))
+		})
 	}
 
 	// Feed workers with work

app/vmselect/netstorage/netstorage_test.go

@@ -1,8 +1,11 @@
 package netstorage
 
 import (
+	"math"
 	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
 func TestMergeSortBlocks(t *testing.T) {
@@ -194,3 +197,111 @@ func TestMergeSortBlocks(t *testing.T) {
 		Values:     []float64{7, 24, 26},
 	})
 }
+
+func TestEqualSamplesPrefix(t *testing.T) {
+	f := func(a, b *sortBlock, expected int) {
+		t.Helper()
+
+		actual := equalSamplesPrefix(a, b)
+		if actual != expected {
+			t.Fatalf("unexpected result: got %d, want %d", actual, expected)
+		}
+	}
+
+	// Empty blocks
+	f(&sortBlock{}, &sortBlock{}, 0)
+
+	// Identical blocks
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, 4)
+
+	// Non-zero NextIdx
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+		NextIdx:    2,
+	}, &sortBlock{
+		Timestamps: []int64{10, 20, 3, 4},
+		Values:     []float64{50, 60, 7, 8},
+		NextIdx:    2,
+	}, 2)
+
+	// Non-zero NextIdx with mismatch
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+		NextIdx:    1,
+	}, &sortBlock{
+		Timestamps: []int64{10, 2, 3, 4},
+		Values:     []float64{50, 6, 7, 80},
+		NextIdx:    1,
+	}, 2)
+
+	// Different lengths
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3},
+		Values:     []float64{5, 6, 7},
+	}, 3)
+
+	// Timestamps diverge
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 30, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, 2)
+
+	// Values diverge
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 60, 7, 8},
+	}, 1)
+
+	// Zero matches
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{5, 6, 7, 8},
+		Values:     []float64{1, 2, 3, 4},
+	}, 0)
+
+	// Compare staleness markers, matching
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, decimal.StaleNaN, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, decimal.StaleNaN, 7, 8},
+	}, 4)
+
+	// Special float values: +Inf, -Inf, 0, -0
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
+	}, 4)
+
+	// Positive zero vs negative zero (bitwise different)
+	f(&sortBlock{
+		Timestamps: []int64{1, 2},
+		Values:     []float64{5, math.Copysign(0, +1)},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2},
+		Values:     []float64{5, math.Copysign(0, -1)},
+	}, 1)
+}

app/vmselect/promql/aggr_incremental_test.go

@@ -103,15 +103,13 @@ func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssEx
 	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
-	wg.Add(workersCount)
-	for i := 0; i < workersCount; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workersCount {
+		wg.Go(func() {
 			for ts := range tsCh {
 				runtime.Gosched() // allow other goroutines performing the work
-				iafc.updateTimeseries(ts, workerID)
+				iafc.updateTimeseries(ts, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	for _, ts := range tssSrc {
 		tsCh <- ts

app/vmselect/promql/eval.go

@@ -477,22 +477,18 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		var tssFirst []*timeseries
 		var errFirst error
 		qtFirst := qt.NewChild("expr1")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
 			qtFirst.Done()
-		}()
+		})
 
 		var tssSecond []*timeseries
 		var errSecond error
 		qtSecond := qt.NewChild("expr2")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
 			qtSecond.Done()
-		}()
+		})
 
 		wg.Wait()
 		if errFirst != nil {
@@ -710,17 +706,13 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
-		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
-		go func(e metricsql.Expr, i int) {
-			defer func() {
-				qtChild.Done()
-				wg.Done()
-			}()
+		wg.Go(func() {
+			defer qtChild.Done()
 			rv, err := evalExpr(qtChild, ec, e)
 			rvs[i] = rv
 			errs[i] = err
-		}(e, i)
+		})
 	}
 	wg.Wait()
 	for _, err := range errs {
@@ -785,7 +777,8 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
-	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	if re.At == nil {
 		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
@@ -835,7 +828,8 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 }
 
 func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -1017,16 +1011,14 @@ func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, time
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workers {
+		wg.Go(func() {
 			var tmpValues []float64
 			var tmpTimestamps []int64
 			for ts := range workChs[workerID] {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 }
@@ -1058,7 +1050,8 @@ func removeNanValues(dstValues []float64, dstTimestamps []int64, values []float6
 
 // evalInstantRollup evaluates instant rollup where ec.Start == ec.End.
 func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64,
+) ([]*timeseries, error) {
 	if ec.Start != ec.End {
 		logger.Panicf("BUG: evalInstantRollup cannot be called on non-empty time range; got %s", ec.timeRangeString())
 	}
@@ -1083,10 +1076,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		rollupResultCacheV.DeleteInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 	}
 	getCachedSeries := func(qt *querytracer.Tracer) ([]*timeseries, int64, error) {
+		rollupResultCacheV.rollupResultCacheRequests.Inc()
 	again:
 		offset := int64(0)
 		tssCached := rollupResultCacheV.GetInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 		if len(tssCached) == 0 {
+			rollupResultCacheV.rollupResultCacheMisses.Inc()
 			// Cache miss. Re-populate the missing data.
 			start := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds()
 			offset = timestamp - start
@@ -1129,6 +1124,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			deleteCachedSeries(qt)
 			goto again
 		}
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 		ec.QueryStats.addSeriesFetched(len(tssCached))
 		return tssCached, offset, nil
 	}
@@ -1537,16 +1533,11 @@ func assertInstantValues(tss []*timeseries) {
 	}
 }
 
-var (
-	rollupResultCacheFullHits    = metrics.NewCounter(`vm_rollup_result_cache_full_hits_total`)
-	rollupResultCachePartialHits = metrics.NewCounter(`vm_rollup_result_cache_partial_hits_total`)
-	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
-
-	memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
-)
+var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
 
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
+) ([]*timeseries, error) {
 	window, err := windowExpr.NonNegativeDuration(ec.Step)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
@@ -1582,19 +1573,20 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 
 	// Search for cached results.
+	rollupResultCacheV.rollupResultCacheRequests.Inc()
 	tssCached, start := rollupResultCacheV.GetSeries(qt, ec, expr, window)
 	ec.QueryStats.addSeriesFetched(len(tssCached))
 	if start > ec.End {
 		qt.Printf("the result is fully cached")
-		rollupResultCacheFullHits.Inc()
+		rollupResultCacheV.rollupResultCacheFullHits.Inc()
 		return tssCached, nil
 	}
 	if start > ec.Start {
 		qt.Printf("partial cache hit")
-		rollupResultCachePartialHits.Inc()
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 	} else {
 		qt.Printf("cache miss")
-		rollupResultCacheMiss.Inc()
+		rollupResultCacheV.rollupResultCacheMisses.Inc()
 	}
 
 	// Fetch missing results, which aren't cached yet.
@@ -1630,7 +1622,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 //
 // pointsPerSeries is used only for estimating the needed memory for query processing
 func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64,
+) ([]*timeseries, error) {
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s: timeRange=%s, step=%d, window=%d", expr.AppendString(nil), ec.timeRangeString(), ec.Step, window)
 		defer qt.Done()
@@ -1720,6 +1713,7 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 		return nil, err
 	}
 	defer rml.Put(uint64(rollupMemorySize))
+	qs.addMemoryUsage(rollupMemorySize)
 	qt.Printf("the rollup evaluation needs an estimated %d bytes of RAM for %d series and %d points per series (summary %d points)",
 		rollupMemorySize, timeseriesLen, pointsPerSeries, rollupPoints)
 
@@ -1753,7 +1747,8 @@ func maxSilenceInterval() int64 {
 
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
@@ -1792,7 +1787,8 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 }
 
 func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
@@ -1832,7 +1828,8 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 }
 
 func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
-	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 {
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64,
+) uint64 {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)

app/vmselect/promql/exec.go

@@ -37,7 +37,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 	if querystats.Enabled() {
 		startTime := time.Now()
 		defer func() {
-			querystats.RegisterQuery(q, ec.End-ec.Start, startTime)
+			querystats.RegisterQuery(q, ec.End-ec.Start, startTime, ec.QueryStats.memoryUsage())
 			ec.QueryStats.addExecutionTimeMsec(startTime)
 		}()
 	}

app/vmselect/promql/query_stats.go

@@ -13,6 +13,8 @@ type QueryStats struct {
 	ExecutionDuration atomic.Pointer[time.Duration]
 	// SeriesFetched contains the number of series fetched from storage or cache.
 	SeriesFetched atomic.Int64
+	// MemoryUsage contains the estimated memory consumption of the query
+	MemoryUsage atomic.Int64
 
 	at *auth.Token
 
@@ -53,3 +55,17 @@ func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
 	d := time.Since(startTime)
 	qs.ExecutionDuration.Store(&d)
 }
+
+func (qs *QueryStats) addMemoryUsage(memoryUsage int64) {
+	if qs == nil {
+		return
+	}
+	qs.MemoryUsage.Store(memoryUsage)
+}
+
+func (qs *QueryStats) memoryUsage() int64 {
+	if qs == nil {
+		return 0
+	}
+	return qs.MemoryUsage.Load()
+}

app/vmselect/promql/rollup.go

@@ -534,7 +534,10 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the rc.LookbackDelta.
+	// Is populated if the preceding sample falls within the rc.LookbackDelta range, or if rc.LookbackDelta is not set.
+	//
+	// It provides an additional check and value for rollup functions such as increase(), changes(),
+	// when the prevValue is NaN due to a gap or a small lookback window.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -713,7 +716,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	// Use step as the scrape interval for instant queries (when start == end).
+	// Set maxPrevInterval for subsequent rfa.prevValue calculations in rollupFunc:
+	// For instant queries, use rc.Step directly as maxPrevInterval.
+	// For range queries, rc.Step is typically too small to serve as the lookback window between two rollup points.
+	// Instead, estimate the scrape interval from raw sample timestamps (using the 0.6 quantile of the last 20 intervals)
+	// and slightly inflate the scrape interval to set maxPrevInterval, allowing for some tolerance to jitter.
 	maxPrevInterval := rc.Step
 	if rc.Start < rc.End {
 		scrapeInterval := getScrapeInterval(timestamps, rc.Step)
@@ -729,22 +736,21 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 	}
 	window := rc.Window
+	// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
+	// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+	// when the window doesn't cover at least two raw samples (this is what most users expect).
+	//
+	// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+	// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
-			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
-			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
-			// when the window doesn't cover at least two raw samples (this is what most users expect).
-			//
-			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
-			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
-			//
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 			window = maxPrevInterval
 		}
+		// Artificial window cannot exceed explicit rc.LookbackDelta, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
-			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
-			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
@@ -869,17 +875,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
 		return defaultInterval
 	}
 
-	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
-	tsPrev := timestamps[0]
-	timestamps = timestamps[1:]
+	// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
+	tsPrev := timestamps[len(timestamps)-1]
+	timestamps = timestamps[:len(timestamps)-1]
 	if len(timestamps) > 20 {
-		timestamps = timestamps[:20]
+		timestamps = timestamps[len(timestamps)-20:]
 	}
 	a := getFloat64s()
 	intervals := a.A[:0]
-	for _, ts := range timestamps {
-		intervals = append(intervals, float64(ts-tsPrev))
-		tsPrev = ts
+	for i := len(timestamps) - 1; i >= 0; i-- {
+		intervals = append(intervals, float64(tsPrev-timestamps[i]))
+		tsPrev = timestamps[i]
 	}
 	scrapeInterval := int64(quantile(0.6, intervals))
 	a.A = intervals
@@ -2107,9 +2113,15 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		prevValue = values[0]
-		values = values[1:]
-		n++
+		// Assume that the value didn't change during the current gap
+		// if realPrevValue exists.
+		if !math.IsNaN(rfa.realPrevValue) {
+			prevValue = rfa.realPrevValue
+		} else {
+			n++
+			prevValue = values[0]
+			values = values[1:]
+		}
 	}
 	for _, v := range values {
 		if v != prevValue {

app/vmselect/promql/rollup_result_cache.go

@@ -83,9 +83,11 @@ func checkRollupResultCacheReset() {
 
 const checkRollupResultCacheResetInterval = 5 * time.Second
 
-var needRollupResultCacheReset atomic.Bool
-var checkRollupResultCacheResetOnce sync.Once
-var rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+var (
+	needRollupResultCacheReset       atomic.Bool
+	checkRollupResultCacheResetOnce  sync.Once
+	rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+)
 
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024 * 1024), // This is a cache for testing.
@@ -178,6 +180,12 @@ func InitRollupResultCache(cachePath string) {
 
 	rollupResultCacheV = &rollupResultCache{
 		c: c,
+
+		rollupResultCacheRequests:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_requests_total`),
+		rollupResultCacheFullHits:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_full_hits_total`),
+		rollupResultCachePartialHits: metrics.GetOrCreateCounter(`vm_rollup_result_cache_partial_hits_total`),
+		rollupResultCacheMisses:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_miss_total`),
+		rollupResultCacheResets:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_resets_total`),
 	}
 }
 
@@ -193,13 +201,18 @@ func StopRollupResultCache() {
 
 type rollupResultCache struct {
 	c *workingsetcache.Cache
-}
 
-var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="promql/rollupResult"}`)
+	rollupResultCacheRequests    *metrics.Counter
+	rollupResultCacheFullHits    *metrics.Counter
+	rollupResultCachePartialHits *metrics.Counter
+	rollupResultCacheMisses      *metrics.Counter
+
+	rollupResultCacheResets *metrics.Counter
+}
 
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
-	rollupResultCacheResets.Inc()
+	rollupResultCacheV.rollupResultCacheResets.Inc()
 	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }

app/vmselect/promql/rollup_test.go

@@ -232,6 +232,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	}
 	var rfa rollupFuncArg
 	rfa.prevValue = nan
+	rfa.realPrevValue = nan
 	rfa.prevTimestamp = 0
 	rfa.values = append(rfa.values, testValues...)
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
@@ -1654,7 +1655,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1674,7 +1675,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1794,7 +1795,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1814,7 +1815,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1888,3 +1889,126 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupChangesWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then changes will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then changes will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 1, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280
+	//
+	// When there are gaps between samples that exceed maxPrevInterval,
+	// either due to changes in the scrape interval or missing scrapes.
+	// For example, if the scrape interval was initially 30s and later changed to 10s,
+	// the auto-calculated scrape interval is 10s, with maxPrevInterval inflated to 15s.
+	//
+	// At t=30s:
+	// prevValue is NaN, as the last sample at t=0s is considered stale for t=30s given the maxPrevInterval.
+	// realPrevValue is 1, taken from t=0s, since LookbackDelta=0 ignores staleness.
+	// the result should be `changes(1, 1) -> 0` instead of `changes(1, NaN)`.
+	// At t=100s:
+	// preValue is also NaN, as the last sample at t=70s is considered stale for t=100s.
+	// realPrevValue is 1, taken from t=70s,
+	// result should be `changes(2, 1) -> 1`.
+	timestamps = []int64{0, 30000, 40000, 50000, 60000, 70000, 100000}
+	values = []float64{1, 1, 1, 1, 1, 1, 2}
+	t.Run("issue-10280", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                100e3,
+			Step:               10e3,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, _ := rc.Do(nil, values, timestamps)
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3, 80e3, 90e3, 100e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/querystats/querystats.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
@@ -15,7 +16,8 @@ import (
 var (
 	lastQueriesCount = flag.Int("search.queryStats.lastQueriesCount", 20000, "Query stats for /api/v1/status/top_queries is tracked on this number of last queries. "+
 		"Zero value disables query stats tracking")
-	minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
+	minQueryDuration    = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
+	minQueryMemoryUsage = flagutil.NewBytes("search.queryStats.minQueryMemoryUsage", 1024, "The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats")
 )
 
 var (
@@ -31,9 +33,9 @@ func Enabled() bool {
 // RegisterQuery registers the query on the given timeRangeMsecs, which has been started at startTime.
 //
 // RegisterQuery must be called when the query is finished.
-func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time) {
+func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
 	initOnce.Do(initQueryStats)
-	qsTracker.registerQuery(query, timeRangeMsecs, startTime)
+	qsTracker.registerQuery(query, timeRangeMsecs, startTime, memoryUsage)
 }
 
 // WriteJSONQueryStats writes query stats to given writer in json format.
@@ -54,6 +56,7 @@ type queryStatRecord struct {
 	timeRangeSecs int64
 	registerTime  time.Time
 	duration      time.Duration
+	memoryUsage   int64
 }
 
 type queryStatKey struct {
@@ -66,8 +69,8 @@ func initQueryStats() {
 	if recordsCount <= 0 {
 		recordsCount = 1
 	} else {
-		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s",
-			*lastQueriesCount, *minQueryDuration)
+		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s, -search.queryStats.minQueryMemoryUsage=%s",
+			*lastQueriesCount, *minQueryDuration, minQueryMemoryUsage)
 	}
 	qsTracker = &queryStatsTracker{
 		a: make([]queryStatRecord, recordsCount),
@@ -78,6 +81,7 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
 	fmt.Fprintf(w, `{"topN":"%d","maxLifetime":"%s",`, topN, maxLifetime)
 	fmt.Fprintf(w, `"search.queryStats.lastQueriesCount":%d,`, *lastQueriesCount)
 	fmt.Fprintf(w, `"search.queryStats.minQueryDuration":"%s",`, *minQueryDuration)
+	fmt.Fprintf(w, `"search.queryStats.minQueryMemoryUsage":"%s",`, minQueryMemoryUsage)
 	fmt.Fprintf(w, `"topByCount":[`)
 	topByCount := qst.getTopByCount(topN, maxLifetime)
 	for i, r := range topByCount {
@@ -102,15 +106,28 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
 			fmt.Fprintf(w, `,`)
 		}
 	}
+
+	fmt.Fprintf(w, `],"topByAvgMemoryUsage":[`)
+	topByAvgMemoryConsumption := qst.getTopByAvgMemoryUsage(topN, maxLifetime)
+	for i, r := range topByAvgMemoryConsumption {
+		fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"avgMemoryBytes":%d,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.memoryUsage, r.count)
+		if i+1 < len(topByAvgMemoryConsumption) {
+			fmt.Fprintf(w, `,`)
+		}
+	}
+
 	fmt.Fprintf(w, `]}`)
 }
 
-func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time) {
+func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
 	registerTime := time.Now()
 	duration := registerTime.Sub(startTime)
 	if duration < *minQueryDuration {
 		return
 	}
+	if memoryUsage < int64(minQueryMemoryUsage.IntN()) {
+		return
+	}
 
 	qst.mu.Lock()
 	defer qst.mu.Unlock()
@@ -126,6 +143,7 @@ func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64,
 	r.timeRangeSecs = timeRangeMsecs / 1000
 	r.registerTime = registerTime
 	r.duration = duration
+	r.memoryUsage = memoryUsage
 }
 
 func (r *queryStatRecord) matches(currentTime time.Time, maxLifetime time.Duration) bool {
@@ -257,3 +275,47 @@ func (qst *queryStatsTracker) getTopBySumDuration(topN int, maxLifetime time.Dur
 	}
 	return a
 }
+
+type queryStatByMemory struct {
+	query         string
+	timeRangeSecs int64
+	memoryUsage   int64
+	count         int
+}
+
+func (qst *queryStatsTracker) getTopByAvgMemoryUsage(topN int, maxLifetime time.Duration) []queryStatByMemory {
+	currentTime := time.Now()
+	qst.mu.Lock()
+	type countSum struct {
+		count int
+		sum   int64
+	}
+	m := make(map[queryStatKey]countSum)
+	for _, r := range qst.a {
+		if r.matches(currentTime, maxLifetime) {
+			k := r.key()
+			ks := m[k]
+			ks.count++
+			ks.sum += r.memoryUsage
+			m[k] = ks
+		}
+	}
+	qst.mu.Unlock()
+
+	var a []queryStatByMemory
+	for k, ks := range m {
+		a = append(a, queryStatByMemory{
+			query:         k.query,
+			timeRangeSecs: k.timeRangeSecs,
+			memoryUsage:   ks.sum / int64(ks.count),
+			count:         ks.count,
+		})
+	}
+	sort.Slice(a, func(i, j int) bool {
+		return a[i].memoryUsage > a[j].memoryUsage
+	})
+	if len(a) > topN {
+		a = a[:topN]
+	}
+	return a
+}

app/vmselect/vmui/assets/MetricsQL-DDLrk-ox.md

@@ -12,6 +12,7 @@ aliases:
 - /MetricsQL.html
 - /metricsql/index.html
 - /metricsql/
+- /MetricsQL/
 ---
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
+  <script type="module" crossorigin src="./assets/index-C1hTBemk.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BR6Q0Fin.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -29,7 +29,8 @@ import (
 )
 
 var (
-	retentionPeriod   = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
+	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -388,11 +389,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotPath := Storage.MustCreateSnapshot()
+		snapshotName := Storage.MustCreateSnapshot()
+
+		// Verify whether the client already closed the connection.
+		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
+		if err := r.Context().Err(); err != nil {
+			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
+			if err := deleteSnapshot(snapshotName); err != nil {
+				logger.Infof("cannot delete just created snapshot: %s", err)
+				return true
+			}
+			return true
+		}
+
 		if prometheusCompatibleResponse {
-			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotName))
 		} else {
-			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotName))
 		}
 		return true
 	case "/list":
@@ -412,23 +425,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-
-		snapshots := Storage.MustListSnapshots()
-		for _, snName := range snapshots {
-			if snName == snapshotName {
-				if err := Storage.DeleteSnapshot(snName); err != nil {
-					err = fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-					jsonResponseError(w, err)
-					snapshotsDeleteErrorsTotal.Inc()
-					return true
-				}
-				fmt.Fprintf(w, `{"status":"ok"}`)
-				return true
-			}
+		if err := deleteSnapshot(snapshotName); err != nil {
+			jsonResponseError(w, err)
+			snapshotsDeleteErrorsTotal.Inc()
+			return true
 		}
-
-		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
-		jsonResponseError(w, err)
+		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
@@ -449,15 +451,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+func deleteSnapshot(snapshotName string) error {
+	snapshots := Storage.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := Storage.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}
+
 func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverCh = make(chan struct{})
 	if snapshotsMaxAge.Duration() <= 0 {
 		return
 	}
 	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Add(1)
-	go func() {
-		defer staleSnapshotsRemoverWG.Done()
+	staleSnapshotsRemoverWG.Go(func() {
 		d := timeutil.AddJitterToDuration(time.Second * 11)
 		t := time.NewTicker(d)
 		defer t.Stop()
@@ -469,7 +482,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 			}
 			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
 		}
-	}()
+	})
 }
 
 func stopStaleSnapshotsRemover() {
@@ -642,6 +655,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSize)
 
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
@@ -657,6 +671,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeBytes)
 
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
@@ -668,6 +683,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeMaxBytes)
 
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
@@ -679,6 +695,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheRequests)
 
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
@@ -690,6 +707,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheMisses)
 
 	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS build-web-stage
+FROM golang:1.26.0 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.22.2
+FROM alpine:3.23.3
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/Makefile

@@ -1,26 +1,23 @@
 # All these commands must run from repository root.
-
 copy-metricsql-docs:
 	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
+vmui-run-npm-command: vmui-package-base-image
+	docker run --rm \
+		--user $(shell id -u):$(shell id -g) \
+		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
+		-w /build/packages/vmui \
+		--entrypoint=/bin/bash \
+		vmui-builder-image -c "[ \"$$VMUI_SKIP_INSTALL\" = \"true\" ] || npm ci; $(NPM_COMMAND)"
+
+vmui-install:
+	NPM_COMMAND="true" $(MAKE) vmui-run-npm-command
+
 vmui-package-base-image:
 	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
 
-vmui-build: copy-metricsql-docs vmui-package-base-image
-	docker run --rm \
-		--user $(shell id -u):$(shell id -g) \
-		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
-		-w /build/packages/vmui \
-		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build"
-
-vmui-anomaly-build: vmui-package-base-image
-	docker run --rm \
-		--user $(shell id -u):$(shell id -g) \
-		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
-		-w /build/packages/vmui \
-		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build:anomaly"
+vmui-build: copy-metricsql-docs
+	NPM_COMMAND="npm run build" $(MAKE) vmui-run-npm-command
 
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
@@ -38,11 +35,11 @@ vmui-update: vmui-build
 vmui-install-dependencies:
 	cd app/vmui/packages/vmui && npm ci
 
-vmui-lint: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run lint
+vmui-lint:
+	NPM_COMMAND="npm run lint" $(MAKE) vmui-run-npm-command
 
-vmui-typecheck: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run typecheck
+vmui-typecheck:
+	NPM_COMMAND="npm run typecheck" $(MAKE) vmui-run-npm-command
 
-vmui-test: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run test
+vmui-test:
+	NPM_COMMAND="npm run test" $(MAKE) vmui-run-npm-command

app/vmui/packages/vmui/.env.vmanomaly

@@ -1 +0,0 @@
-VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -1,23 +0,0 @@
-import { readFile } from "fs/promises";
-import { IndexHtmlTransform } from "vite";
-
-/**
- * Vite plugin to dynamically load index.html based on the current mode.
- * If a specific mode-based index file (e.g., index.vmanomaly.html) exists, it is used.
- * Otherwise, the default index.html is loaded.
- */
-export default function dynamicIndexHtmlPlugin({ mode }) {
-  return {
-    name: "vm-dynamic-index-html",
-    transformIndexHtml: {
-      order: "pre",
-      handler: async () => {
-        try {
-          return await readFile(`./index.${mode}.html`, "utf8");
-        } catch (error) {
-          return await readFile("./index.html", "utf8");
-        }
-      }
-    } as IndexHtmlTransform
-  };
-}

app/vmui/packages/vmui/eslint.config.js

@@ -46,7 +46,7 @@ export default [...compat.extends(
   settings: {
     react: {
       pragma: "React",
-      version: "detect",
+      version: "19.0",
     },
 
     linkComponents: ["Hyperlink", {
@@ -69,10 +69,11 @@ export default [...compat.extends(
       "varsIgnorePattern": "^_",
       "ignoreRestSiblings": true
     }],
-    
+
     "unused-imports/no-unused-imports": "error",
 
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
+    "object-curly-spacing": [2, "always"],
 
     "react/jsx-max-props-per-line": [1, {
       maximum: 1,
@@ -81,13 +82,23 @@ export default [...compat.extends(
     "react/jsx-first-prop-new-line": [1, "multiline"],
 
     // Disable core indent rule due to recursion issues in ESLint 9; use JSX-specific rules instead
-    indent: "off",
+    indent: ["error", 2, {
+      SwitchCase: 1,
+      ignoredNodes: [
+        "JSXElement",
+        "JSXElement *",
+        "JSXFragment",
+        "JSXFragment *",
+      ],
+    }],
     "react/jsx-indent": ["error", 2],
     "react/jsx-indent-props": ["error", 2],
 
+    "linebreak-style": ["error", "unix"],
+    quotes: ["error", "double"],
+    semi: ["error", "always"],
     // Formatting rules moved out of ESLint core; omit here to avoid deprecation noise
     "react/prop-types": 0,
     "react/react-in-jsx-scope": "off",
-
   },
 }];

app/vmui/packages/vmui/index.vmanomaly.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg" />
-  <link rel="apple-touch-icon" href="/favicon.svg" />
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaMetrics Anomaly Detection</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-  <meta name="twitter:image" content="/preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta property="og:url" content="https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta property="og:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-<script type="module" src="/src/index.tsx"></script>
-</body>
-</html>

app/vmui/packages/vmui/package.json

@@ -7,10 +7,8 @@
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
     "start": "vite",
-    "start:playground": "cross-env PLAYGROUND=METRICS npm run start",
-    "start:anomaly": "vite --mode vmanomaly",
+    "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
@@ -18,47 +16,48 @@
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dev": "vitest"
+    "test:dev": "vitest",
+    "precommit": "npm run lint:local && npm run typecheck && npm run test"
   },
   "dependencies": {
     "classnames": "^2.5.1",
-    "dayjs": "^1.11.13",
+    "dayjs": "^1.11.19",
     "lodash.debounce": "^4.0.8",
-    "marked": "^16.0.0",
-    "preact": "^10.26.9",
-    "qs": "^6.14.0",
+    "marked": "^17.0.1",
+    "preact": "^10.28.3",
+    "qs": "^6.14.1",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.6.3",
+    "react-router-dom": "^7.13.0",
     "uplot": "^1.6.32",
-    "vite": "^7.1.11",
-    "web-vitals": "^5.0.3"
+    "vite": "^7.3.1",
+    "web-vitals": "^5.1.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.30.1",
-    "@preact/preset-vite": "^2.10.2",
-    "@testing-library/jest-dom": "^6.6.3",
+    "@eslint/eslintrc": "^3.3.3",
+    "@eslint/js": "^9.39.2",
+    "@preact/preset-vite": "^2.10.3",
+    "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^24.0.12",
+    "@types/node": "^25.2.0",
     "@types/qs": "^6.14.0",
-    "@types/react": "^19.1.8",
+    "@types/react": "^19.2.10",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.36.0",
-    "@typescript-eslint/parser": "^8.36.0",
-    "cross-env": "^7.0.3",
-    "eslint": "^9.30.1",
+    "@typescript-eslint/eslint-plugin": "^8.54.0",
+    "@typescript-eslint/parser": "^8.54.0",
+    "cross-env": "^10.1.0",
+    "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
-    "eslint-plugin-unused-imports": "^4.1.4",
-    "globals": "^16.3.0",
+    "eslint-plugin-unused-imports": "^4.3.0",
+    "globals": "^17.3.0",
     "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^26.1.0",
+    "jsdom": "^28.0.0",
     "postcss": "^8.5.6",
-    "rollup-plugin-visualizer": "^6.0.3",
-    "sass-embedded": "^1.89.2",
-    "typescript": "^5.8.3",
-    "vitest": "^3.2.4"
+    "rollup-plugin-visualizer": "^6.0.5",
+    "sass-embedded": "^1.97.3",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.18"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -1,41 +0,0 @@
-import { FC, useState } from "preact/compat";
-import { HashRouter, Route, Routes } from "react-router-dom";
-import AppContextProvider from "./contexts/AppContextProvider";
-import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
-import AnomalyLayout from "./layouts/AnomalyLayout/AnomalyLayout";
-import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
-import router from "./router";
-import CustomPanel from "./pages/CustomPanel";
-
-const AppAnomaly: FC = () => {
-  const [loadedTheme, setLoadedTheme] = useState(false);
-
-  return <>
-    <HashRouter>
-      <AppContextProvider>
-        <>
-          <ThemeProvider onLoaded={setLoadedTheme}/>
-          {loadedTheme && (
-            <Routes>
-              <Route
-                path={"/"}
-                element={<AnomalyLayout/>}
-              >
-                <Route
-                  path={"/"}
-                  element={<ExploreAnomaly/>}
-                />
-                <Route
-                  path={router.query}
-                  element={<CustomPanel/>}
-                />
-              </Route>
-            </Routes>
-          )}
-        </>
-      </AppContextProvider>
-    </HashRouter>
-  </>;
-};
-
-export default AppAnomaly;

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -12,6 +12,7 @@ aliases:
 - /MetricsQL.html
 - /metricsql/index.html
 - /metricsql/
+- /MetricsQL/
 ---
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -14,12 +14,11 @@ export type QueryGroup = {
 interface LegendProps {
   labels: LegendItemType[];
   query: string[];
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
 
-const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPanel, onChange }) => {
+const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange }) => {
   const { groupByLabel } = useLegendGroup();
   const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
@@ -33,7 +32,6 @@ const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPan
             key={group}
             labels={items}
             group={group}
-            isAnomalyView={isAnomalyView}
             onChange={onChange}
           />
         ))}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -13,7 +13,6 @@ import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
@@ -22,7 +21,7 @@ interface LegendGroupProps extends LegendProps {
   group: string | number;
 }
 
-const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onChange }) => {
+const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
   const { isTableView } = useLegendView();
   const { groupByLabel } = useLegendGroup();
   const copyToClipboard = useCopyToClipboard();
@@ -39,14 +38,14 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
 
   const Content = isTableView ? LegendTable : LegendLines;
 
-  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false"
-  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false";
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT;
 
   const expandedWarning = (
     <span className="vm-legend-group-header__warning">
       Legend collapsed by default ({sortedLabels.length} series) — click to expand.
     </span>
-  )
+  );
 
   return (
     <div
@@ -81,7 +80,6 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
       >
         <Content
           labels={sortedLabels}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -13,11 +13,10 @@ import { getLabelAlias } from "../../../../../utils/metric";
 interface LegendItemProps {
   legend: LegendItemType;
   onChange?: (item: LegendItemType, metaKey: boolean) => void;
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, isAnomalyView }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields }) => {
   const copyToClipboard = useCopyToClipboard();
   const { hideStats } = useShowStats();
 
@@ -52,12 +51,10 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, is
       })}
       onClick={createHandlerClick(legend)}
     >
-      {!isAnomalyView && (
-        <div
-          className="vm-legend-item__marker"
-          style={{ backgroundColor: legend.color }}
-        />
-      )}
+      <div
+        className="vm-legend-item__marker"
+        style={{ backgroundColor: legend.color }}
+      />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.hasAlias && legend.label}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendLines.tsx

@@ -2,7 +2,7 @@ import { FC } from "preact/compat";
 import LegendItem from "../LegendItem/LegendItem";
 import { LegendProps } from "../LegendGroup";
 
-const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields, onChange }) => {
+const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) => {
 
   return (
     <div className="vm-legend-item-container">
@@ -10,7 +10,6 @@ const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields,
         <LegendItem
           key={legendItem.label}
           legend={legendItem}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/LegendAnomaly.tsx

@@ -1,82 +0,0 @@
-import { FC, useMemo } from "preact/compat";
-import { ForecastType, SeriesItem } from "../../../../types";
-import { anomalyColors } from "../../../../utils/color";
-import "./style.scss";
-
-type Props = {
-  series: SeriesItem[];
-};
-
-const titles: Partial<Record<ForecastType, string>> = {
-  [ForecastType.yhat]: "yhat",
-  [ForecastType.yhatLower]: "yhat_upper - yhat_lower",
-  [ForecastType.yhatUpper]: "yhat_upper - yhat_lower",
-  [ForecastType.anomaly]: "anomalies",
-  [ForecastType.training]: "training data",
-  [ForecastType.actual]: "y"
-};
-
-const LegendAnomaly: FC<Props> = ({ series }) => {
-
-  const uniqSeriesStyles = useMemo(() => {
-    const uniqSeries = series.reduce((accumulator, currentSeries) => {
-      const hasForecast = Object.prototype.hasOwnProperty.call(currentSeries, "forecast");
-      const isNotUpper = currentSeries.forecast !== ForecastType.yhatUpper;
-      const isUniqForecast = !accumulator.find(s => s.forecast === currentSeries.forecast);
-      if (hasForecast && isUniqForecast && isNotUpper) {
-        accumulator.push(currentSeries);
-      }
-      return accumulator;
-    }, [] as SeriesItem[]);
-
-    const trainingSeries = {
-      ...uniqSeries[0],
-      forecast: ForecastType.training,
-      color: anomalyColors[ForecastType.training],
-    };
-    uniqSeries.splice(1, 0, trainingSeries);
-
-    return uniqSeries.map(s => ({
-      ...s,
-      color: typeof s.stroke === "string" ? s.stroke : anomalyColors[s.forecast || ForecastType.actual],
-    }));
-  }, [series]);
-
-  return <>
-    <div className="vm-legend-anomaly">
-      {/* TODO: remove .filter() after the correct training data has been added */}
-      {uniqSeriesStyles.filter(f => f.forecast !== ForecastType.training).map((s, i) => (
-        <div
-          key={`${i}_${s.forecast}`}
-          className="vm-legend-anomaly-item"
-        >
-          <svg>
-            {s.forecast === ForecastType.anomaly ? (
-              <circle
-                cx="15"
-                cy="7"
-                r="4"
-                fill={s.color}
-                stroke={s.color}
-                strokeWidth="1.4"
-              />
-            ) : (
-              <line
-                x1="0"
-                y1="7"
-                x2="30"
-                y2="7"
-                stroke={s.color}
-                strokeWidth={s.width || 1}
-                strokeDasharray={s.dash?.join(",")}
-              />
-            )}
-          </svg>
-          <div className="vm-legend-anomaly-item__title">{titles[s.forecast || ForecastType.actual]}</div>
-        </div>
-      ))}
-    </div>
-  </>;
-};
-
-export default LegendAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/style.scss

@@ -1,23 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-legend-anomaly {
-  position: relative;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  flex-wrap: wrap;
-  gap: calc($padding-large * 2);
-  cursor: default;
-
-  &-item {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    gap: $padding-small;
-
-    svg {
-      width: 30px;
-      height: 14px;
-    }
-  }
-}

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -13,7 +13,6 @@ import {
   getRangeY,
   getScales,
   handleDestroy,
-  setBand,
   setSelect
 } from "../../../../utils/uplot";
 import { MetricResult } from "../../../../api/types";
@@ -40,7 +39,6 @@ export interface LineChartProps {
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
   layoutSize: ElementSize;
   height?: number;
-  isAnomalyView?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
 }
@@ -55,7 +53,6 @@ const LineChart: FC<LineChartProps> = ({
   setPeriod,
   layoutSize,
   height,
-  isAnomalyView,
   spanGaps = false,
   showAllPoints = false,
 }) => {
@@ -75,7 +72,7 @@ const LineChart: FC<LineChartProps> = ({
     seriesFocus,
     setCursor,
     resetTooltips
-  } = useLineTooltip({ u: uPlotInst, metrics, series, unit, isAnomalyView });
+  } = useLineTooltip({ u: uPlotInst, metrics, series, unit });
 
   const options: uPlotOptions = {
     ...getDefaultOptions({ width: layoutSize.width, height }),
@@ -111,7 +108,6 @@ const LineChart: FC<LineChartProps> = ({
     if (!uPlotInst) return;
     delSeries(uPlotInst);
     addSeries(uPlotInst, series, spanGaps, showAllPoints);
-    setBand(uPlotInst, series);
     uPlotInst.redraw();
   }, [series, spanGaps, showAllPoints]);
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -29,7 +29,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
-  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE")
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
   const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
 
   const [limits, setLimits] = useState(seriesLimits);
@@ -58,7 +58,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   }, [limits]);
 
   useEffect(() => {
-    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`)
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
   }, [legendCollapse]);
 
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -9,7 +9,6 @@ import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../ut
 import useBoolean from "../../../../hooks/useBoolean";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
-import { getTenantIdFromUrl } from "../../../../utils/tenants";
 
 interface ServerConfiguratorProps {
   onClose: () => void;
@@ -39,10 +38,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   };
 
   const handleApply = useCallback(() => {
-    const tenantIdFromUrl = getTenantIdFromUrl(serverUrl);
-    if (tenantIdFromUrl !== "") {
-      dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
-    }
     dispatch({ type: "SET_SERVER", payload: serverUrl });
     onClose();
   }, [serverUrl]);
@@ -60,12 +55,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     }
   }, [enabledStorage]);
 
-  useEffect(() => {
-    if (enabledStorage) {
-      saveToStorage("SERVER_URL", serverUrl);
-    }
-  }, [serverUrl]);
-
   useEffect(() => {
     // the tenant selector can change the serverUrl
     if (stateServerUrl === serverUrl) return;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
+import { FC, useState, useRef, useMemo } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
@@ -10,14 +10,14 @@ import { getAppModeEnable } from "../../../../utils/app-mode";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import TextField from "../../../Main/TextField/TextField";
-import { getTenantIdFromUrl, replaceTenantId } from "../../../../utils/tenants";
+import { replaceTenantId } from "../../../../utils/tenants";
 import useBoolean from "../../../../hooks/useBoolean";
 
 const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   const appModeEnable = getAppModeEnable();
   const { isMobile } = useDeviceDetect();
 
-  const { tenantId: tenantIdState, serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
   const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
 
@@ -48,10 +48,8 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   }, [accountIds]);
 
   const createHandlerChange = (value: string) => () => {
-    const tenant = value;
-    dispatch({ type: "SET_TENANT_ID", payload: tenant });
     if (serverUrl) {
-      const updateServerUrl = replaceTenantId(serverUrl, tenant);
+      const updateServerUrl = replaceTenantId(serverUrl, value);
       if (updateServerUrl === serverUrl) return;
       dispatch({ type: "SET_SERVER", payload: updateServerUrl });
       timeDispatch({ type: "RUN_QUERY" });
@@ -59,16 +57,6 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
     handleCloseOptions();
   };
 
-  useEffect(() => {
-    const id = getTenantIdFromUrl(serverUrl);
-
-    if (tenantIdState && tenantIdState !== id) {
-      createHandlerChange(tenantIdState)();
-    } else {
-      createHandlerChange(id)();
-    }
-  }, [serverUrl]);
-
   if (!showTenantSelector) return null;
 
   return (
@@ -83,7 +71,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               <span className="vm-mobile-option__icon"><StorageIcon/></span>
               <div className="vm-mobile-option-text">
                 <span className="vm-mobile-option-text__label">Tenant ID</span>
-                <span className="vm-mobile-option-text__value">{tenantIdState}</span>
+                <span className="vm-mobile-option-text__value">{tenantId}</span>
               </div>
               <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
             </div>
@@ -106,7 +94,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               )}
               onClick={toggleOpenOptions}
             >
-              {tenantIdState}
+              {tenantId}
             </Button>
           )}
         </div>
@@ -138,7 +126,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               className={classNames({
                 "vm-list-item": true,
                 "vm-list-item_mobile": isMobile,
-                "vm-list-item_active": id === tenantIdState
+                "vm-list-item_active": id === tenantId
               })}
               key={id}
               onClick={createHandlerChange(id)}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -3,19 +3,18 @@ import { useEffect, useMemo, useState } from "preact/compat";
 import { ErrorTypes } from "../../../../../types";
 import { getAccountIds } from "../../../../../api/accountId";
 import { getAppModeEnable, getAppModeParams } from "../../../../../utils/app-mode";
-import { getTenantIdFromUrl } from "../../../../../utils/tenants";
 
 export const useFetchAccountIds = () => {
   const { useTenantID } = getAppModeParams();
   const appModeEnable = getAppModeEnable();
-  const { serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
 
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
   const [accountIds, setAccountIds] = useState<string[]>([]);
 
   const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
-  const isServerUrlWithTenant = useMemo(() => !!getTenantIdFromUrl(serverUrl), [serverUrl]);
+  const isServerUrlWithTenant = useMemo(() => !!tenantId, [tenantId]);
   const preventFetch = appModeEnable ? !useTenantID : !isServerUrlWithTenant;
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -6,6 +6,7 @@ import { QueryContextType } from "../../../types";
 import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
 import { QueryEditorAutocompleteProps } from "./QueryEditor";
 import { getExprLastPart, getValueByContext, getContext } from "./autocompleteUtils";
+import { extractCurrentLabel, extractLabelMatchers, extractMetric, splitByCursor } from "./utils/parser";
 
 const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   value,
@@ -20,45 +21,39 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   const metricsqlFunctions = useGetMetricsQL(includeFunctions);
 
   const values = useMemo(() => {
-    if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" };
-    const beforeCursor = value.substring(0, caretPosition[0]);
-    const afterCursor = value.substring(caretPosition[1]);
-    return { beforeCursor, afterCursor };
+    return splitByCursor(value, caretPosition);
   }, [value, caretPosition]);
 
-  const exprLastPart = useMemo(() => getExprLastPart(values.beforeCursor), [values]);
+  const exprLastPart = useMemo(() => {
+    return getExprLastPart(values.beforeCursor);
+  }, [values.beforeCursor]);
 
   const metric = useMemo(() => {
-    const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
-    const matchAlt = [...exprLastPart.matchAll(regex1)];
-    if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
-      return matchAlt[0].groups.metricName;
-    }
-
-    const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
-    const match = [...exprLastPart.matchAll(regex2)];
-    if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
-      return match[0].groups.metricName;
-    }
-
-    return "";
+    return extractMetric(exprLastPart);
   }, [exprLastPart]);
 
   const label = useMemo(() => {
-    const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
-    const match = exprLastPart.match(regexp);
-    return match ? match[match.length - 1] : "";
+    return extractCurrentLabel(exprLastPart);
   }, [exprLastPart]);
 
-  const context = useMemo(() => getContext(values.beforeCursor, metric, label), [values, metric, label]);
+  const context = useMemo(() => {
+    return getContext(values.beforeCursor, metric, label);
+  }, [values.beforeCursor, metric, label]);
 
-  const valueByContext = useMemo(() => getValueByContext(values.beforeCursor), [values.beforeCursor]);
+  const valueByContext = useMemo(() => {
+    return getValueByContext(values.beforeCursor);
+  }, [values.beforeCursor]);
+
+  const labelMatchers = useMemo(() => {
+    return extractLabelMatchers(values.beforeCursor, label);
+  }, [values.beforeCursor, label]);
 
   const { metrics, labels, labelValues, loading } = useFetchQueryOptions({
     valueByContext,
     metric,
     label,
     context,
+    labelMatchers,
   });
 
   const options = useMemo(() => {
@@ -72,18 +67,18 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
       default:
         return [];
     }
-  }, [context, metrics, labels, labelValues]);
+  }, [context, metrics, labels, labelValues, metricsqlFunctions]);
 
   const handleSelect = useCallback((insert: string) => {
     // Find the start and end of valueByContext in the query string
-    const value = values.beforeCursor;
+    const beforeCursor = values.beforeCursor;
     let valueAfterCursor = values.afterCursor;
-    const startIndexOfValueByContext = value.lastIndexOf(valueByContext, caretPosition[0]);
+    const startIndexOfValueByContext = beforeCursor.lastIndexOf(valueByContext, caretPosition[0]);
     const endIndexOfValueByContext = startIndexOfValueByContext + valueByContext.length;
 
     // Split the original string into parts: before, during, and after valueByContext
-    const beforeValueByContext = value.substring(0, startIndexOfValueByContext);
-    const afterValueByContext = value.substring(endIndexOfValueByContext);
+    const beforeValueByContext = beforeCursor.substring(0, startIndexOfValueByContext);
+    const afterValueByContext = beforeCursor.substring(endIndexOfValueByContext);
 
     // Add quotes around the value if the context is labelValue
     if (context === QueryContextType.labelValue) {
@@ -104,7 +99,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     // Assemble the new value with the inserted text
     const newVal = `${beforeValueByContext}${insert}${afterValueByContext}${valueAfterCursor}`;
     onSelect(newVal, beforeValueByContext.length + insert.length);
-  }, [values]);
+  }, [values.beforeCursor, values.afterCursor, valueByContext, caretPosition, context, onSelect]);
 
   useEffect(() => {
     if (!anchorEl.current) {
@@ -142,7 +137,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
 
     span.remove();
     marker.remove();
-  }, [anchorEl, caretPosition, hasHelperText]);
+  }, [anchorEl, caretPosition, hasHelperText, values.beforeCursor, values.afterCursor]);
 
   return (
     <>

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/utils/parser.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect } from "vitest";
+import {
+  splitByCursor,
+  extractMetric,
+  extractCurrentLabel,
+  extractLabelMatchers,
+} from "./parser";
+
+describe("splitByCursor", () => {
+  it("splits by caret when selection is collapsed", () => {
+    const res = splitByCursor("abcdef", [2, 2]);
+    expect(res).toEqual({ beforeCursor: "ab", afterCursor: "cdef" });
+  });
+
+  it("returns whole value as beforeCursor when selection is not collapsed", () => {
+    const res = splitByCursor("abcdef", [1, 3]);
+    expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
+  });
+
+  it("handles caret at 0", () => {
+    const res = splitByCursor("abc", [0, 0]);
+    expect(res).toEqual({ beforeCursor: "", afterCursor: "abc" });
+  });
+
+  it("handles caret at end", () => {
+    const res = splitByCursor("abc", [3, 3]);
+    expect(res).toEqual({ beforeCursor: "abc", afterCursor: "" });
+  });
+
+  it("treats reversed selection as non-collapsed (browser may return [end,start])", () => {
+    const res = splitByCursor("abcdef", [4, 2]);
+    expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
+  });
+});
+
+describe("extractMetric", () => {
+  it("extracts metric from plain selector", () => {
+    expect(extractMetric("kube_pod_info{job=\"x\"}")).toBe("kube_pod_info");
+  });
+
+  it("extracts metric from plain expr with leading spaces", () => {
+    expect(extractMetric("   http_requests_total")).toBe("http_requests_total");
+  });
+
+  it("extracts metric from expr with braces right after metric", () => {
+    expect(extractMetric("foo_bar{a=\"b\"}")).toBe("foo_bar");
+  });
+
+  it("extracts metric before grouping modifiers (by/without/on/ignoring)", () => {
+    expect(extractMetric("sum(kube_pod_info) by (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) without (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) on (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) ignoring (pod)")).toBe("kube_pod_info");
+  });
+
+  it("returns empty string when no metric found", () => {
+    expect(extractMetric("{job=\"x\"}")).toBe("");
+    expect(extractMetric("")).toBe("");
+    expect(extractMetric("()")).toBe("");
+  });
+});
+
+describe("extractCurrentLabel", () => {
+  it("returns last label before operator", () => {
+    expect(extractCurrentLabel("metric{job=\"foo\", instance=\"bar\"}")).toBe(
+      "instance"
+    );
+  });
+
+  it("supports spaces around operator", () => {
+    expect(extractCurrentLabel("metric{job=\"foo\", instance = \"bar\"}")).toBe(
+      "instance"
+    );
+  });
+
+  it("supports regexp operators", () => {
+    expect(extractCurrentLabel("metric{pod=~\"api-.*\",namespace=\"dev\"}")).toBe(
+      "namespace"
+    );
+  });
+
+  it("supports label chars : - . /", () => {
+    expect(extractCurrentLabel("m{foo-bar.baz/qux=\"1\"}")).toBe("foo-bar.baz/qux");
+  });
+
+  it("returns empty string when no label pattern", () => {
+    expect(extractCurrentLabel("metric{}").trim()).toBe("");
+    expect(extractCurrentLabel("metric")).toBe("");
+  });
+});
+
+describe("extractLabelMatchers", () => {
+  it("returns all matchers (quoted only)", () => {
+    const expr = "metric{job=\"foo\", instance=\"bar\"}";
+    expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\"", "instance=\"bar\""]);
+  });
+
+  it("keeps original spacing", () => {
+    const expr = "metric{ job = \"foo\" , instance = \"bar\" }";
+    expect(extractLabelMatchers(expr)).toEqual(["job = \"foo\"", "instance = \"bar\""]);
+  });
+
+  it("supports !=, =~, !~", () => {
+    const expr = "m{env!=\"prod\",pod=~\"api-.*\",zone!~\"eu-.*\"}";
+    expect(extractLabelMatchers(expr)).toEqual([
+      "env!=\"prod\"",
+      "pod=~\"api-.*\"",
+      "zone!~\"eu-.*\"",
+    ]);
+  });
+
+  it("excludes only the specified currentLabel matcher (exact label, not prefix)", () => {
+    const expr = "m{job=\"foo\", instance=\"bar\", pod=~\"api-.*\"}";
+    expect(extractLabelMatchers(expr, "instance")).toEqual([
+      "job=\"foo\"",
+      "pod=~\"api-.*\"",
+    ]);
+  });
+
+  it("does not exclude other labels that share a prefix with currentLabel", () => {
+    const expr = "m{instance=\"bar\", insight=\"x\"}";
+    expect(extractLabelMatchers(expr, "insight")).toEqual(["instance=\"bar\""]);
+  });
+
+  it("excludes currentLabel matcher even with spaces around operator", () => {
+    const expr = "m{job=\"foo\", instance = \"bar\"}";
+    expect(extractLabelMatchers(expr, "instance")).toEqual(["job=\"foo\""]);
+  });
+
+  it("returns [] when no matchers", () => {
+    expect(extractLabelMatchers("m{}")).toEqual([]);
+    expect(extractLabelMatchers("m")).toEqual([]);
+  });
+
+  it("does not include unclosed quotes", () => {
+    const expr = "m{job=\"foo\", instance=\"ba";
+    expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\""]);
+  });
+});

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/utils/parser.ts

@@ -0,0 +1,52 @@
+
+export const splitByCursor = (
+  value: string,
+  caret: [number, number]
+) => {
+  if (caret[0] !== caret[1]) {
+    return { beforeCursor: value, afterCursor: "" };
+  }
+
+  return {
+    beforeCursor: value.substring(0, caret[0]),
+    afterCursor: value.substring(caret[1]),
+  };
+};
+
+
+export const extractMetric = (expr: string): string => {
+  const fnRegex = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
+  const fnMatch = [...expr.matchAll(fnRegex)];
+
+  if (fnMatch[0]?.groups?.metricName) {
+    return fnMatch[0].groups.metricName;
+  }
+
+  const plainRegex = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
+  const match = [...expr.matchAll(plainRegex)];
+  return match[0]?.groups?.metricName || "";
+};
+
+export const extractCurrentLabel = (expr: string): string => {
+  const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
+  const match = expr.match(regexp);
+  return match ? match[match.length - 1] : "";
+};
+
+
+export const extractLabelMatchers = (
+  expr: string,
+  currentLabel?: string
+): string[] => {
+  const regexp = /([a-z_:-][\w\-.:/]*)\s*(?:=|!=|=~|!~)\s*"[^"]*"/g;
+
+  const matches = [...expr.matchAll(regexp)];
+  // m[1] = label name
+  // m[0] = full matcher string
+
+  if (!currentLabel) return matches.map(m => m[0]);
+
+  return matches
+    .filter(m => m[1] !== currentLabel)
+    .map(m => m[0]);
+};

app/vmui/packages/vmui/src/components/ExploreAlerts/helpers.ts

@@ -17,4 +17,4 @@ export const formatDuration = (raw: number) => {
 export const formatEventTime = (raw: string) => {
   const t = dayjs(raw);
   return t.year() <= 1 ? "Never" : t.format("DD MMM YYYY HH:mm:ss");
-}
+};

app/vmui/packages/vmui/src/components/ExploreAnomaly/AnomalyConfig.tsx

@@ -1,132 +0,0 @@
-import { FC, useState } from "preact/compat";
-import Button from "../Main/Button/Button";
-import TextField from "../Main/TextField/TextField";
-import Modal from "../Main/Modal/Modal";
-import Spinner from "../Main/Spinner/Spinner";
-import { DownloadIcon, ErrorIcon } from "../Main/Icons";
-import useBoolean from "../../hooks/useBoolean";
-import useDeviceDetect from "../../hooks/useDeviceDetect";
-import { useAppState } from "../../state/common/StateContext";
-import classNames from "classnames";
-import "./style.scss";
-import { useQueryState } from "../../state/query/QueryStateContext";
-import { useTimeState } from "../../state/time/TimeStateContext";
-import { getStepFromDuration } from "../../utils/time";
-
-const AnomalyConfig: FC = () => {
-  const { serverUrl } = useAppState();
-  const { isMobile } = useDeviceDetect();
-
-  const {
-    value: isModalOpen,
-    setTrue: setOpenModal,
-    setFalse: setCloseModal,
-  } = useBoolean(false);
-
-  const { query } = useQueryState();
-  const { period } = useTimeState();
-  const [isLoading, setIsLoading] = useState(false);
-  const [textConfig, setTextConfig] = useState<string>("");
-  const [downloadUrl, setDownloadUrl] = useState<string>("");
-  const [error, setError] = useState<string>("");
-
-  const fetchConfig = async () => {
-    setIsLoading(true);
-    try {
-      const queryParam = encodeURIComponent(query[0] || "");
-      const stepParam = encodeURIComponent(period.step || getStepFromDuration(period.end - period.start, false));
-
-      const url = `${serverUrl}/api/vmanomaly/config.yaml?query=${queryParam}&step=${stepParam}`;
-      const response = await fetch(url);
-      const contentType = response.headers.get("Content-Type");
-      if (!response.ok) {
-        const bodyText = await response.text();
-        setError(` ${response.status} ${response.statusText}: ${bodyText}`);
-      } else if (contentType == "application/yaml") {
-        const blob = await response.blob();
-        const yamlAsString = await blob.text();
-        setTextConfig(yamlAsString);
-        setDownloadUrl(URL.createObjectURL(blob));
-      } else {
-        setError("Response Content-Type is not YAML, does `Server URL` point to VMAnomaly server?");
-      }
-    } catch (error) {
-      console.error(error);
-      setError(String(error));
-    }
-    setIsLoading(false);
-  };
-
-  const handleOpenModal = () => {
-    setOpenModal();
-    setError("");
-    URL.revokeObjectURL(downloadUrl);
-    setTextConfig("");
-    setDownloadUrl("");
-    return fetchConfig();
-  };
-
-  return (
-    <>
-      <Button
-        color="secondary"
-        variant="outlined"
-        onClick={handleOpenModal}
-      >
-        Open Config
-      </Button>
-      {isModalOpen && (
-        <Modal
-          title="Download config"
-          onClose={setCloseModal}
-        >
-          <div
-            className={classNames({
-              "vm-anomaly-config": true,
-              "vm-anomaly-config_mobile": isMobile,
-            })}
-          >
-            {isLoading && (
-              <Spinner
-                containerStyles={{ position: "relative" }}
-                message={"Loading config..."}
-              />
-            )}
-            {!isLoading && error && (
-              <div className="vm-anomaly-config-error">
-                <div className="vm-anomaly-config-error__icon"><ErrorIcon/></div>
-                <h3 className="vm-anomaly-config-error__title">Cannot download config</h3>
-                <p className="vm-anomaly-config-error__text">{error}</p>
-              </div>
-            )}
-            {!isLoading && textConfig && (
-              <TextField
-                value={textConfig}
-                label={"config.yaml"}
-                type="textarea"
-                disabled={true}
-              />
-            )}
-            <div className="vm-anomaly-config-footer">
-              {downloadUrl && (
-                <a
-                  href={downloadUrl}
-                  download={"config.yaml"}
-                >
-                  <Button
-                    variant="contained"
-                    startIcon={<DownloadIcon/>}
-                  >
-                    download
-                  </Button>
-                </a>
-              )}
-            </div>
-          </div>
-        </Modal>
-      )}
-    </>
-  );
-};
-
-export default AnomalyConfig;