fix

Related to
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10189

Debug notes
https://github.com/VictoriaMetrics/debug-notes/tree/main/gh10189

.github/ISSUE_TEMPLATE/question.yml

@@ -5,7 +5,7 @@ body:
   - type: textarea
     id: describe-the-component
     attributes:
-      label: Is your question request related to a specific component?
+      label: Is your question related to a specific component?
       placeholder: |
         VictoriaMetrics, vmagent, vmalert, vmui, etc...
     validations:

.github/scripts/lint-changelog-tip.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env sh
+
+set -e
+
+CHANGELOG_FILE="docs/victoriametrics/changelog/CHANGELOG.md"
+
+GITHUB_BASE_REF=${GITHUB_BASE_REF:-"master"}
+GIT_REMOTE=${GIT_REMOTE:-"origin"}
+
+git diff "${GIT_REMOTE}/${GITHUB_BASE_REF}"...HEAD -- $CHANGELOG_FILE > diff.txt
+if ! grep -q "^+" diff.txt; then
+  echo "No additions in CHANGELOG.md"
+  exit 0
+fi
+
+ADDED_LINES=$(grep "^+\S" diff.txt | sed 's/^+//')
+
+START_TIP=$(grep -n "^## tip" "$CHANGELOG_FILE" | head -1 | cut -d: -f1)
+if [ -z "$START_TIP" ]; then
+  echo "ERROR: ${CHANGELOG_FILE} does not contain a ## tip section"
+  exit 1
+fi
+
+END_TIP=$(awk "NR>$START_TIP && /^## / {print NR; exit}" "${CHANGELOG_FILE}")
+if [ -z "$END_TIP" ]; then
+  END_TIP=$(wc -l < "$CHANGELOG_FILE")
+fi
+
+BAD=0
+while IFS= read -r line; do
+  # Grep exact line inside the file and get line numbers
+  MATCHES=$(grep -n -F "$line" "$CHANGELOG_FILE" | cut -d: -f1)
+  for m in $MATCHES; do
+    if [ "$m" -lt "$START_TIP" ] || [ "$m" -gt "$END_TIP" ]; then
+      echo "'$line' on line ${m} is outside ## tip section (lines ${START_TIP}-${END_TIP})"
+      BAD=1
+    fi
+  done
+done << EOF
+$ADDED_LINES
+EOF
+
+if [ "$BAD" -ne 0 ]; then
+  echo "CHANGELOG modifications must be placed inside the ## tip section."
+  exit 1
+fi
+
+echo "CHANGELOG modifications are valid."

.github/workflows/build.yml

@@ -61,7 +61,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/changelog-linter.yml

@@ -0,0 +1,19 @@
+name: 'changelog-linter'
+
+on:
+  pull_request:
+    paths:
+      - "docs/victoriametrics/changelog/CHANGELOG.md"
+
+jobs:
+  tip-lint:
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: 'actions/checkout@v6'
+        with:
+          # needed for proper diff
+          fetch-depth: 0
+
+      - name: 'Validate that changelog changes are under ## tip'
+        run: |
+          GITHUB_BASE_REF=${{ github.base_ref }} ./.github/scripts/lint-changelog-tip.sh

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/codeql-analysis-go.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Set up Go
         id: go

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -97,7 +97,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/vmui.yml

@@ -32,35 +32,41 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
-      - name: Setup Node
-        uses: actions/setup-node@v6
+      - name: Cache node_modules
+        id: cache
+        uses: actions/cache@v5
         with:
-          node-version: '24.x'
+          path: app/vmui/packages/vmui/node_modules
+          key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
+          restore-keys: |
+            vmui-deps-${{ runner.os }}-
 
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make vmui-install
 
       - name: Run lint
         id: lint
         run: make vmui-lint
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run tests
         id: test
         run: make vmui-test
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run typecheck
         id: typecheck
         run: make vmui-typecheck
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
         uses: ataylorme/eslint-annotate-action@v3

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2026 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.4.0
+GOLANGCI_LINT_VERSION := 2.7.2
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -435,7 +435,7 @@ release-vmutils-windows-goarch: \
 		vmctl-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
-	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics $(PPROF_FILE)
 
 fmt:
 	gofmt -l -w -s ./lib
@@ -471,7 +471,23 @@ integration-test:
 
 apptest:
 	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestCluster.*"
+	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+
+integration-test-legacy: victoria-metrics vmbackup vmrestore
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.132.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
+	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
 	GOEXPERIMENT=synctest go test -bench=. ./lib/...
@@ -500,7 +516,8 @@ app-local-windows-goarch:
 	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
-	qtc
+	qtc -dir=lib
+	qtc -dir=app
 
 install-qtc:
 	which qtc || go install github.com/valyala/quicktemplate/qtc@latest

app/victoria-metrics/main.go

@@ -134,6 +134,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/victoria-metrics/self_scraper.go

@@ -10,9 +10,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -27,11 +29,9 @@ var selfScraperWG sync.WaitGroup
 
 func startSelfScraper() {
 	selfScraperStopCh = make(chan struct{})
-	selfScraperWG.Add(1)
-	go func() {
-		defer selfScraperWG.Done()
+	selfScraperWG.Go(func() {
 		selfScraper(*selfScrapeInterval)
-	}()
+	})
 }
 
 func stopSelfScraper() {
@@ -48,6 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
 
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
+	var metadataRows prometheus.MetadataRows
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
@@ -57,8 +58,12 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		// VictoriaMetrics components don't expose metadata yet, only need to parse samples
-		rows.UnmarshalWithErrLogger(s, nil)
+		// Parse metrics and optionally metadata when enabled
+		if prommetadata.IsEnabled() {
+			rows, metadataRows = prometheus.UnmarshalWithMetadata(rows, metadataRows, s, nil)
+		} else {
+			rows.UnmarshalWithErrLogger(s, nil)
+		}
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]
@@ -91,6 +96,19 @@ func selfScraper(scrapeInterval time.Duration) {
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
+		if len(metadataRows.Rows) > 0 {
+			mms := make([]metricsmetadata.Row, 0, len(metadataRows.Rows))
+			for _, mm := range metadataRows.Rows {
+				mms = append(mms, metricsmetadata.Row{
+					MetricFamilyName: bytesutil.ToUnsafeBytes(mm.Metric),
+					Help:             bytesutil.ToUnsafeBytes(mm.Help),
+					Type:             mm.Type,
+				})
+			}
+			if err := vmstorage.AddMetadataRows(mms); err != nil {
+				logger.Errorf("cannot store self-scraped metrics metadata: %s", err)
+			}
+		}
 	}
 	for {
 		select {

app/vmagent/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -244,6 +245,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
@@ -350,6 +352,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(nil, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -644,6 +657,17 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -765,6 +789,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vmagent/newrelic/request_handler.go

@@ -78,7 +78,7 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
-	rowsInserted.Add(len(rows))
+	rowsInserted.Add(samplesCount)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(samplesCount)
 	}

app/vmagent/opentelemetry/request_handler.go

@@ -25,7 +25,7 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// InsertHandler processes metrics from given reader.
+// InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
 		return insertRows(at, tss, mms, nil)

app/vmagent/remotewrite/client.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -203,14 +202,10 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
+		return float64(concurrency)
 	})
-	for i := 0; i < concurrency; i++ {
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			c.runWorker()
-		}()
+	for range concurrency {
+		c.wg.Go(c.runWorker)
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }
@@ -554,9 +549,9 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 // For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
 func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
 	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
 	if err != nil {
-		return nil, fmt.Errorf("zstd: decompress: %s", err)
+		return nil, err
 	}
 
 	return snappy.Encode(nil, plainBlock), nil

app/vmagent/remotewrite/pendingseries.go

@@ -48,11 +48,7 @@ func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Boo
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
-	ps.periodicFlusherWG.Add(1)
-	go func() {
-		defer ps.periodicFlusherWG.Done()
-		ps.periodicFlusher()
-	}()
+	ps.periodicFlusherWG.Go(ps.periodicFlusher)
 	return &ps
 }
 

app/vmagent/remotewrite/relabel.go

@@ -9,14 +9,14 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"go.yaml.in/yaml/v3"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -139,6 +139,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
+
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
@@ -176,19 +177,9 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+// isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
+	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.

app/vmagent/remotewrite/remotewrite.go

@@ -59,7 +59,7 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -176,13 +176,6 @@ func Init() {
 		})
 	}
 
-	if *queues > maxQueues {
-		*queues = maxQueues
-	}
-	if *queues <= 0 {
-		*queues = 1
-	}
-
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
@@ -215,9 +208,7 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
-	configReloaderWG.Add(1)
-	go func() {
-		defer configReloaderWG.Done()
+	configReloaderWG.Go(func() {
 		for {
 			select {
 			case <-configReloaderStopCh:
@@ -227,7 +218,7 @@ func Init() {
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
 		}
-	}()
+	})
 }
 
 func dropDanglingQueues() {
@@ -267,17 +258,6 @@ func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
-	if maxInmemoryBlocks / *queues > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * *queues
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
 	rwctxIdx := make([]int, len(urls))
 	if retryMaxTime.String() != "" {
@@ -292,7 +272,7 @@ func initRemoteWriteCtxs(urls []string) {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
 		rwctxIdx[i] = i
 	}
 
@@ -558,11 +538,9 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	// Push metadata to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
 				if forceDropSamplesOnFailure {
@@ -571,7 +549,7 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 				}
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -603,15 +581,13 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -633,13 +609,11 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 		if len(shard) == 0 {
 			continue
 		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
+		wg.Go(func() {
+			if !rwctx.TryPushTimeSeries(shard, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx, shard)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -848,7 +822,7 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
@@ -863,6 +837,23 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
+	queuesSize := queues.GetOptionalArg(argIdx)
+	if queuesSize > maxQueues {
+		queuesSize = maxQueues
+	} else if queuesSize <= 0 {
+		queuesSize = 1
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks/queuesSize > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * queuesSize
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
@@ -880,16 +871,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, *queues, sanitizedURL)
+	c.init(argIdx, queuesSize, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
 	rd := roundDigits.GetOptionalArg(argIdx)
-	pssLen := *queues
+	pssLen := queuesSize
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
 		// since every pendingSeries can saturate up to a single CPU.

app/vmagent/zabbixconnector/request_handler.go

@@ -0,0 +1,80 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="zabbixconnector"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="zabbixconnector"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := len(rows)
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+
+		samplesLen := len(samples)
+		samples = append(samples, prompb.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+
+		tssDst = append(tssDst, prompb.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/config/config_test.go

@@ -116,7 +116,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "invalid annotations")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -343,7 +343,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 			},
 		},
 	}, true, "bad prometheus expr")
-
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -76,11 +76,14 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
-		fields, _ := q.GetStatsByFields()
-		for i := range fields {
+		labels, err := q.GetStatsLabels()
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
+		}
+		for i := range labels {
 			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
 			// making the result meaningless and may lead to cardinality issues.
-			if fields[i] == "_time" {
+			if labels[i] == "_time" {
 				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 			}
 		}

app/vmalert/main.go

@@ -81,9 +81,7 @@ absolute path to all .tpl files in root.
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
-var (
-	extURL *url.URL
-)
+var extURL *url.URL
 
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
@@ -161,7 +159,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("failed to init: %s", err)
+		logger.Fatalf("failed to create manager: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)

app/vmalert/manager.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -45,13 +46,15 @@ func (m *manager) ruleAPI(gID, rID uint64) (rule.ApiRule, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
-	g, ok := m.groups[gID]
+	group, ok := m.groups[gID]
 	if !ok {
 		return rule.ApiRule{}, fmt.Errorf("can't find group with id %d", gID)
 	}
+	g := group.ToAPI()
+	ruleID := strconv.FormatUint(rID, 10)
 	for _, r := range g.Rules {
-		if r.ID() == rID {
-			return r.ToAPI(), nil
+		if r.ID == ruleID {
+			return r, nil
 		}
 	}
 	return rule.ApiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
@@ -62,17 +65,20 @@ func (m *manager) alertAPI(gID, aID uint64) (*rule.ApiAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
-	g, ok := m.groups[gID]
+	group, ok := m.groups[gID]
 	if !ok {
 		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
+	g := group.ToAPI()
 	for _, r := range g.Rules {
-		ar, ok := r.(*rule.AlertingRule)
-		if !ok {
+		if r.Type != rule.TypeAlerting {
 			continue
 		}
-		if apiAlert := ar.AlertToAPI(aID); apiAlert != nil {
-			return apiAlert, nil
+		alertID := strconv.FormatUint(aID, 10)
+		for _, a := range r.Alerts {
+			if a.ID == alertID {
+				return a, nil
+			}
 		}
 	}
 	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)

app/vmalert/manager_test.go

@@ -65,11 +65,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
@@ -79,7 +77,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/alert.go

@@ -80,14 +80,15 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Type     string
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+	Type      string
+	Labels    map[string]string
+	Value     float64
+	Expr      string
+	AlertID   uint64
+	GroupID   uint64
+	ActiveAt  time.Time
+	For       time.Duration
+	IsPartial bool
 }
 
 var tplHeaders = []string{
@@ -101,6 +102,7 @@ var tplHeaders = []string{
 	"{{ $groupID := .GroupID }}",
 	"{{ $activeAt := .ActiveAt }}",
 	"{{ $for := .For }}",
+	"{{ $isPartial := .IsPartial }}",
 }
 
 // ExecTemplate executes the Alert template for given
@@ -166,8 +168,8 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl
 		ctmpl, _ := tmpl.Clone()
 		ctmpl = ctmpl.Option("missingkey=zero")
 		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
-			r[key] = text
-			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
+			r[key] = err.Error()
+			eg.Add(fmt.Errorf("(key: %q, value: %q): %w", key, text, err))
 			continue
 		}
 		r[key] = buf.String()
@@ -184,13 +186,13 @@ type tplData struct {
 func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
 	tpl, err := tpl.Parse(text)
 	if err != nil {
-		return fmt.Errorf("error parsing annotation template: %w", err)
+		return fmt.Errorf("error parsing template: %w", err)
 	}
 	if !execute {
 		return nil
 	}
 	if err = tpl.Execute(dst, data); err != nil {
-		return fmt.Errorf("error evaluating annotation template: %w", err)
+		return fmt.Errorf("error evaluating template: %w", err)
 	}
 	return nil
 }

app/vmalert/notifier/alertmanager.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -86,6 +86,11 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, alertLabels []
 	err := am.send(ctx, alerts, alertLabels, headers)
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
+		// the context can be cancelled on graceful shutdown
+		// or on group update. So no need to handle the error as usual.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
 		am.metrics.alertsSendErrors.Add(len(alerts))
 		am.lastError = err.Error()
 	} else {
@@ -166,11 +171,6 @@ const alertManagerPath = "/api/v2/alerts"
 func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
 	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration,
 ) (*AlertManager, error) {
-
-	if err := httputil.CheckURL(alertManagerURL); err != nil {
-		return nil, fmt.Errorf("invalid alertmanager URL: %w", err)
-	}
-
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig

app/vmalert/notifier/config_watcher_test.go

@@ -212,18 +212,16 @@ consul_sd_configs:
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/init.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -229,6 +230,9 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
+		if err := httputil.CheckURL(addr); err != nil {
+			return nil, fmt.Errorf("invalid notifier.url %q: %w", addr, err)
+		}
 		addr = strings.TrimSuffix(addr, "/")
 		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
@@ -266,7 +270,7 @@ func GetTargets() map[TargetType][]Target {
 	if getActiveNotifiers == nil {
 		return nil
 	}
-	var targets = make(map[TargetType][]Target)
+	targets := make(map[TargetType][]Target)
 	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()

app/vmalert/notifier/init_test.go

@@ -55,9 +55,9 @@ func TestInitNegative(t *testing.T) {
 		*blackHole = oldBlackHole
 	}()
 
-	f := func(path, addr string, bh bool) {
+	f := func(path string, addr []string, bh bool) {
 		*configPath = path
-		*addrs = flagutil.ArrayString{addr}
+		*addrs = flagutil.ArrayString(addr)
 		*blackHole = bh
 		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
@@ -65,9 +65,12 @@ func TestInitNegative(t *testing.T) {
 	}
 
 	// *configPath, *addrs and *blackhole are mutually exclusive
-	f("/dummy/path", "127.0.0.1", false)
-	f("/dummy/path", "", true)
-	f("", "127.0.0.1", true)
+	f("/dummy/path", []string{"127.0.0.1"}, false)
+	f("/dummy/path", []string{}, true)
+	f("", []string{"127.0.0.1"}, true)
+	// addr cannot be ""
+	f("", []string{""}, false)
+	f("", []string{"127.0.0.1", ""}, false)
 }
 
 func TestBlackHole(t *testing.T) {

app/vmalert/rule/alerting.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -246,16 +247,6 @@ func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
 	return alerts
 }
 
-// GetAlert returns alert if id exists
-func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
-	ar.alertsMu.RLock()
-	defer ar.alertsMu.RUnlock()
-	if ar.alerts == nil {
-		return nil
-	}
-	return ar.alerts[id]
-}
-
 func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
 	if !ar.Debug {
 		return
@@ -321,6 +312,11 @@ type labelSet struct {
 // On k conflicts in origin set, the original value is preferred and copied
 // to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
 func (ls *labelSet) add(k, v string) {
+	// do not add label with empty value, since it has no meaning.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9984
+	if v == "" {
+		return
+	}
 	ls.processed[k] = v
 	ov, ok := ls.origin[k]
 	if !ok {
@@ -350,14 +346,13 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		ls.processed[l.Name] = l.Value
 	}
 
+	// labels only support limited templating variables,
+	// including `labels`, `value` and `expr`, to avoid breaking alert states or causing cardinality issue with results
 	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
 	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to expand labels: %w", err)
-	}
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -368,7 +363,7 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
 		ls.add(alertGroupNameLabel, ar.GroupName)
 	}
-	return ls, nil
+	return ls, err
 }
 
 // execRange executes alerting rule on the given time range similarly to exec.
@@ -394,11 +389,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, err
 		}
 		alertID := hash(ls.processed)
-		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
-		if err != nil {
-			return nil, err
-		}
-		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
+		a := ar.newAlert(s, time.Time{}, ls.processed, nil) // initial alert
 
 		prevT := time.Time{}
 		for i := range s.Values {
@@ -414,8 +405,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
-				// re-template the annotations as active timestamp is changed
-				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -461,7 +450,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			ar.metrics.errors.Inc()
 		}
 	}()
@@ -470,7 +459,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
+	isPartial := isPartialResponse(res)
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartial)
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err
@@ -484,8 +474,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	for i, m := range res.Data {
 		ls, err := ar.expandLabelTemplates(m, qFn)
 		if err != nil {
+			// only set error in current state, but do not break alert processing
 			curState.Err = err
-			return nil, curState.Err
+			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
 		}
 		at := ts
 		alertID := hash(ls.processed)
@@ -495,10 +486,11 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				at = a.ActiveAt
 			}
 		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls, isPartial)
 		if err != nil {
+			// only set error in current state, but do not break alert processing
 			curState.Err = err
-			return nil, curState.Err
+			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
 		}
 		expandedLabels[i] = ls
 		expandedAnnotations[i] = as
@@ -607,25 +599,26 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand label templates: %s", err)
+		return ls, fmt.Errorf("failed to expand label templates: %s", err)
 	}
 	return ls, nil
 }
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet, isPartial bool) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
-		Value:    m.Values[0],
-		Type:     ar.Type.String(),
-		Labels:   ls.origin,
-		Expr:     ar.Expr,
-		AlertID:  hash(ls.processed),
-		GroupID:  ar.GroupID,
-		ActiveAt: activeAt,
-		For:      ar.For,
+		Value:     m.Values[0],
+		Type:      ar.Type.String(),
+		Labels:    ls.origin,
+		Expr:      ar.Expr,
+		AlertID:   hash(ls.processed),
+		GroupID:   ar.GroupID,
+		ActiveAt:  activeAt,
+		For:       ar.For,
+		IsPartial: isPartial,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand annotation templates: %s", err)
+		return as, fmt.Errorf("failed to expand annotation templates: %s", err)
 	}
 	return as, nil
 }

app/vmalert/rule/alerting_test.go

@@ -664,7 +664,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -684,7 +684,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
-			Annotations: map[string]string{"activeAt": "1000"},
+			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
 			ActiveAt:    time.Unix(1, 0),
 			Start:       time.Unix(5, 0),
@@ -705,7 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -1120,7 +1120,7 @@ func TestAlertingRuleLimit_Success(t *testing.T) {
 }
 
 func TestAlertingRule_Template(t *testing.T) {
-	f := func(rule *AlertingRule, metrics []datasource.Metric, alertsExpected map[uint64]*notifier.Alert) {
+	f := func(rule *AlertingRule, metrics []datasource.Metric, isResponsePartial bool, alertsExpected map[uint64]*notifier.Alert) {
 		t.Helper()
 
 		fakeGroup := Group{
@@ -1133,6 +1133,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			entries: make([]StateEntry, 10),
 		}
 		fq.Add(metrics...)
+		fq.SetPartialResponse(isResponsePartial)
 
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -1163,7 +1164,7 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 1, "instance", "foo"),
 		metricWithValueAndLabels(t, 1, "instance", "bar"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
 				"summary": `common: Too high connection number for "foo"`,
@@ -1192,14 +1193,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}".{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 			"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
 		metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 			Labels: map[string]string{
 				alertNameLabel:       "override label",
@@ -1207,7 +1208,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "foo",
 			},
 			Annotations: map[string]string{
-				"summary":     `first: Too high connection number for "foo"`,
+				"summary":     `first: Too high connection number for "foo".`,
 				"description": `override: It is 2 connections for "foo"`,
 			},
 		},
@@ -1218,7 +1219,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "bar",
 			},
 			Annotations: map[string]string{
-				"summary":     `second: Too high connection number for "bar"`,
+				"summary":     `second: Too high connection number for "bar".`,
 				"description": `override: It is 10 connections for "bar"`,
 			},
 		},
@@ -1231,7 +1232,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}.{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1239,7 +1240,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			alertNameLabel, "originAlertname",
 			alertGroupNameLabel, "originGroupname",
 			"instance", "foo"),
-	}, map[uint64]*notifier.Alert{
+	}, true, map[uint64]*notifier.Alert{
 		hash(map[string]string{
 			alertNameLabel:        "OriginLabels",
 			"exported_alertname":  "originAlertname",
@@ -1255,7 +1256,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":            "foo",
 			},
 			Annotations: map[string]string{
-				"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+				"summary": `Alert "originAlertname(originGroupname)" for instance foo. WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.`,
 			},
 		},
 	})
@@ -1370,8 +1371,10 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 
 	ar := &AlertingRule{
 		Labels: map[string]string{
-			"instance": "override", // this should override instance with new value
-			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
+			"instance":      "override", // this should override instance with new value
+			"group":         "vmalert",  // this shouldn't have effect since value in metric is equal
+			"invalid_label": "{{ .Values.mustRuntimeFail }}",
+			"empty_label":   "", // this should be dropped
 		},
 		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
 		Name:      "AlertingRulesError",
@@ -1379,10 +1382,11 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 	}
 
 	expectedOriginLabels := map[string]string{
-		"instance":   "0.0.0.0:8800",
-		"group":      "vmalert",
-		"alertname":  "ConfigurationReloadFailure",
-		"alertgroup": "vmalert",
+		"instance":      "0.0.0.0:8800",
+		"group":         "vmalert",
+		"alertname":     "ConfigurationReloadFailure",
+		"alertgroup":    "vmalert",
+		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1392,11 +1396,12 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
+		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+	if err == nil || !strings.Contains(err.Error(), "error evaluating template") {
+		t.Fatalf("unexpected error %q", err.Error())
 	}
 
 	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {

app/vmalert/rule/recording.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -197,7 +198,7 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 
 	defer func() {
 		rr.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			rr.metrics.errors.Inc()
 		}
 	}()
@@ -236,7 +237,8 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 			Labels: stringToLabels(k),
 			Samples: []prompb.Sample{
 				{Value: decimal.StaleNaN, Timestamp: ts.UnixNano() / 1e6},
-			}})
+			},
+		})
 	}
 	rr.lastEvaluation = curEvaluation
 	return tss, nil
@@ -291,6 +293,11 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompb.TimeSeries {
 	}
 	// add extra labels configured by user
 	for k := range rr.Labels {
+		// do not add label with empty value, since it has no meaning.
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9984
+		if rr.Labels[k] == "" {
+			continue
+		}
 		existingLabel := promrelabel.GetLabelByName(m.Labels, k)
 		if existingLabel != nil { // there is a conflict between extra and existing label
 			if existingLabel.Value == rr.Labels[k] {

app/vmalert/rule/rule_test.go

@@ -65,17 +65,15 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	r := &AlertingRule{state: &ruleState{entries: make([]StateEntry, 20)}}
 	const workers = 50
 	const iterations = 100
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func() {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Go(func() {
 			for i := 0; i < iterations; i++ {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()
 			}
-		}()
+		})
 	}
 	wg.Wait()
 }

app/vmalert/rule/web.go

@@ -209,15 +209,6 @@ func (ar *AlertingRule) AlertsToAPI() []*ApiAlert {
 	return alerts
 }
 
-// AlertToAPI generates apiAlert object from alert by its id(hash)
-func (ar *AlertingRule) AlertToAPI(id uint64) *ApiAlert {
-	a := ar.GetAlert(id)
-	if a == nil {
-		return nil
-	}
-	return NewAlertAPI(ar, a)
-}
-
 // NewAlertAPI creates apiAlert for notifier.Alert
 func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 	aa := &ApiAlert{

app/vmalert/web.go

@@ -412,18 +412,18 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	defer rh.m.groupsMu.RUnlock()
 
 	var gAlerts []rule.GroupAlerts
-	for _, g := range rh.m.groups {
+	for _, group := range rh.m.groups {
 		var alerts []*rule.ApiAlert
+		g := group.ToAPI()
 		for _, r := range g.Rules {
-			a, ok := r.(*rule.AlertingRule)
-			if !ok {
+			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			alerts = append(alerts, a.AlertsToAPI()...)
+			alerts = append(alerts, r.Alerts...)
 		}
 		if len(alerts) > 0 {
 			gAlerts = append(gAlerts, rule.GroupAlerts{
-				Group:  g.ToAPI(),
+				Group:  g,
 				Alerts: alerts,
 			})
 		}
@@ -444,12 +444,12 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 		if !rf.matchesGroup(group) {
 			continue
 		}
-		for _, r := range group.Rules {
-			a, ok := r.(*rule.AlertingRule)
-			if !ok {
+		g := group.ToAPI()
+		for _, r := range g.Rules {
+			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
 		}
 	}
 

app/vmalert/web.qtpl

@@ -9,6 +9,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -78,6 +79,8 @@
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
+        Version {%s buildinfo.Version %} <br>
+
         API:<br>
         {% for _, p := range apiLinks  %}
             {%code p, doc := p[0], p[1] %}
@@ -602,11 +605,11 @@
         <table class="table table-striped table-hover table-sm">
             <thead>
                 <tr>
-                    <th scope="col" title="The time when event was created">Updated at</th>
+                    <th scope="col" title="The time when the rule was executed">Updated at</th>
                     <th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
                     {% if seriesFetchedEnabled %}<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
                     <th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
-                    <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
+                    <th scope="col" class="text-center" title="The time used in execution query request">Execution timestamp</th>
                     <th scope="col" class="text-center" title="cURL command with request example">cURL</th>
                 </tr>
             </thead>

app/vmauth/auth_config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"flag"
 	"fmt"
 	"math"
@@ -94,6 +95,8 @@ type UserInfo struct {
 	rt http.RoundTripper
 
 	requests         *metrics.Counter
+	requestErrors    *metrics.Counter
+	backendRequests  *metrics.Counter
 	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
@@ -105,13 +108,29 @@ type HeadersConf struct {
 	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
-func (ui *UserInfo) beginConcurrencyLimit() error {
+func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		ui.concurrencyLimitReached.Inc()
-		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+		// The number of concurrently executed requests for the given user equals the limt.
+		// Wait until some of the currently executed requests are finished, so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case ui.concurrencyLimitCh <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			err := ctx.Err()
+			if errors.Is(err, context.DeadlineExceeded) {
+				// The current request couldn't be executed until the request timeout.
+				ui.concurrencyLimitReached.Inc()
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
+					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
+			}
+
+			return fmt.Errorf("cannot start executing the request because %d concurrent requests from the user %s are executed: %w",
+				ui.getMaxConcurrentRequests(), ui.name(), err)
+		}
 	}
 }
 
@@ -127,6 +146,28 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 	return mcr
 }
 
+func (ui *UserInfo) stopHealthChecks() {
+	if ui == nil {
+		return
+	}
+
+	if ui.URLPrefix != nil {
+		bus := ui.URLPrefix.bus.Load()
+		bus.stopHealthChecks()
+	}
+	if ui.DefaultURL != nil {
+		bus := ui.DefaultURL.bus.Load()
+		bus.stopHealthChecks()
+	}
+	for i := range ui.URLMaps {
+		um := &ui.URLMaps[i]
+		if um.URLPrefix != nil {
+			bus := um.URLPrefix.bus.Load()
+			bus.stopHealthChecks()
+		}
+	}
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -262,7 +303,7 @@ type URLPrefix struct {
 	// the list of backend urls
 	//
 	// the list can be dynamically updated if `discover_backend_ips` option is set.
-	bus atomic.Pointer[[]*backendURL]
+	bus atomic.Pointer[backendURLs]
 
 	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
 	discoverBackendIPs bool
@@ -286,21 +327,91 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 	}
 }
 
+type backendURLs struct {
+	healthChecksContext context.Context
+	healthChecksCancel  func()
+	healthChecksWG      sync.WaitGroup
+
+	bus []*backendURL
+}
+
+func newBackendURLs() *backendURLs {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &backendURLs{
+		healthChecksContext: ctx,
+		healthChecksCancel:  cancel,
+	}
+}
+
+func (bus *backendURLs) add(u *url.URL) {
+	bus.bus = append(bus.bus, &backendURL{
+		url:                u,
+		healthCheckContext: bus.healthChecksContext,
+		healthCheckWG:      &bus.healthChecksWG,
+	})
+}
+
+func (bus *backendURLs) stopHealthChecks() {
+	bus.healthChecksCancel()
+	bus.healthChecksWG.Wait()
+}
+
 type backendURL struct {
-	brokenDeadline     atomic.Uint64
+	broken atomic.Bool
+
+	healthCheckContext context.Context
+	healthCheckWG      *sync.WaitGroup
+
 	concurrentRequests atomic.Int32
 
 	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
-	ct := fasttime.UnixTimestamp()
-	return ct < bu.brokenDeadline.Load()
+	return bu.broken.Load()
 }
 
 func (bu *backendURL) setBroken() {
-	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	bu.brokenDeadline.Store(deadline)
+	if bu.broken.CompareAndSwap(false, true) {
+		bu.healthCheckWG.Go(func() {
+			bu.runHealthCheck()
+			bu.broken.Store(false)
+		})
+	}
+}
+
+func (bu *backendURL) runHealthCheck() {
+	port := bu.url.Port()
+	if port == "" {
+		port = "80"
+	}
+	addr := net.JoinHostPort(bu.url.Hostname(), port)
+
+	t := time.NewTicker(*failTimeout)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			// Verify network connectivity via TCP dial before marking backend healthy.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9997
+			ctx, cancel := context.WithTimeout(bu.healthCheckContext, time.Second)
+			c, err := netutil.Dialer.DialContext(ctx, "tcp", addr)
+			cancel()
+			if err != nil {
+				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
+					return
+				}
+				logger.Warnf("ignoring the backend at %s for %s because of dial error: %s", addr, *failTimeout, err)
+				continue
+			}
+
+			_ = c.Close()
+			return
+		case <-bu.healthCheckContext.Done():
+			return
+		}
+	}
 }
 
 func (bu *backendURL) get() {
@@ -312,8 +423,8 @@ func (bu *backendURL) put() {
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	pbus := up.bus.Load()
-	return len(*pbus)
+	bus := up.bus.Load()
+	return len(bus.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
@@ -324,16 +435,15 @@ func (up *URLPrefix) getBackendsCount() int {
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
-	pbus := up.bus.Load()
-	bus := *pbus
-	if len(bus) == 0 {
+	bus := up.bus.Load()
+	if len(bus.bus) == 0 {
 		return nil
 	}
 
 	if up.loadBalancingPolicy == "first_available" {
-		return getFirstAvailableBackendURL(bus)
+		return getFirstAvailableBackendURL(bus.bus)
 	}
-	return getLeastLoadedBackendURL(bus, &up.n)
+	return getLeastLoadedBackendURL(bus.bus, &up.n)
 }
 
 func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
@@ -407,25 +517,24 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	cancel()
 
 	// generate new backendURLs for the resolved IPs
-	var busNew []*backendURL
+	busNew := newBackendURLs()
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
-			busNew = append(busNew, &backendURL{
-				url: &buCopy,
-			})
+			busNew.add(&buCopy)
 		}
 	}
 
-	pbus := up.bus.Load()
-	if areEqualBackendURLs(*pbus, busNew) {
+	bus := up.bus.Load()
+	if areEqualBackendURLs(bus.bus, busNew.bus) {
 		return
 	}
 
 	// Store new backend urls
-	up.bus.Store(&busNew)
+	up.bus.Store(busNew)
+	bus.stopHealthChecks()
 }
 
 func areEqualBackendURLs(a, b []*backendURL) bool {
@@ -456,20 +565,23 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	for i := 1; i < len(bus); i++ {
 		if !bus[i].isBroken() {
 			bu = bus[i]
-			break
+			bu.get()
+			return bu
 		}
 	}
-	bu.get()
-	return bu
+	return nil
 }
 
-// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+// getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
+		if bu.isBroken() {
+			return nil
+		}
 		bu.get()
 		return bu
 	}
@@ -494,7 +606,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMinIdx := n % uint32(len(bus))
 	minRequests := bus[buMinIdx].concurrentRequests.Load()
-	for i := uint32(0); i < uint32(len(bus)); i++ {
+	for i := uint32(1); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
@@ -508,6 +620,9 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		}
 	}
 	buMin := bus[buMinIdx]
+	if buMin.isBroken() {
+		return nil
+	}
 	buMin.get()
 	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)
 	return buMin
@@ -626,11 +741,9 @@ func initAuthConfig() {
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	stopCh = make(chan struct{})
-	authConfigWG.Add(1)
-	go func() {
-		defer authConfigWG.Done()
+	authConfigWG.Go(func() {
 		authConfigReloader(sighupCh)
-	}()
+	})
 }
 
 func stopAuthConfig() {
@@ -702,7 +815,7 @@ func reloadAuthConfig() (bool, error) {
 
 	ok, err := reloadAuthConfigData(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 	if !ok {
 		return false, nil
@@ -732,6 +845,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 
 	acPrev := authConfig.Load()
 	if acPrev != nil {
+		acPrev.UnauthorizedUser.stopHealthChecks()
+		for i := range acPrev.Users {
+			acPrev.Users[i].stopHealthChecks()
+		}
+
 		metrics.UnregisterSet(acPrev.ms, true)
 	}
 	metrics.RegisterSet(ac.ms)
@@ -778,6 +896,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
 		}
 		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
@@ -826,6 +946,8 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
 		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
@@ -1060,13 +1182,11 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 	}
 
 	// Initialize up.bus
-	bus := make([]*backendURL, len(up.busOriginal))
-	for i, bu := range up.busOriginal {
-		bus[i] = &backendURL{
-			url: bu,
-		}
+	bus := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		bus.add(bu)
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -753,7 +753,7 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	up.loadBalancingPolicy = "least_loaded"
 
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	fn := func(ns ...int) {
 		t.Helper()
@@ -825,7 +825,7 @@ func TestBrokenBackend(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	// explicitly mark one of the backends as broken
 	bus[1].setBroken()
@@ -848,7 +848,7 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 		up.discoverBackendAddrsIfNeeded()
 		pbus := up.bus.Load()
-		bus := *pbus
+		bus := pbus.bus
 
 		if len(bus) != 1 {
 			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
@@ -942,16 +942,14 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	bus := make([]*backendURL, len(us))
+	bus := newBackendURLs()
 	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		bus[i] = &backendURL{
-			url: pu,
-		}
+		bus.add(pu)
 		urls[i] = pu
 	}
 	up := &URLPrefix{}
@@ -960,7 +958,7 @@ func mustParseURLs(us []string) *URLPrefix {
 	} else {
 		up.vOriginal = us
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 	up.busOriginal = urls
 	return up
 }

app/vmauth/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ioutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -40,22 +41,38 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
-		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
+	idleConnTimeout        = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
-	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
-	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
-	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
-		"in per-user config")
+	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+
+	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
+		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
+		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
+		"See also -requestBufferSize")
+
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests "+
+		"and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 100, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This provides fairness and isolation between users, preventing a single user from consuming all the available resources. "+
+		"It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. "+
+		"This default can be overridden for individual users via max_concurrent_requests option in per-user config. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration to wait before rejecting incoming requests if concurrency limit "+
+		"specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. "+
+		"Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. "+
+		"This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
+	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -151,7 +168,6 @@ func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bo
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
@@ -208,26 +224,124 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 	ui.requests.Inc()
 
-	// Limit the concurrency of requests to backends
-	concurrencyLimitOnce.Do(concurrencyLimitInit)
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(); err != nil {
-			handleConcurrencyLimitError(w, r, err)
-			<-concurrencyLimitCh
-			return
-		}
-	default:
-		concurrentRequestsLimitReached.Inc()
-		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
+	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
+	defer cancel()
+
+	// Acquire global concurrency limit.
+	if err := beginConcurrencyLimit(ctx); err != nil {
 		handleConcurrencyLimitError(w, r, err)
 		return
 	}
+	defer endConcurrencyLimit()
+
+	// Set read deadline for reading the initial chunk for the request body.
+	rc := http.NewResponseController(w)
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		logger.Panicf("BUG: expecting valid deadline for the context")
+	}
+	if err := rc.SetReadDeadline(deadline); err != nil {
+		logger.Panicf("BUG: cannot set read deadline: %s", err)
+	}
+
+	// Read the initial chunk for the request body.
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+	bb, err := bufferRequestBody(ctx, r.Body, userName)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	r.Body = bb
+
+	// Disable the read deadline for the rest of the request body.
+	if err := rc.SetReadDeadline(time.Time{}); err != nil {
+		logger.Panicf("BUG: cannot reset read deadline: %s", err)
+	}
+
+	// Acquire concurrency limit for the given user.
+	if err := ui.beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer ui.endConcurrencyLimit()
+
+	// Process the request.
 	processRequest(w, r, ui)
-	ui.endConcurrencyLimit()
+}
+
+func beginConcurrencyLimit(ctx context.Context) error {
+	concurrencyLimitOnce.Do(concurrencyLimitInit)
+	select {
+	case concurrencyLimitCh <- struct{}{}:
+		return nil
+	default:
+		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
+		// so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			err := ctx.Err()
+			if errors.Is(err, context.DeadlineExceeded) {
+				// The current request couldn't be executed until the request timeout.
+				concurrentRequestsLimitReached.Inc()
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+					*maxQueueDuration, cap(concurrencyLimitCh))
+			}
+			return fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+		}
+	}
+}
+
+func endConcurrencyLimit() {
 	<-concurrencyLimitCh
 }
 
+func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
+	if r == nil {
+		// This is a GET request with nil reader.
+		return nil, nil
+	}
+
+	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
+	if maxBufSize <= 0 {
+		return r, nil
+	}
+
+	lr := ioutil.GetLimitedReader(r, int64(maxBufSize))
+	defer ioutil.PutLimitedReader(lr)
+
+	start := time.Now()
+	buf, err := io.ReadAll(lr)
+	bufferRequestBodyDuration.UpdateDuration(start)
+
+	if err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			rejectSlowClientRequests.Inc()
+
+			d := time.Since(start)
+
+			return nil, &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
+					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
+				StatusCode: http.StatusBadRequest,
+			}
+		}
+
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot read request body: %w", err),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+
+	bb := newBufferedBody(r, buf, maxBufSize)
+	return bb, nil
+}
+
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
@@ -253,9 +367,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	r.Body = rtb
-
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
@@ -263,18 +374,19 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			break
 		}
 		targetURL := bu.url
-		// Don't change path and add request_path query param for default route.
 		if isDefault {
+			// Don't change path and add request_path query param for default route.
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
-		} else { // Update path for regular routes.
+		} else {
+			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
 	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
 		if needLocalRetry && !wasLocalRetry {
 			wasLocalRetry = true
 			goto again
@@ -284,17 +396,20 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if ok {
 			return
 		}
+
 		bu.setBroken()
+		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
 		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
-	ui.backendErrors.Inc()
+	ui.requestErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo, bu *backendURL) (bool, bool) {
+	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
 	req.URL = targetURL
@@ -308,28 +423,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		}
 	}
 
-	rtb, rtbOK := req.Body.(*readTrackingBody)
+	bb, bbOK := req.Body.(*bufferedBody)
+	canRetry := !bbOK || bb.canRetry()
+
 	res, err := ui.rt.RoundTrip(req)
 
-	if ctxErr := r.Context().Err(); ctxErr != nil {
-		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
-		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
-		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
-		err = ctxErr
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
 	}
+
 	if err != nil {
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			// Do not retry canceled or timed out requests
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			if errors.Is(err, context.DeadlineExceeded) {
-				// Timed out request must be counted as errors, since this usually means that the backend is slow.
-				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-				ui.backendErrors.Inc()
-			}
-			return false, false
-		}
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
@@ -337,41 +443,51 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
+			bu.setBroken()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
 			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			if bbOK {
+				bb.resetReader()
+			}
 			return false, true
 		}
 
-		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
+		// Retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, requestURI, targetURL, err)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
-		_ = res.Body.Close()
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request body has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
+
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
+			remoteAddr, requestURI, targetURL, res.StatusCode, retryStatusCodes)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	removeHopHeaders(res.Header)
@@ -381,11 +497,18 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
+
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
+	}
+
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
-
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+		ui.requestErrors.Inc()
 		return true, false
 	}
 	return true, false
@@ -513,6 +636,10 @@ var (
 	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
+	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
+
+	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -596,6 +723,13 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not return any response for the request canceled by the client,
+		// since the connection to the client is already closed.
+		clientCanceledRequests.Inc()
+		return
+	}
+
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
 		Err:        err,
@@ -604,122 +738,78 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// readTrackingBody must be obtained via getReadTrackingBody()
-type readTrackingBody struct {
-	// maxBodySize is the maximum body size to cache in buf.
+// bufferedBody serves two purposes:
+//  1. Enables request retries when the body size does not exceed maxBodySize
+//     by fully buffering the body in memory.
+//  2. Prevents slow clients from reducing effective server capacity by
+//     buffering the request body before acquiring a per-user concurrency slot.
+//
+// See bufferRequestBody for details on how bufferedBody is used.
+type bufferedBody struct {
+	// r contains reader for reading the data after buf is read.
 	//
-	// Bigger bodies cannot be retried.
-	maxBodySize int
-
-	// r contains reader for initial data reading
+	// r is nil if buf contains all the data.
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
-	// If more than maxBodySize is read from r, then cannotRetry is set to true.
+	// buf contains the initial buffer read from r.
 	buf []byte
 
-	// readBuf points to the cached data at buf, which must be read in the next call to Read().
-	readBuf []byte
+	// bufOffset is the offset at buf for already read bytes.
+	bufOffset int
 
-	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
-	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
+	// cannotRetry is set to true after Close() call on non-nil r.
 	cannotRetry bool
-
-	// bufComplete is set to true when buf contains complete request body read from r.
-	bufComplete bool
 }
 
-func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	// do not use sync.Pool there
-	// since http.RoundTrip may still use request body after return
-	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
-	rtb := &readTrackingBody{}
-	if maxBodySize < 0 {
-		maxBodySize = 0
+func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
+	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+
+	if len(buf) < maxBufSize {
+		// Read the full request body into buf.
+		r = nil
 	}
-	rtb.maxBodySize = maxBodySize
 
-	if r == nil {
-		// This is GET request without request body
-		r = (*zeroReader)(nil)
+	return &bufferedBody{
+		r:   r,
+		buf: buf,
 	}
-	rtb.r = r
-	return rtb
-}
-
-type zeroReader struct{}
-
-func (r *zeroReader) Read(_ []byte) (int, error) {
-	return 0, io.EOF
-}
-func (r *zeroReader) Close() error {
-	return nil
 }
 
 // Read implements io.Reader interface.
-func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(rtb.readBuf) > 0 {
-		n := copy(p, rtb.readBuf)
-		rtb.readBuf = rtb.readBuf[n:]
+func (bb *bufferedBody) Read(p []byte) (int, error) {
+	if bb.cannotRetry {
+		return 0, fmt.Errorf("cannot read already closed body")
+	}
+	if bb.bufOffset < len(bb.buf) {
+		n := copy(p, bb.buf[bb.bufOffset:])
+		bb.bufOffset += n
 		return n, nil
 	}
-
-	if rtb.r == nil {
-		if rtb.bufComplete {
-			return 0, io.EOF
-		}
-		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+	if bb.r == nil {
+		return 0, io.EOF
 	}
-
-	n, err := rtb.r.Read(p)
-	if rtb.cannotRetry {
-		return n, err
-	}
-
-	if len(rtb.buf)+n > rtb.maxBodySize {
-		rtb.cannotRetry = true
-		return n, err
-	}
-	rtb.buf = append(rtb.buf, p[:n]...)
-	if err == io.EOF {
-		rtb.bufComplete = true
-	}
-	return n, err
+	return bb.r.Read(p)
 }
 
-func (rtb *readTrackingBody) canRetry() bool {
-	if rtb.cannotRetry {
-		return false
-	}
-	if rtb.bufComplete {
-		return true
-	}
-	return rtb.r != nil
+func (bb *bufferedBody) canRetry() bool {
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
-func (rtb *readTrackingBody) Close() error {
-	if !rtb.cannotRetry {
-		rtb.readBuf = rtb.buf
-	} else {
-		rtb.readBuf = nil
+func (bb *bufferedBody) Close() error {
+	bb.resetReader()
+	if bb.r != nil {
+		bb.cannotRetry = true
+		return bb.r.Close()
 	}
-
-	// Close rtb.r only if the request body is completely read or if it is too big.
-	// http.Roundtrip performs body.Close call even without any Read calls,
-	// so this hack allows us to reuse request body.
-	if rtb.bufComplete || rtb.cannotRetry {
-		if rtb.r == nil {
-			return nil
-		}
-		err := rtb.r.Close()
-		rtb.r = nil
-		return err
-	}
-
 	return nil
 }
 
+func (bb *bufferedBody) resetReader() {
+	bb.bufOffset = 0
+}
+
 func debugInfo(u *url.URL, r *http.Request) string {
 	s := &strings.Builder{}
 	fmt.Fprintf(s, " (host: %q; ", r.Host)

app/vmauth/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
@@ -546,28 +548,300 @@ func (w *fakeResponseWriter) WriteHeader(statusCode int) {
 	}
 }
 
-func TestReadTrackingBody_RetrySuccess(t *testing.T) {
+// This is needed for net/http.ResponseController
+func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
+	return nil
+}
+
+func TestBufferRequestBody_Success(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	f := func(body *bytes.Buffer, requestBufferSizeFlag, maxRequestBodySizeToRetryFlag string) {
+		t.Helper()
+
+		expectedResponse := "statusCode=200"
+		if body.Len() > 0 {
+			expectedResponse += "\n" + body.String()
+		}
+
+		if err := requestBufferSize.Set(requestBufferSizeFlag); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		if err := maxRequestBodySizeToRetry.Set(maxRequestBodySizeToRetryFlag); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if !backendCalled {
+			t.Fatalf("backend is not called")
+		}
+	}
+
+	// no body, no buffering, no retry
+	f(bytes.NewBuffer(nil), "0", "0")
+
+	// no body, buffering on, no retry
+	f(bytes.NewBuffer(nil), "100", "0")
+
+	// no body, no buffering, retry on
+	f(bytes.NewBuffer(nil), "0", "100")
+
+	// no body, buffering on, retry on
+	f(bytes.NewBuffer(nil), "100", "100")
+
+	// body smaller than buffer, retry max on
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "101", "101")
+
+	// body smaller than buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "501", "0")
+
+	// body same size as buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "500", "0")
+
+	// body bigger than a buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "499", "0")
+
+	// body bigger than tmpBuf 8KiB used in buffering
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16384", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16385", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16383", "")
+}
+
+func TestBufferRequestBody_Failure(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	defaultMaxQueueDuration := *maxQueueDuration
+	defer func() {
+		*maxQueueDuration = defaultMaxQueueDuration
+	}()
+
+	f := func(body *mockBody, expectedResponse string) {
+		t.Helper()
+
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+		if err := requestBufferSize.Set("2048"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		*maxQueueDuration = 100 * time.Millisecond
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if backendCalled {
+			t.Fatalf("backend is called")
+		}
+	}
+
+	// an error at the beginning of reading
+	f(&mockBody{err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+
+	// an error after reading 1024 bytes, buffer size is 2048 bytes
+	f(&mockBody{head: make([]byte, 1024), err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+}
+
+type mockBody struct {
+	head []byte
+	err  error
+	tail []byte
+}
+
+func (r *mockBody) Read(p []byte) (n int, err error) {
+	if len(r.head) > 0 {
+		n = copy(p, r.head)
+		r.head = r.head[n:]
+		return n, nil
+	}
+
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	if len(r.tail) > 0 {
+		n = copy(p, r.tail)
+		r.tail = r.tail[n:]
+		return n, nil
+	}
+
+	return 0, io.EOF
+}
+
+func TestBufferedBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
-			data, err := io.ReadAll(rtb)
+			data, err := io.ReadAll(rb)
 			if err != nil {
 				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
 			}
 			if string(data) != s {
 				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
 			}
-			if err := rtb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
-			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true at iteration %d", i)
+			if err := rb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing bufferedBody at iteration %d: %s", i, err)
 			}
 		}
 	}
@@ -577,19 +851,48 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
+func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
+			t.Fatalf("canRetry must return true")
+		}
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
-			n, err := io.ReadFull(rtb, buf)
+			n, err := io.ReadFull(rb, buf)
 			if err != nil {
 				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
 			}
@@ -599,26 +902,20 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 			if string(buf) != s[:i] {
 				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
 			}
-			if err := rtb.Close(); err != nil {
+			if err := rb.Close(); err != nil {
 				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
 			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
-			}
 		}
 
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 	}
 
@@ -627,30 +924,53 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
+func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set("0"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if canRetry {
+			t.Fatalf("canRetry() must return false because of too big request body")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rtb, buf)
+		n, err := io.ReadFull(rb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after reading one byte")
-		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
@@ -658,14 +978,11 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 		if dataRead != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		data, err = io.ReadAll(rtb)
+		data, err = io.ReadAll(rb)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -679,35 +996,48 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		data, err = io.ReadAll(rb)
+		if err != nil {
+			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		data, err = io.ReadAll(rtb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 

app/vmbackup/main.go

@@ -212,7 +212,7 @@ func newSrcFS() (*fslocal.FS, error) {
 }
 
 func newDstFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *dst)
+	fs, err := actions.NewRemoteFS(ctx, *dst, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
@@ -255,7 +255,7 @@ func newOriginFS(ctx context.Context) (common.OriginFS, error) {
 	if len(*origin) == 0 {
 		return &fsnil.FS{}, nil
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}
@@ -266,7 +266,7 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	if len(*origin) == 0 {
 		return nil, fmt.Errorf("-origin cannot be empty when -snapshotName and -snapshot.createURL aren't set")
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}

app/vmctl/flags.go

@@ -123,32 +123,32 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination.",
 		},
 		&cli.StringFlag{
 			Name:  vmCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmServerName,
-			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
+			Usage: "Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
+			Usage: "Whether to skip tls verification when connecting to '--vm-addr'",
 			Value: false,
 		},
 		&cli.IntFlag{
@@ -468,7 +468,7 @@ var (
 			Name: vmNativeFilterMatch,
 			Usage: "Time series selector to match series for export. For example, select {instance!=\"localhost\"} will " +
 				"match all series with \"instance\" label different to \"localhost\".\n" +
-				" See more details here https://github.com/VictoriaMetrics/VictoriaMetrics#how-to-export-data-in-native-format",
+				" See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format",
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{
@@ -598,7 +598,7 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
@@ -625,8 +625,8 @@ var (
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
 			Usage: "Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format " +
-				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API." +
-				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
+				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. " +
+				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. " +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},

app/vmctl/influx.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -37,7 +38,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run() error {
+func (ip *influxProcessor) run(ctx context.Context) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -47,7 +48,7 @@ func (ip *influxProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 
@@ -62,10 +63,8 @@ func (ip *influxProcessor) run() error {
 	ip.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(ip.cc)
-	for i := 0; i < ip.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range ip.cc {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if err := ip.do(s); err != nil {
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
@@ -73,7 +72,7 @@ func (ip *influxProcessor) run() error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import

app/vmctl/main.go

@@ -103,7 +103,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose))
-					return otsdbProcessor.run()
+					return otsdbProcessor.run(ctx)
 				},
 			},
 			{
@@ -164,7 +164,7 @@ func main() {
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode),
 						c.Bool(globalVerbose))
-					return processor.run()
+					return processor.run(ctx)
 				},
 			},
 			{
@@ -279,7 +279,7 @@ func main() {
 						cc:        c.Int(promConcurrency),
 						isVerbose: c.Bool(globalVerbose),
 					}
-					return pp.run()
+					return pp.run(ctx)
 				},
 			},
 			{

app/vmctl/opentsdb.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -37,7 +38,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbos
 	}
 }
 
-func (op *otsdbProcessor) run() error {
+func (op *otsdbProcessor) run(ctx context.Context) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -53,7 +54,7 @@ func (op *otsdbProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d metrics to import. Continue?", len(metrics))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 	op.im.ResetStats()
@@ -88,10 +89,8 @@ func (op *otsdbProcessor) run() error {
 			bar.Finish()
 		}(bar)
 		var wg sync.WaitGroup
-		wg.Add(op.otsdbcc)
-		for i := 0; i < op.otsdbcc; i++ {
-			go func() {
-				defer wg.Done()
+		for range op.otsdbcc {
+			wg.Go(func() {
 				for s := range seriesCh {
 					if err := op.do(s); err != nil {
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
@@ -99,7 +98,7 @@ func (op *otsdbProcessor) run() error {
 					}
 					bar.Increment()
 				}
-			}()
+			})
 		}
 		/*
 			Loop through all series for this metric, processing all retentions and time ranges

app/vmctl/prometheus.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
@@ -30,7 +33,7 @@ type prometheusProcessor struct {
 	isVerbose bool
 }
 
-func (pp *prometheusProcessor) run() error {
+func (pp *prometheusProcessor) run(ctx context.Context) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -39,7 +42,7 @@ func (pp *prometheusProcessor) run() error {
 		return fmt.Errorf("found no blocks to import")
 	}
 	question := fmt.Sprintf("Found %d blocks to import. Continue?", len(blocks))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 
@@ -60,19 +63,19 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
-		var labels []vm.LabelPair
+		var labelPairs []vm.LabelPair
 		series := ss.At()
 
-		for _, label := range series.Labels() {
+		series.Labels().Range(func(label labels.Label) {
 			if label.Name == "__name__" {
 				name = label.Value
-				continue
+				return
 			}
-			labels = append(labels, vm.LabelPair{
-				Name:  label.Name,
-				Value: label.Value,
+			labelPairs = append(labelPairs, vm.LabelPair{
+				Name:  strings.Clone(label.Name),
+				Value: strings.Clone(label.Value),
 			})
-		}
+		})
 		if name == "" {
 			return fmt.Errorf("failed to find `__name__` label in labelset for block %v", b.Meta().ULID)
 		}
@@ -98,7 +101,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 		}
 		ts := vm.TimeSeries{
 			Name:       name,
-			LabelPairs: labels,
+			LabelPairs: labelPairs,
 			Timestamps: timestamps,
 			Values:     values,
 		}
@@ -121,10 +124,8 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	pp.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(pp.cc)
-	for i := 0; i < pp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range pp.cc {
+		wg.Go(func() {
 			for br := range blockReadersCh {
 				if err := pp.do(br); err != nil {
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
@@ -132,7 +133,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 	// any error breaks the import
 	for _, br := range blocks {

app/vmctl/remoteread.go

@@ -47,7 +47,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 
 	question := fmt.Sprintf("Selected time range %q - %q will be split into %d ranges according to %q step. Continue?",
 		rrp.filter.timeStart.String(), rrp.filter.timeEnd.String(), len(ranges), rrp.filter.chunk)
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 
@@ -66,10 +66,8 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	errCh := make(chan error)
 
 	var wg sync.WaitGroup
-	wg.Add(rrp.cc)
-	for i := 0; i < rrp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range rrp.cc {
+		wg.Go(func() {
 			for r := range rangeC {
 				if err := rrp.do(ctx, r); err != nil {
 					errCh <- fmt.Errorf("request failed for: %s", err)
@@ -77,7 +75,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	for _, r := range ranges {

app/vmctl/utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -15,7 +16,7 @@ const barTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "
 // isSilent should be inited in main
 var isSilent bool
 
-func prompt(question string) bool {
+func prompt(ctx context.Context, question string) bool {
 	if isSilent {
 		return true
 	}
@@ -25,15 +26,32 @@ func prompt(question string) bool {
 	}
 	reader := bufio.NewReader(os.Stdin)
 	fmt.Print(question, " [Y/n] ")
-	answer, err := reader.ReadString('\n')
-	if err != nil {
+
+	answerCh := make(chan string, 1)
+	errCh := make(chan error, 1)
+
+	go func() {
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			errCh <- err
+			return
+		}
+		answerCh <- answer
+	}()
+
+	select {
+	case <-ctx.Done():
+		fmt.Println("\nCanceled.")
+		return false
+	case err := <-errCh:
 		panic(err)
+	case answer := <-answerCh:
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer == "" || answer == "yes" || answer == "y" {
+			return true
+		}
+		return false
 	}
-	answer = strings.TrimSpace(strings.ToLower(answer))
-	if answer == "" || answer == "yes" || answer == "y" {
-		return true
-	}
-	return false
 }
 
 func wrapErr(vmErr *vm.ImportError, verbose bool) error {

app/vmctl/vm/vm.go

@@ -156,15 +156,13 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		cfg.BatchSize = 1e5
 	}
 
-	im.wg.Add(int(cfg.Concurrency))
-	for i := 0; i < int(cfg.Concurrency); i++ {
+	for i := range int(cfg.Concurrency) {
 		pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i)
 		bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0)
 
-		go func(bar barpool.Bar) {
-			defer im.wg.Done()
+		im.wg.Go(func() {
 			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
-		}(bar)
+		})
 	}
 	im.ResetStats()
 	return im, nil

app/vmctl/vm_native.go

@@ -79,7 +79,7 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 			return fmt.Errorf("failed to get tenants: %w", err)
 		}
 		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	}
@@ -233,7 +233,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		// do not prompt for intercluster because there could be many tenants,
 		// and we don't want to interrupt the process when moving to the next tenant.
 		question := foundSeriesMsg + ". Continue?"
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	} else {
@@ -249,9 +249,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var wg sync.WaitGroup
 	for i := 0; i < p.cc; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
@@ -266,7 +264,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 					}
 				}
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import

app/vminsert/common/insert_ctx.go

@@ -11,9 +11,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -50,8 +52,9 @@ var (
 type InsertCtx struct {
 	Labels sortedLabels
 
-	mrs            []storage.MetricRow
-	metricNamesBuf []byte
+	mrs           []storage.MetricRow
+	mms           []metricsmetadata.Row
+	metricNameBuf []byte
 
 	relabelCtx    relabel.Ctx
 	streamAggrCtx streamAggrCtx
@@ -73,8 +76,13 @@ func (ctx *InsertCtx) Reset(rowsLen int) {
 	}
 	mrs = slicesutil.SetLength(mrs, rowsLen)
 	ctx.mrs = mrs[:0]
+	mms := ctx.mms
+	for i := range mms {
+		cleanMetricMetadata(&mms[i])
+	}
+	ctx.mms = mms[:0]
 
-	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
 	ctx.relabelCtx.Reset()
 	ctx.streamAggrCtx.Reset()
 	ctx.skipStreamAggr = false
@@ -84,11 +92,20 @@ func cleanMetricRow(mr *storage.MetricRow) {
 	mr.MetricNameRaw = nil
 }
 
+func cleanMetricMetadata(mm *metricsmetadata.Row) {
+	mm.MetricFamilyName = nil
+	mm.Unit = nil
+	mm.Help = nil
+	mm.Type = 0
+	mm.ProjectID = 0
+	mm.AccountID = 0
+}
+
 func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
-	start := len(ctx.metricNamesBuf)
-	ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
-	ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
-	metricNameRaw := ctx.metricNamesBuf[start:]
+	start := len(ctx.metricNameBuf)
+	ctx.metricNameBuf = append(ctx.metricNameBuf, prefix...)
+	ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf, labels)
+	metricNameRaw := ctx.metricNameBuf[start:]
 	return metricNameRaw[:len(metricNameRaw):len(metricNameRaw)]
 }
 
@@ -143,7 +160,7 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.MetricNameRaw = metricNameRaw
 	mr.Timestamp = timestamp
 	mr.Value = value
-	if len(ctx.metricNamesBuf) > 16*1024*1024 {
+	if len(ctx.metricNameBuf) > 16*1024*1024 {
 		if err := ctx.FlushBufs(); err != nil {
 			return err
 		}
@@ -151,6 +168,55 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	return nil
 }
 
+// WriteMetadata writes given prometheus protobuf  metadata into the storage.
+func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
+	if len(mmpbs) == 0 {
+		return nil
+	}
+	mms := ctx.mms
+	mms = slicesutil.SetLength(mms, len(mmpbs))
+	for idx, mmpb := range mmpbs {
+		mm := &mms[idx]
+		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
+		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
+		mm.Type = mmpb.Type
+		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+	}
+
+	err := vmstorage.AddMetadataRows(mms)
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store metrics metadata: %w", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
+// WritePromMetadata writes given prometheus metric metadata into the storage
+func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
+	if len(mmps) == 0 {
+		return nil
+	}
+	mms := ctx.mms
+	mms = slicesutil.SetLength(mms, len(mmps))
+	for idx, mmpb := range mmps {
+		mm := &mms[idx]
+		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
+		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
+		mm.Type = mmpb.Type
+	}
+
+	err := vmstorage.AddMetadataRows(mms)
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store prometheus metrics metadata: %w", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
 // AddLabelBytes adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.

app/vminsert/common/streamaggr.go

@@ -111,9 +111,7 @@ func InitStreamAggr() {
 	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 
 	// Start config reloader.
-	saCfgReloaderWG.Add(1)
-	go func() {
-		defer saCfgReloaderWG.Done()
+	saCfgReloaderWG.Go(func() {
 		for {
 			select {
 			case <-sighupCh:
@@ -122,7 +120,7 @@ func InitStreamAggr() {
 			}
 			reloadStreamAggrConfig()
 		}
-	}()
+	})
 }
 
 func reloadStreamAggrConfig() {

app/vminsert/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -231,6 +232,17 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -423,6 +435,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vminsert/opentelemetry/request_handler.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
@@ -14,8 +15,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentelemetry"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentelemetry"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="opentelemetry"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="opentelemetry"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
 // InsertHandler processes opentelemetry metrics.
@@ -33,12 +35,12 @@ func InsertHandler(req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
-		return insertRows(tss, extraLabels)
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(tss, mms, extraLabels)
 	})
 }
 
-func insertRows(tss []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -75,5 +77,14 @@ func insertRows(tss []prompb.TimeSeries, extraLabels []prompb.Label) error {
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
-	return ctx.FlushBufs()
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/prometheusimport/request_handler.go

@@ -1,6 +1,7 @@
 package prometheusimport
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
@@ -15,8 +16,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="prometheus"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="prometheus"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
@@ -30,14 +32,14 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, _ []prometheus.Metadata) error {
-		return insertRows(rows, extraLabels)
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
+		return insertRows(rows, mms, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(rows []prometheus.Row, extraLabels []prompb.Label) error {
+func insertRows(rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -64,5 +66,15 @@ func insertRows(rows []prometheus.Row, extraLabels []prompb.Label) error {
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
-	return ctx.FlushBufs()
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+
+	if prommetadata.IsEnabled() {
+		if err := ctx.WritePromMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/prompush/push.go

@@ -4,13 +4,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	rowsInserted         = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
+	rowsPerInsert        = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	metadataRowsInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="promscrape"}`)
 )
 
 const maxRowsPerBlock = 10000
@@ -41,6 +43,13 @@ func Push(wr *prompb.WriteRequest) {
 		}
 		push(ctx, tssBlock)
 	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(wr.Metadata); err != nil {
+			logger.Errorf("cannot write promscrape metrics metadata to storage: %s", err)
+		} else {
+			metadataRowsInserted.Add(len(wr.Metadata))
+		}
+	}
 }
 
 func push(ctx *common.InsertCtx, tss []prompb.TimeSeries) {

app/vminsert/promremotewrite/request_handler.go

@@ -1,10 +1,12 @@
 package promremotewrite
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -12,8 +14,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
@@ -23,12 +26,12 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
-		return insertRows(tss, extraLabels)
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(tss, mms, extraLabels)
 	})
 }
 
-func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -68,5 +71,15 @@ func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompb.Label) erro
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
-	return ctx.FlushBufs()
+
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/zabbixconnector/request_handler.go

@@ -0,0 +1,67 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="zabbixconnector"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(rows, extraLabels)
+	})
+}
+
+func insertRows(rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsTotal := len(rows)
+	ctx.Reset(rowsTotal)
+	hasRelabeling := relabel.HasRelabeling()
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		for k := range r.Tags {
+			t := &r.Tags[k]
+			ctx.AddLabelBytes(t.Key, t.Value)
+		}
+		for k := range extraLabels {
+			label := &extraLabels[k]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		ctx.SortLabelsIfNeeded()
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vmrestore/main.go

@@ -104,7 +104,7 @@ func newDstFS() (*fslocal.FS, error) {
 }
 
 func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *src)
+	fs, err := actions.NewRemoteFS(ctx, *src, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", *src, err)
 	}

app/vmselect/graphite/transform.go

@@ -3896,27 +3896,9 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 	seriesCh := make(chan *series, goroutines)
 	errCh := make(chan error, 1)
 	var wg sync.WaitGroup
-	wg.Add(goroutines)
-	go func() {
-		var err error
-		for {
-			s, e := nextSeries()
-			if e != nil || s == nil {
-				err = e
-				break
-			}
-			seriesCh <- s
-		}
-		close(seriesCh)
-		wg.Wait()
-		close(resultCh)
-		errCh <- err
-		close(errCh)
-	}()
 	var skipProcessing atomic.Bool
-	for i := 0; i < goroutines; i++ {
-		go func() {
-			defer wg.Done()
+	for range goroutines {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if skipProcessing.Load() {
 					continue
@@ -3934,8 +3916,24 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 					}
 				}
 			}
-		}()
+		})
 	}
+	go func() {
+		var err error
+		for {
+			s, e := nextSeries()
+			if e != nil || s == nil {
+				err = e
+				break
+			}
+			seriesCh <- s
+		}
+		close(seriesCh)
+		wg.Wait()
+		close(resultCh)
+		errCh <- err
+		close(errCh)
+	}()
 	wrapper := func() (*series, error) {
 		r := <-resultCh
 		if r == nil {

app/vmselect/main.go

@@ -421,6 +421,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/metadata":
+		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+		metadataRequests.Inc()
+		if err := prometheus.MetadataHandler(qt, startTime, w, r); err != nil {
+			metadataErrors.Inc()
+			httpserver.SendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+
 	default:
 		return false
 	}
@@ -510,7 +520,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"for accessing vmalert flag '-vmalert.proxyURL' must be configured"}`)
 			return true
 		}
-		proxyVMAlertRequests(w, r)
+		proxyVMAlertRequests(w, r, path)
 		return true
 	}
 
@@ -548,7 +558,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
@@ -558,7 +568,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
@@ -568,18 +578,12 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprint(w, `{"status":"success","data":{"notifiers":[]}}`)
 		return true
-	case "/api/v1/metadata":
-		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
-		metadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
-		return true
 	case "/api/v1/status/buildinfo":
 		buildInfoRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -708,7 +712,9 @@ var (
 	alertsRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	notifiersRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/notifiers"}`)
 
-	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
+	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
+	metadataErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/metadata"}`)
+
 	buildInfoRequests      = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
 	queryExemplarsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/query_exemplars"}`)
 
@@ -719,7 +725,7 @@ var (
 	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
-func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
+func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 	defer func() {
 		err := recover()
 		if err == nil || err == http.ErrAbortHandler {
@@ -730,8 +736,10 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	r.Host = vmalertProxyHost
-	vmalertProxy.ServeHTTP(w, r)
+	req := r.Clone(r.Context())
+	req.URL.Path = strings.TrimPrefix(path, "prometheus")
+	req.Host = vmalertProxyHost
+	vmalertProxy.ServeHTTP(w, req)
 }
 
 var (

app/vmselect/netstorage/netstorage.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 )
 
 var (
@@ -295,14 +296,12 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := range workChs {
-		wg.Add(1)
-		qtChild := qt.NewChild("worker #%d", i)
-		go func(workerID uint) {
-			timeseriesWorker(qtChild, workChs, workerID)
+	for workerID := range workChs {
+		qtChild := qt.NewChild("worker #%d", workerID)
+		wg.Go(func() {
+			timeseriesWorker(qtChild, workChs, uint(workerID))
 			qtChild.Done()
-			wg.Done()
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 
@@ -513,12 +512,10 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go func(workerID uint) {
-			unpackWorker(workChs, workerID)
-			wg.Done()
-		}(uint(i))
+	for workerID := range workers {
+		wg.Go(func() {
+			unpackWorker(workChs, uint(workerID))
+		})
 	}
 	wg.Wait()
 
@@ -865,6 +862,23 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	return labelValues, nil
 }
 
+// GetMetricsMetadata returns time series metric names metadata for the given args
+func GetMetricsMetadata(qt *querytracer.Tracer, limit int, metricName string) ([]*metricsmetadata.Row, error) {
+	qt = qt.NewChild("get metrics metadata: limit=%d, metric_name=%q", limit, metricName)
+	defer qt.Done()
+
+	metadata := vmstorage.Storage.GetMetadataRows(qt, limit, metricName)
+
+	sort.Slice(metadata, func(i, j int) bool {
+		return string(metadata[i].MetricFamilyName) < string(metadata[j].MetricFamilyName)
+	})
+	if limit > 0 && len(metadata) >= limit {
+		metadata = metadata[:limit]
+	}
+
+	return metadata, nil
+}
+
 // GraphiteTagValues returns tag values for the given tagName until the given deadline.
 func GraphiteTagValues(qt *querytracer.Tracer, tagName, filter string, limit int, deadline searchutil.Deadline) ([]string, error) {
 	qt = qt.NewChild("get graphite tag values for tagName=%s, filter=%s, limit=%d", tagName, filter, limit)
@@ -1002,12 +1016,10 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		mustStop      atomic.Bool
 	)
 	var wg sync.WaitGroup
-	wg.Add(gomaxprocs)
-	for i := 0; i < gomaxprocs; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range gomaxprocs {
+		wg.Go(func() {
 			for xw := range workCh {
-				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
+				if err := f(&xw.mn, &xw.b, tr, uint(workerID)); err != nil {
 					errGlobalLock.Lock()
 					if errGlobal == nil {
 						errGlobal = err
@@ -1018,7 +1030,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 				xw.reset()
 				exportWorkPool.Put(xw)
 			}
-		}(uint(i))
+		})
 	}
 
 	// Feed workers with work

app/vmselect/prometheus/metadata_response.qtpl

@@ -0,0 +1,35 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+) %}
+
+{% stripspace %}
+MetadataResponse generates response for /api/v1/metadata
+See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+{% func MetadataResponse( result []*metricsmetadata.Row, qt *querytracer.Tracer) %}
+{
+	"status":"success",
+	"data": {
+		{% code
+			mapItems := len(result)
+			currentItem := 0
+		 %}
+		{% for _, row := range result %}
+			"{%s string(row.MetricFamilyName) %}": [
+					{
+						"type": {%q= row.Type.String() %},
+            {% if len(row.Unit) > 0 -%}
+						"unit": {%q= string(row.Unit) %},
+            {% endif -%}
+						"help": {%q= string(row.Help) %}
+					}
+			]
+			{% if currentItem != mapItems-1 %},{% endif %}
+			{% code currentItem++ %}
+		{% endfor %}
+	}
+	{%= dumpQueryTrace(qt) %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/prometheus/metadata_response.qtpl.go

@@ -0,0 +1,108 @@
+// Code generated by qtc from "metadata_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/metadata_response.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/metadata_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+)
+
+// MetadataResponse generates response for /api/v1/metadataSee https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+func StreamMetadataResponse(qw422016 *qt422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+	qw422016.N().S(`{"status":"success","data": {`)
+//line app/vmselect/prometheus/metadata_response.qtpl:14
+	mapItems := len(result)
+	currentItem := 0
+
+//line app/vmselect/prometheus/metadata_response.qtpl:17
+	for _, row := range result {
+//line app/vmselect/prometheus/metadata_response.qtpl:17
+		qw422016.N().S(`"`)
+//line app/vmselect/prometheus/metadata_response.qtpl:18
+		qw422016.E().S(string(row.MetricFamilyName))
+//line app/vmselect/prometheus/metadata_response.qtpl:18
+		qw422016.N().S(`": [{"type":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:20
+		qw422016.N().Q(row.Type.String())
+//line app/vmselect/prometheus/metadata_response.qtpl:20
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:21
+		if len(row.Unit) > 0 {
+//line app/vmselect/prometheus/metadata_response.qtpl:21
+			qw422016.N().S(`"unit":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:22
+			qw422016.N().Q(string(row.Unit))
+//line app/vmselect/prometheus/metadata_response.qtpl:22
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:23
+		}
+//line app/vmselect/prometheus/metadata_response.qtpl:23
+		qw422016.N().S(`"help":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:24
+		qw422016.N().Q(string(row.Help))
+//line app/vmselect/prometheus/metadata_response.qtpl:24
+		qw422016.N().S(`}]`)
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+		if currentItem != mapItems-1 {
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+		}
+//line app/vmselect/prometheus/metadata_response.qtpl:28
+		currentItem++
+
+//line app/vmselect/prometheus/metadata_response.qtpl:29
+	}
+//line app/vmselect/prometheus/metadata_response.qtpl:29
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/metadata_response.qtpl:31
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:31
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}
+
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+func WriteMetadataResponse(qq422016 qtio422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	StreamMetadataResponse(qw422016, result, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}
+
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+func MetadataResponse(result []*metricsmetadata.Row, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	WriteMetadataResponse(qb422016, result, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	return qs422016
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}

app/vmselect/prometheus/prometheus.go

@@ -639,6 +639,37 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	return nil
 }
 
+// MetadataHandler processes /api/v1/metadata request.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+func MetadataHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+
+	limit, err := httputil.GetInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	metricName := r.FormValue("metric")
+
+	metadata, err := netstorage.GetMetricsMetadata(qt, limit, metricName)
+	if err != nil {
+		return fmt.Errorf("cannot get metadata: %w", err)
+	}
+	qt.Done()
+	w.Header().Set("Content-Type", "application/json")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetadataResponse(bw, metadata, qt)
+	if err := bw.Flush(); err != nil {
+		return fmt.Errorf("cannot send metadata response to remote client: %w", err)
+	}
+
+	return nil
+}
+
 var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels"}`)
 
 // SeriesCountHandler processes /api/v1/series/count request.

app/vmselect/promql/aggr_incremental_test.go

@@ -103,15 +103,13 @@ func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssEx
 	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
-	wg.Add(workersCount)
-	for i := 0; i < workersCount; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workersCount {
+		wg.Go(func() {
 			for ts := range tsCh {
 				runtime.Gosched() // allow other goroutines performing the work
-				iafc.updateTimeseries(ts, workerID)
+				iafc.updateTimeseries(ts, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	for _, ts := range tssSrc {
 		tsCh <- ts

app/vmselect/promql/eval.go

@@ -477,22 +477,18 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		var tssFirst []*timeseries
 		var errFirst error
 		qtFirst := qt.NewChild("expr1")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
 			qtFirst.Done()
-		}()
+		})
 
 		var tssSecond []*timeseries
 		var errSecond error
 		qtSecond := qt.NewChild("expr2")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
 			qtSecond.Done()
-		}()
+		})
 
 		wg.Wait()
 		if errFirst != nil {
@@ -710,17 +706,13 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
-		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
-		go func(e metricsql.Expr, i int) {
-			defer func() {
-				qtChild.Done()
-				wg.Done()
-			}()
+		wg.Go(func() {
+			defer qtChild.Done()
 			rv, err := evalExpr(qtChild, ec, e)
 			rvs[i] = rv
 			errs[i] = err
-		}(e, i)
+		})
 	}
 	wg.Wait()
 	for _, err := range errs {
@@ -785,7 +777,8 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
-	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	if re.At == nil {
 		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
@@ -835,7 +828,8 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 }
 
 func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -1017,16 +1011,14 @@ func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, time
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workers {
+		wg.Go(func() {
 			var tmpValues []float64
 			var tmpTimestamps []int64
 			for ts := range workChs[workerID] {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 }
@@ -1058,7 +1050,8 @@ func removeNanValues(dstValues []float64, dstTimestamps []int64, values []float6
 
 // evalInstantRollup evaluates instant rollup where ec.Start == ec.End.
 func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64,
+) ([]*timeseries, error) {
 	if ec.Start != ec.End {
 		logger.Panicf("BUG: evalInstantRollup cannot be called on non-empty time range; got %s", ec.timeRangeString())
 	}
@@ -1083,10 +1076,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		rollupResultCacheV.DeleteInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 	}
 	getCachedSeries := func(qt *querytracer.Tracer) ([]*timeseries, int64, error) {
+		rollupResultCacheV.rollupResultCacheRequests.Inc()
 	again:
 		offset := int64(0)
 		tssCached := rollupResultCacheV.GetInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 		if len(tssCached) == 0 {
+			rollupResultCacheV.rollupResultCacheMisses.Inc()
 			// Cache miss. Re-populate the missing data.
 			start := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds()
 			offset = timestamp - start
@@ -1129,6 +1124,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			deleteCachedSeries(qt)
 			goto again
 		}
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 		ec.QueryStats.addSeriesFetched(len(tssCached))
 		return tssCached, offset, nil
 	}
@@ -1169,60 +1165,6 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			},
 		}
 		return evalExpr(qt, ec, be)
-	case "rate":
-		if iafc != nil {
-			if !strings.EqualFold(iafc.ae.Name, "sum") {
-				qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
-				return evalAt(qt, timestamp, window)
-			}
-			qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
-			afe := expr.(*metricsql.AggrFuncExpr)
-			fe := afe.Args[0].(*metricsql.FuncExpr)
-			feIncrease := *fe
-			feIncrease.Name = "increase"
-			// copy RollupExpr to drop possible offset,
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-			newArg.Offset = nil
-			feIncrease.Args = []metricsql.Expr{newArg}
-			d := newArg.Window.Duration(ec.Step)
-			if d == 0 {
-				d = ec.Step
-			}
-			afeIncrease := *afe
-			afeIncrease.Args = []metricsql.Expr{&feIncrease}
-			be := &metricsql.BinaryOpExpr{
-				Op:              "/",
-				KeepMetricNames: true,
-				Left:            &afeIncrease,
-				Right: &metricsql.NumberExpr{
-					N: float64(d) / 1000,
-				},
-			}
-			return evalExpr(qt, ec, be)
-		}
-		qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
-		fe := expr.(*metricsql.FuncExpr)
-		feIncrease := *fe
-		feIncrease.Name = "increase"
-		// copy RollupExpr to drop possible offset,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-		newArg.Offset = nil
-		feIncrease.Args = []metricsql.Expr{newArg}
-		d := newArg.Window.Duration(ec.Step)
-		if d == 0 {
-			d = ec.Step
-		}
-		be := &metricsql.BinaryOpExpr{
-			Op:              "/",
-			KeepMetricNames: fe.KeepMetricNames,
-			Left:            &feIncrease,
-			Right: &metricsql.NumberExpr{
-				N: float64(d) / 1000,
-			},
-		}
-		return evalExpr(qt, ec, be)
 	case "max_over_time":
 		if iafc != nil {
 			if !strings.EqualFold(iafc.ae.Name, "max") {
@@ -1591,16 +1533,11 @@ func assertInstantValues(tss []*timeseries) {
 	}
 }
 
-var (
-	rollupResultCacheFullHits    = metrics.NewCounter(`vm_rollup_result_cache_full_hits_total`)
-	rollupResultCachePartialHits = metrics.NewCounter(`vm_rollup_result_cache_partial_hits_total`)
-	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
-
-	memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
-)
+var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
 
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
+) ([]*timeseries, error) {
 	window, err := windowExpr.NonNegativeDuration(ec.Step)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
@@ -1636,19 +1573,20 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 
 	// Search for cached results.
+	rollupResultCacheV.rollupResultCacheRequests.Inc()
 	tssCached, start := rollupResultCacheV.GetSeries(qt, ec, expr, window)
 	ec.QueryStats.addSeriesFetched(len(tssCached))
 	if start > ec.End {
 		qt.Printf("the result is fully cached")
-		rollupResultCacheFullHits.Inc()
+		rollupResultCacheV.rollupResultCacheFullHits.Inc()
 		return tssCached, nil
 	}
 	if start > ec.Start {
 		qt.Printf("partial cache hit")
-		rollupResultCachePartialHits.Inc()
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 	} else {
 		qt.Printf("cache miss")
-		rollupResultCacheMiss.Inc()
+		rollupResultCacheV.rollupResultCacheMisses.Inc()
 	}
 
 	// Fetch missing results, which aren't cached yet.
@@ -1684,7 +1622,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 //
 // pointsPerSeries is used only for estimating the needed memory for query processing
 func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64,
+) ([]*timeseries, error) {
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s: timeRange=%s, step=%d, window=%d", expr.AppendString(nil), ec.timeRangeString(), ec.Step, window)
 		defer qt.Done()
@@ -1807,7 +1746,8 @@ func maxSilenceInterval() int64 {
 
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
@@ -1846,7 +1786,8 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 }
 
 func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
@@ -1886,7 +1827,8 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 }
 
 func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
-	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 {
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64,
+) uint64 {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)

app/vmselect/promql/rollup.go

@@ -534,7 +534,10 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the rc.LookbackDelta.
+	// Is populated if the preceding sample falls within the rc.LookbackDelta range, or if rc.LookbackDelta is not set.
+	//
+	// It provides an additional check and value for rollup functions such as increase(), changes(),
+	// when the prevValue is NaN due to a gap or a small lookback window.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -713,7 +716,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	// Use step as the scrape interval for instant queries (when start == end).
+	// Set maxPrevInterval for subsequent rfa.prevValue calculations in rollupFunc:
+	// For instant queries, use rc.Step directly as maxPrevInterval.
+	// For range queries, rc.Step is typically too small to serve as the lookback window between two rollup points.
+	// Instead, estimate the scrape interval from raw sample timestamps (using the 0.6 quantile of the last 20 intervals)
+	// and slightly inflate the scrape interval to set maxPrevInterval, allowing for some tolerance to jitter.
 	maxPrevInterval := rc.Step
 	if rc.Start < rc.End {
 		scrapeInterval := getScrapeInterval(timestamps, rc.Step)
@@ -729,22 +736,21 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 	}
 	window := rc.Window
+	// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
+	// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+	// when the window doesn't cover at least two raw samples (this is what most users expect).
+	//
+	// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+	// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
-			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
-			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
-			// when the window doesn't cover at least two raw samples (this is what most users expect).
-			//
-			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
-			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
-			//
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 			window = maxPrevInterval
 		}
+		// Artificial window cannot exceed explicit rc.LookbackDelta, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
-			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
-			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
@@ -869,17 +875,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
 		return defaultInterval
 	}
 
-	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
-	tsPrev := timestamps[0]
-	timestamps = timestamps[1:]
+	// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
+	tsPrev := timestamps[len(timestamps)-1]
+	timestamps = timestamps[:len(timestamps)-1]
 	if len(timestamps) > 20 {
-		timestamps = timestamps[:20]
+		timestamps = timestamps[len(timestamps)-20:]
 	}
 	a := getFloat64s()
 	intervals := a.A[:0]
-	for _, ts := range timestamps {
-		intervals = append(intervals, float64(ts-tsPrev))
-		tsPrev = ts
+	for i := len(timestamps) - 1; i >= 0; i-- {
+		intervals = append(intervals, float64(tsPrev-timestamps[i]))
+		tsPrev = timestamps[i]
 	}
 	scrapeInterval := int64(quantile(0.6, intervals))
 	a.A = intervals
@@ -2107,9 +2113,15 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		prevValue = values[0]
-		values = values[1:]
-		n++
+		// Assume that the value didn't change during the current gap
+		// if realPrevValue exists.
+		if !math.IsNaN(rfa.realPrevValue) {
+			prevValue = rfa.realPrevValue
+		} else {
+			n++
+			prevValue = values[0]
+			values = values[1:]
+		}
 	}
 	for _, v := range values {
 		if v != prevValue {

app/vmselect/promql/rollup_result_cache.go

@@ -83,9 +83,11 @@ func checkRollupResultCacheReset() {
 
 const checkRollupResultCacheResetInterval = 5 * time.Second
 
-var needRollupResultCacheReset atomic.Bool
-var checkRollupResultCacheResetOnce sync.Once
-var rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+var (
+	needRollupResultCacheReset       atomic.Bool
+	checkRollupResultCacheResetOnce  sync.Once
+	rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+)
 
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024 * 1024), // This is a cache for testing.
@@ -132,7 +134,7 @@ func InitRollupResultCache(cachePath string) {
 		c = workingsetcache.New(cacheSize)
 		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 	}
-	if *disableCache {
+	if *disableCache && len(rollupResultCachePath) > 0 && !*resetRollupResultCacheOnStartup {
 		c.Reset()
 	}
 
@@ -178,6 +180,12 @@ func InitRollupResultCache(cachePath string) {
 
 	rollupResultCacheV = &rollupResultCache{
 		c: c,
+
+		rollupResultCacheRequests:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_requests_total`),
+		rollupResultCacheFullHits:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_full_hits_total`),
+		rollupResultCachePartialHits: metrics.GetOrCreateCounter(`vm_rollup_result_cache_partial_hits_total`),
+		rollupResultCacheMisses:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_miss_total`),
+		rollupResultCacheResets:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_resets_total`),
 	}
 }
 
@@ -193,13 +201,18 @@ func StopRollupResultCache() {
 
 type rollupResultCache struct {
 	c *workingsetcache.Cache
-}
 
-var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="promql/rollupResult"}`)
+	rollupResultCacheRequests    *metrics.Counter
+	rollupResultCacheFullHits    *metrics.Counter
+	rollupResultCachePartialHits *metrics.Counter
+	rollupResultCacheMisses      *metrics.Counter
+
+	rollupResultCacheResets *metrics.Counter
+}
 
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
-	rollupResultCacheResets.Inc()
+	rollupResultCacheV.rollupResultCacheResets.Inc()
 	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }

app/vmselect/promql/rollup_test.go

@@ -232,6 +232,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	}
 	var rfa rollupFuncArg
 	rfa.prevValue = nan
+	rfa.realPrevValue = nan
 	rfa.prevTimestamp = 0
 	rfa.values = append(rfa.values, testValues...)
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
@@ -1654,7 +1655,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1674,7 +1675,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1794,7 +1795,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1814,7 +1815,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1888,3 +1889,126 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupChangesWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then changes will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then changes will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 1, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280
+	//
+	// When there are gaps between samples that exceed maxPrevInterval,
+	// either due to changes in the scrape interval or missing scrapes.
+	// For example, if the scrape interval was initially 30s and later changed to 10s,
+	// the auto-calculated scrape interval is 10s, with maxPrevInterval inflated to 15s.
+	//
+	// At t=30s:
+	// prevValue is NaN, as the last sample at t=0s is considered stale for t=30s given the maxPrevInterval.
+	// realPrevValue is 1, taken from t=0s, since LookbackDelta=0 ignores staleness.
+	// the result should be `changes(1, 1) -> 0` instead of `changes(1, NaN)`.
+	// At t=100s:
+	// preValue is also NaN, as the last sample at t=70s is considered stale for t=100s.
+	// realPrevValue is 1, taken from t=70s,
+	// result should be `changes(2, 1) -> 1`.
+	timestamps = []int64{0, 30000, 40000, 50000, 60000, 70000, 100000}
+	values = []float64{1, 1, 1, 1, 1, 1, 2}
+	t.Run("issue-10280", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                100e3,
+			Step:               10e3,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, _ := rc.Do(nil, values, timestamps)
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3, 80e3, 90e3, 100e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/promql/transform_test.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math/rand"
 	"reflect"
 	"strconv"
 	"strings"
@@ -280,6 +281,87 @@ func timeseriesToPromMetrics(tss []*timeseries) string {
 	return strings.Join(a, "\n")
 }
 
+func TestTransformFuncSort(t *testing.T) {
+	f := func(isDesc bool, metrics, expectedMetrics string) {
+		t.Helper()
+
+		tss := promMetricsToTimeseries(metrics)
+
+		// Input tss order is not stable in VictoriaMetrics
+		// Shuffle tss to reflect that
+		// Commenting out the shuffle to make the test stable
+		rand.Shuffle(len(tss), func(i, j int) {
+			tss[i], tss[j] = tss[j], tss[i]
+		})
+
+		sortFunc := newTransformFuncSort(isDesc)
+		sorted, err := sortFunc(&transformFuncArg{
+			args: [][]*timeseries{tss},
+		})
+		if err != nil {
+			t.Fatalf("sort failed: %s", err)
+		}
+
+		result := timeseriesToPromMetrics(sorted)
+		if result != expectedMetrics {
+			t.Fatalf("unexpected sort result:\ngot\n%s\nwant\n%s", result, expectedMetrics)
+		}
+	}
+
+	// Test asc sort with different values
+	f(
+		false,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+		`foo{label="c"} 1 123
+foo{label="b"} 2 123
+foo{label="a"} 3 123`,
+	)
+
+	// Test desc sort with different values
+	f(
+		true,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+	)
+
+	// Test asc sort with mixed values
+	f(
+		false,
+		`foo{label="a"} 1 123
+	foo{label="b"} 1 123
+	foo{label="c"} 2 123
+	foo{label="d"} 2 123
+	foo{label="e"} 3 123
+	`,
+		`foo{label="a"} 1 123
+foo{label="b"} 1 123
+foo{label="c"} 2 123
+foo{label="d"} 2 123
+foo{label="e"} 3 123`,
+	)
+
+	// Test desc sort with mixed values
+	f(
+		true,
+		`foo{label="a"} 1 123
+	foo{label="b"} 1 123
+	foo{label="c"} 2 123
+	foo{label="d"} 2 123
+	foo{label="e"} 3 123`,
+		`foo{label="e"} 3 123
+foo{label="c"} 2 123
+foo{label="d"} 2 123
+foo{label="a"} 1 123
+foo{label="b"} 1 123`,
+	)
+}
+
 func TestGetNumPrefix(t *testing.T) {
 	f := func(s, prefixExpected string) {
 		t.Helper()

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-zpalCSif.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-DY9kCvzk.js">
+  <script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-CBxdwuZH.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -22,13 +22,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 var (
-	retentionPeriod   = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
+	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -90,6 +92,9 @@ var (
 		"In most cases, this value should not be changed. The maximum allowed value is 23h.")
 
 	logNewSeriesAuthKey = flagutil.NewPassword("logNewSeriesAuthKey", "authKey, which must be passed in query string to /internal/log_new_series. It overrides -httpAuth.*")
+
+	metadataStorageSize = flagutil.NewBytes("storage.maxMetadataStorageSize", 0, "Overrides max size for metrics metadata entries in-memory storage. "+
+		"If set to 0 or a negative value, defaults to 1% of allowed memory.")
 )
 
 // CheckTimeRange returns true if the given tr is denied for querying.
@@ -114,12 +119,13 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	}
 
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
-	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
+	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
 	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
 	storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
+	storage.SetMetadataStorageSize(metadataStorageSize.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
 	mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
@@ -194,6 +200,19 @@ func AddRows(mrs []storage.MetricRow) error {
 	return nil
 }
 
+// AddMetadataRows adds mrs to the storage.
+//
+// The caller should limit the number of concurrent calls to AddMetadataRows() in order to limit memory usage.
+func AddMetadataRows(mms []metricsmetadata.Row) error {
+	if Storage.IsReadOnly() {
+		return errReadOnly
+	}
+	WG.Add(1)
+	Storage.AddMetadataRows(mms)
+	WG.Done()
+	return nil
+}
+
 var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
 
 // RegisterMetricNames registers all the metrics from mrs in the storage.
@@ -370,11 +389,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotPath := Storage.MustCreateSnapshot()
+		snapshotName := Storage.MustCreateSnapshot()
+
+		// Verify whether the client already closed the connection.
+		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
+		if err := r.Context().Err(); err != nil {
+			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
+			if err := deleteSnapshot(snapshotName); err != nil {
+				logger.Infof("cannot delete just created snapshot: %s", err)
+				return true
+			}
+			return true
+		}
+
 		if prometheusCompatibleResponse {
-			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotName))
 		} else {
-			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotName))
 		}
 		return true
 	case "/list":
@@ -394,23 +425,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-
-		snapshots := Storage.MustListSnapshots()
-		for _, snName := range snapshots {
-			if snName == snapshotName {
-				if err := Storage.DeleteSnapshot(snName); err != nil {
-					err = fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-					jsonResponseError(w, err)
-					snapshotsDeleteErrorsTotal.Inc()
-					return true
-				}
-				fmt.Fprintf(w, `{"status":"ok"}`)
-				return true
-			}
+		if err := deleteSnapshot(snapshotName); err != nil {
+			jsonResponseError(w, err)
+			snapshotsDeleteErrorsTotal.Inc()
+			return true
 		}
-
-		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
-		jsonResponseError(w, err)
+		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
@@ -431,15 +451,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+func deleteSnapshot(snapshotName string) error {
+	snapshots := Storage.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := Storage.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}
+
 func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverCh = make(chan struct{})
 	if snapshotsMaxAge.Duration() <= 0 {
 		return
 	}
 	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Add(1)
-	go func() {
-		defer staleSnapshotsRemoverWG.Done()
+	staleSnapshotsRemoverWG.Go(func() {
 		d := timeutil.AddJitterToDuration(time.Second * 11)
 		t := time.NewTicker(d)
 		defer t.Stop()
@@ -451,7 +482,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 			}
 			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
 		}
-	}()
+	})
 }
 
 func stopStaleSnapshotsRemover() {
@@ -482,7 +513,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
-	idbm := &m.IndexDBMetrics
+	idbm := &m.TableMetrics.IndexDBMetrics
 
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetFreeSpace(*DataPath))
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *DataPath), uint64(minFreeDiskSpaceBytes.N))
@@ -610,75 +641,82 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteCounterUint64(w, `vm_missing_metric_names_for_metric_id_total`, idbm.MissingMetricNamesForMetricID)
 
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_syncs_total`, m.DateMetricIDCacheSyncsCount)
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_resets_total`, m.DateMetricIDCacheResetsCount)
-
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/tsid"}`, m.TSIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricIDs"}`, m.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricName"}`, m.MetricNameCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/date_metricID"}`, m.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/metricID"}`, idbm.MetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/date_metricID"}`, m.DateMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSizeBytes()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheMaxSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheMaxSizeBytes()))
 
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricIDs"}`, m.MetricIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricName"}`, m.MetricNameCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricIDs"}`, m.MetricIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricName"}`, m.MetricNameCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 
-	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/tsid"}`, m.TSIDCacheCollisions)
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/metricName"}`, m.MetricNameCacheCollisions)
 
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/metricID"}`, idbm.MetricIDCacheSyncsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSyncsCount)
+
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/metricID"}`, idbm.MetricIDCacheRotationsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheRotationsCount)
+
+	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+
 	metrics.WriteGaugeUint64(w, `vm_next_retention_seconds`, m.NextRetentionSeconds)
 
 	if *trackMetricNamesStats {
@@ -689,6 +727,11 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled`, tm.ScheduledDownsamplingPartitions)
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled_size_bytes`, tm.ScheduledDownsamplingPartitionsSize)
+
+	metrics.WriteGaugeUint64(w, `vm_metrics_metadata_storage_items`, m.MetadataStorageItemsCurrent)
+	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_size_bytes`, m.MetadataStorageCurrentSizeBytes)
+	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_max_size_bytes`, m.MetadataStorageMaxSizeBytes)
+
 }
 
 func jsonResponseError(w http.ResponseWriter, err error) {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.4 AS build-web-stage
+FROM golang:1.25.6 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/Makefile

@@ -1,26 +1,26 @@
 # All these commands must run from repository root.
-
 copy-metricsql-docs:
 	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
 vmui-package-base-image:
 	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
 
-vmui-build: copy-metricsql-docs vmui-package-base-image
+vmui-run-npm-command: vmui-package-base-image
 	docker run --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
 		-w /build/packages/vmui \
 		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build"
+		vmui-builder-image -c "[ \"$$VMUI_SKIP_INSTALL\" = \"true\" ] || npm ci; $(NPM_COMMAND)"
 
-vmui-anomaly-build: vmui-package-base-image
-	docker run --rm \
-		--user $(shell id -u):$(shell id -g) \
-		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
-		-w /build/packages/vmui \
-		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build:anomaly"
+vmui-install:
+	NPM_COMMAND="true" $(MAKE) vmui-run-npm-command
+
+vmui-package-base-image:
+	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
+
+vmui-build: copy-metricsql-docs
+	NPM_COMMAND="npm run build" $(MAKE) vmui-run-npm-command
 
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
@@ -38,11 +38,11 @@ vmui-update: vmui-build
 vmui-install-dependencies:
 	cd app/vmui/packages/vmui && npm ci
 
-vmui-lint: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run lint
+vmui-lint:
+	NPM_COMMAND="npm run lint" $(MAKE) vmui-run-npm-command
 
-vmui-typecheck: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run typecheck
+vmui-typecheck:
+	NPM_COMMAND="npm run typecheck" $(MAKE) vmui-run-npm-command
 
-vmui-test: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run test
+vmui-test:
+	NPM_COMMAND="npm run test" $(MAKE) vmui-run-npm-command

app/vmui/packages/vmui/.env.vmanomaly

@@ -1 +0,0 @@
-VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -1,23 +0,0 @@
-import { readFile } from "fs/promises";
-import { IndexHtmlTransform } from "vite";
-
-/**
- * Vite plugin to dynamically load index.html based on the current mode.
- * If a specific mode-based index file (e.g., index.vmanomaly.html) exists, it is used.
- * Otherwise, the default index.html is loaded.
- */
-export default function dynamicIndexHtmlPlugin({ mode }) {
-  return {
-    name: "vm-dynamic-index-html",
-    transformIndexHtml: {
-      order: "pre",
-      handler: async () => {
-        try {
-          return await readFile(`./index.${mode}.html`, "utf8");
-        } catch (error) {
-          return await readFile("./index.html", "utf8");
-        }
-      }
-    } as IndexHtmlTransform
-  };
-}

app/vmui/packages/vmui/eslint.config.js

@@ -46,7 +46,7 @@ export default [...compat.extends(
   settings: {
     react: {
       pragma: "React",
-      version: "detect",
+      version: "19.0",
     },
 
     linkComponents: ["Hyperlink", {
@@ -69,10 +69,11 @@ export default [...compat.extends(
       "varsIgnorePattern": "^_",
       "ignoreRestSiblings": true
     }],
-    
+
     "unused-imports/no-unused-imports": "error",
 
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
+    "object-curly-spacing": [2, "always"],
 
     "react/jsx-max-props-per-line": [1, {
       maximum: 1,
@@ -81,13 +82,23 @@ export default [...compat.extends(
     "react/jsx-first-prop-new-line": [1, "multiline"],
 
     // Disable core indent rule due to recursion issues in ESLint 9; use JSX-specific rules instead
-    indent: "off",
+    indent: ["error", 2, {
+      SwitchCase: 1,
+      ignoredNodes: [
+        "JSXElement",
+        "JSXElement *",
+        "JSXFragment",
+        "JSXFragment *",
+      ],
+    }],
     "react/jsx-indent": ["error", 2],
     "react/jsx-indent-props": ["error", 2],
 
+    "linebreak-style": ["error", "unix"],
+    quotes: ["error", "double"],
+    semi: ["error", "always"],
     // Formatting rules moved out of ESLint core; omit here to avoid deprecation noise
     "react/prop-types": 0,
     "react/react-in-jsx-scope": "off",
-
   },
 }];

app/vmui/packages/vmui/index.vmanomaly.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg" />
-  <link rel="apple-touch-icon" href="/favicon.svg" />
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaMetrics Anomaly Detection</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-  <meta name="twitter:image" content="/preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta property="og:url" content="https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta property="og:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-<script type="module" src="/src/index.tsx"></script>
-</body>
-</html>

app/vmui/packages/vmui/package.json

@@ -7,10 +7,8 @@
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
     "start": "vite",
-    "start:playground": "cross-env PLAYGROUND=METRICS npm run start",
-    "start:anomaly": "vite --mode vmanomaly",
+    "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
@@ -18,47 +16,48 @@
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dev": "vitest"
+    "test:dev": "vitest",
+    "precommit": "npm run lint:local && npm run typecheck && npm run test"
   },
   "dependencies": {
     "classnames": "^2.5.1",
-    "dayjs": "^1.11.13",
+    "dayjs": "^1.11.19",
     "lodash.debounce": "^4.0.8",
-    "marked": "^16.0.0",
-    "preact": "^10.26.9",
-    "qs": "^6.14.0",
+    "marked": "^17.0.1",
+    "preact": "^10.28.2",
+    "qs": "^6.14.1",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.6.3",
+    "react-router-dom": "^7.12.0",
     "uplot": "^1.6.32",
-    "vite": "^7.1.11",
-    "web-vitals": "^5.0.3"
+    "vite": "^7.3.1",
+    "web-vitals": "^5.1.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.30.1",
+    "@eslint/eslintrc": "^3.3.3",
+    "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.2",
-    "@testing-library/jest-dom": "^6.6.3",
+    "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^24.0.12",
+    "@types/node": "^25.0.8",
     "@types/qs": "^6.14.0",
-    "@types/react": "^19.1.8",
+    "@types/react": "^19.2.8",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.36.0",
-    "@typescript-eslint/parser": "^8.36.0",
-    "cross-env": "^7.0.3",
-    "eslint": "^9.30.1",
+    "@typescript-eslint/eslint-plugin": "^8.53.0",
+    "@typescript-eslint/parser": "^8.53.0",
+    "cross-env": "^10.1.0",
+    "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
-    "eslint-plugin-unused-imports": "^4.1.4",
-    "globals": "^16.3.0",
+    "eslint-plugin-unused-imports": "^4.3.0",
+    "globals": "^17.0.0",
     "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^26.1.0",
+    "jsdom": "^27.4.0",
     "postcss": "^8.5.6",
-    "rollup-plugin-visualizer": "^6.0.3",
-    "sass-embedded": "^1.89.2",
-    "typescript": "^5.8.3",
-    "vitest": "^3.2.4"
+    "rollup-plugin-visualizer": "^6.0.5",
+    "sass-embedded": "^1.97.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.17"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -1,41 +0,0 @@
-import { FC, useState } from "preact/compat";
-import { HashRouter, Route, Routes } from "react-router-dom";
-import AppContextProvider from "./contexts/AppContextProvider";
-import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
-import AnomalyLayout from "./layouts/AnomalyLayout/AnomalyLayout";
-import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
-import router from "./router";
-import CustomPanel from "./pages/CustomPanel";
-
-const AppAnomaly: FC = () => {
-  const [loadedTheme, setLoadedTheme] = useState(false);
-
-  return <>
-    <HashRouter>
-      <AppContextProvider>
-        <>
-          <ThemeProvider onLoaded={setLoadedTheme}/>
-          {loadedTheme && (
-            <Routes>
-              <Route
-                path={"/"}
-                element={<AnomalyLayout/>}
-              >
-                <Route
-                  path={"/"}
-                  element={<ExploreAnomaly/>}
-                />
-                <Route
-                  path={router.query}
-                  element={<CustomPanel/>}
-                />
-              </Route>
-            </Routes>
-          )}
-        </>
-      </AppContextProvider>
-    </HashRouter>
-  </>;
-};
-
-export default AppAnomaly;

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -20,6 +20,7 @@ export interface ChartTooltipProps {
   info?: ReactNode;
   marker?: string;
   show?: boolean;
+  duplicateCount?: number;
   onClose?: (id: string) => void;
 }
 
@@ -35,6 +36,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   statsFormatted,
   isSticky,
   marker,
+  duplicateCount = 0,
   onClose
 }) => {
   const tooltipRef = useRef<HTMLDivElement>(null);
@@ -156,6 +158,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
         <p className="vm-chart-tooltip-data__value">
           <b>{value}</b>{unit}
         </p>
+        {duplicateCount > 1 && <p>(overlapping points: {duplicateCount})</p>}
       </div>
       {statsFormatted && (
         <table className="vm-chart-tooltip-stats">

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -14,12 +14,11 @@ export type QueryGroup = {
 interface LegendProps {
   labels: LegendItemType[];
   query: string[];
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
 
-const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPanel, onChange }) => {
+const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange }) => {
   const { groupByLabel } = useLegendGroup();
   const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
@@ -33,7 +32,6 @@ const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPan
             key={group}
             labels={items}
             group={group}
-            isAnomalyView={isAnomalyView}
             onChange={onChange}
           />
         ))}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -8,11 +8,11 @@ import { useHideDuplicateFields } from "./hooks/useHideDuplicateFields";
 import Accordion from "../../../Main/Accordion/Accordion";
 import { useLegendGroup } from "./hooks/useLegendGroup";
 import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
@@ -21,7 +21,7 @@ interface LegendGroupProps extends LegendProps {
   group: string | number;
 }
 
-const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onChange }) => {
+const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
   const { isTableView } = useLegendView();
   const { groupByLabel } = useLegendGroup();
   const copyToClipboard = useCopyToClipboard();
@@ -38,17 +38,26 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
 
   const Content = isTableView ? LegendTable : LegendLines;
 
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false";
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT;
+
+  const expandedWarning = (
+    <span className="vm-legend-group-header__warning">
+      Legend collapsed by default ({sortedLabels.length} series) — click to expand.
+    </span>
+  );
+
   return (
     <div
       className="vm-legend-group"
       key={group}
     >
       <Accordion
-        defaultExpanded={sortedLabels.length < DEFAULT_MAX_SERIES.chart}
+        defaultExpanded={defaultExpanded}
         title={(
           <div className="vm-legend-group-header">
             <div className="vm-legend-group-header-title">
-              Group by{groupByLabel ? "" : " query"}: <b>{group}</b>
+              Group by{groupByLabel ? "" : " query"}: <b>{group}</b> {!defaultExpanded && expandedWarning}
             </div>
             {!!duplicateFields.length && (
               <div className="vm-legend-group-header-labels">
@@ -71,7 +80,6 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
       >
         <Content
           labels={sortedLabels}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -13,11 +13,10 @@ import { getLabelAlias } from "../../../../../utils/metric";
 interface LegendItemProps {
   legend: LegendItemType;
   onChange?: (item: LegendItemType, metaKey: boolean) => void;
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, isAnomalyView }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields }) => {
   const copyToClipboard = useCopyToClipboard();
   const { hideStats } = useShowStats();
 
@@ -52,12 +51,10 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, is
       })}
       onClick={createHandlerClick(legend)}
     >
-      {!isAnomalyView && (
-        <div
-          className="vm-legend-item__marker"
-          style={{ backgroundColor: legend.color }}
-        />
-      )}
+      <div
+        className="vm-legend-item__marker"
+        style={{ backgroundColor: legend.color }}
+      />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.hasAlias && legend.label}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendLines.tsx

@@ -2,7 +2,7 @@ import { FC } from "preact/compat";
 import LegendItem from "../LegendItem/LegendItem";
 import { LegendProps } from "../LegendGroup";
 
-const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields, onChange }) => {
+const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) => {
 
   return (
     <div className="vm-legend-item-container">
@@ -10,7 +10,6 @@ const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields,
         <LegendItem
           key={legendItem.label}
           legend={legendItem}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/style.scss

@@ -32,6 +32,14 @@
         }
       }
 
+      &__warning {
+        flex-grow: 1;
+        text-align: right;
+        padding-right: calc($padding-large * 2);
+        font-size: $font-size-small;
+        color: $color-warning;
+      }
+
       &-labels {
         display: flex;
         flex-wrap: wrap;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/LegendAnomaly.tsx

@@ -1,82 +0,0 @@
-import { FC, useMemo } from "preact/compat";
-import { ForecastType, SeriesItem } from "../../../../types";
-import { anomalyColors } from "../../../../utils/color";
-import "./style.scss";
-
-type Props = {
-  series: SeriesItem[];
-};
-
-const titles: Partial<Record<ForecastType, string>> = {
-  [ForecastType.yhat]: "yhat",
-  [ForecastType.yhatLower]: "yhat_upper - yhat_lower",
-  [ForecastType.yhatUpper]: "yhat_upper - yhat_lower",
-  [ForecastType.anomaly]: "anomalies",
-  [ForecastType.training]: "training data",
-  [ForecastType.actual]: "y"
-};
-
-const LegendAnomaly: FC<Props> = ({ series }) => {
-
-  const uniqSeriesStyles = useMemo(() => {
-    const uniqSeries = series.reduce((accumulator, currentSeries) => {
-      const hasForecast = Object.prototype.hasOwnProperty.call(currentSeries, "forecast");
-      const isNotUpper = currentSeries.forecast !== ForecastType.yhatUpper;
-      const isUniqForecast = !accumulator.find(s => s.forecast === currentSeries.forecast);
-      if (hasForecast && isUniqForecast && isNotUpper) {
-        accumulator.push(currentSeries);
-      }
-      return accumulator;
-    }, [] as SeriesItem[]);
-
-    const trainingSeries = {
-      ...uniqSeries[0],
-      forecast: ForecastType.training,
-      color: anomalyColors[ForecastType.training],
-    };
-    uniqSeries.splice(1, 0, trainingSeries);
-
-    return uniqSeries.map(s => ({
-      ...s,
-      color: typeof s.stroke === "string" ? s.stroke : anomalyColors[s.forecast || ForecastType.actual],
-    }));
-  }, [series]);
-
-  return <>
-    <div className="vm-legend-anomaly">
-      {/* TODO: remove .filter() after the correct training data has been added */}
-      {uniqSeriesStyles.filter(f => f.forecast !== ForecastType.training).map((s, i) => (
-        <div
-          key={`${i}_${s.forecast}`}
-          className="vm-legend-anomaly-item"
-        >
-          <svg>
-            {s.forecast === ForecastType.anomaly ? (
-              <circle
-                cx="15"
-                cy="7"
-                r="4"
-                fill={s.color}
-                stroke={s.color}
-                strokeWidth="1.4"
-              />
-            ) : (
-              <line
-                x1="0"
-                y1="7"
-                x2="30"
-                y2="7"
-                stroke={s.color}
-                strokeWidth={s.width || 1}
-                strokeDasharray={s.dash?.join(",")}
-              />
-            )}
-          </svg>
-          <div className="vm-legend-anomaly-item__title">{titles[s.forecast || ForecastType.actual]}</div>
-        </div>
-      ))}
-    </div>
-  </>;
-};
-
-export default LegendAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/style.scss

@@ -1,23 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-legend-anomaly {
-  position: relative;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  flex-wrap: wrap;
-  gap: calc($padding-large * 2);
-  cursor: default;
-
-  &-item {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    gap: $padding-small;
-
-    svg {
-      width: 30px;
-      height: 14px;
-    }
-  }
-}

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -13,7 +13,6 @@ import {
   getRangeY,
   getScales,
   handleDestroy,
-  setBand,
   setSelect
 } from "../../../../utils/uplot";
 import { MetricResult } from "../../../../api/types";
@@ -40,7 +39,6 @@ export interface LineChartProps {
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
   layoutSize: ElementSize;
   height?: number;
-  isAnomalyView?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
 }
@@ -55,7 +53,6 @@ const LineChart: FC<LineChartProps> = ({
   setPeriod,
   layoutSize,
   height,
-  isAnomalyView,
   spanGaps = false,
   showAllPoints = false,
 }) => {
@@ -75,7 +72,7 @@ const LineChart: FC<LineChartProps> = ({
     seriesFocus,
     setCursor,
     resetTooltips
-  } = useLineTooltip({ u: uPlotInst, metrics, series, unit, isAnomalyView });
+  } = useLineTooltip({ u: uPlotInst, metrics, series, unit });
 
   const options: uPlotOptions = {
     ...getDefaultOptions({ width: layoutSize.width, height }),
@@ -111,7 +108,6 @@ const LineChart: FC<LineChartProps> = ({
     if (!uPlotInst) return;
     delSeries(uPlotInst);
     addSeries(uPlotInst, series, spanGaps, showAllPoints);
-    setBand(uPlotInst, series);
     uPlotInst.redraw();
   }, [series, spanGaps, showAllPoints]);