vmagent: maintenance mode support

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11191

The default `-search.maxConcurrentRequests` for cluster `vmselect` was capped at 16 via `min(2*cgroup.AvailableCPUs(), 16)`. This meant larger vmselect nodes kept the same default concurrency once they had more than 8 CPUs.

This changes the default to `2*cgroup.AvailableCPUs()`, matching the style already used by `vmstorage` and
`clusternative.maxConcurrentRequests`.

Capping it at 16 makes sense for vmsingle, since it handles both ingestion and querying, with ingestion being the higher priority. Select has no such limitation and should scale to all available resources.

---------

Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>

.github/workflows/build.yml

@@ -65,7 +65,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -73,11 +73,19 @@ jobs:
         id: go
         uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
+          cache: false
           go-version-file: 'go.mod'
+
+      - name: Cache Go build artifacts
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-build-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('go.sum', 'Makefile') }}
+          restore-keys: |
+            go-build-${{ matrix.os }}-${{ matrix.arch }}-
+
       - run: go version
 
       - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/changelog-linter.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           fetch-depth: 0  # we need full history for commit verification
           persist-credentials: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/codeql-analysis-go.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           path: __vm
           persist-credentials: false
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: 'vm-runner'
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -79,7 +79,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -106,7 +106,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/vmui.yml

@@ -26,13 +26,11 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

.golangci.yml

@@ -23,3 +23,4 @@ linters:
         text: 'SA(4003|1019|5011):'
     paths:
       - ^app/vmui/
+      - app/vmui/packages/vmui/node_modules/

Makefile

@@ -447,7 +447,7 @@ vet:
 	go vet ./app/...
 	go vet ./apptest/...
 
-check-all: fmt vet golangci-lint govulncheck
+check-all: fmt vet golangci-lint
 
 clean-checkers: remove-golangci-lint remove-govulncheck
 
@@ -471,8 +471,9 @@ test-full-386:
 
 apptest:
 	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+	go test ./apptest/... -skip="^Test(Cluster|Mixed|Legacy).*"
 
+# App tests for legacy indexDB
 apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
@@ -489,6 +490,22 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
+# App tests for mixed setups where vmsingle and vmcluster coexist.
+apptest-mixed: victoria-metrics-race
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VMSELECT_PATH=$${DIR}/vmselect-prod \
+	go test ./apptest/tests -run="^TestMixed.*"
+
 benchmark:
 	go test -run=NO_TESTS -bench=. ./lib/...
 	go test -run=NO_TESTS -bench=. ./app/...

README.md

@@ -2,7 +2,6 @@
 
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
-[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)

app/victoria-metrics/main.go

@@ -63,6 +63,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -89,7 +90,7 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
-	vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()
@@ -170,3 +171,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+	vmselect.InitSecretFlags()
+}

app/vmagent/influx/request_handler.go

@@ -51,7 +51,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vmagent/main.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"embed"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -84,7 +86,7 @@ var (
 	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 
-	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
+	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", true, "Enables multitenancy via HTTP headers. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
 )
 
@@ -115,7 +117,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remotewrite.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	opentelemetry.Init()
@@ -462,7 +464,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()
@@ -547,6 +551,33 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/remotewrite/maintenance":
+		if !remotewrite.CheckMaintenanceAuthKey(w, r) {
+			return true
+		}
+		remoteWriteMaintenanceRequests.Inc()
+		if v := r.FormValue("enable"); v != "" {
+			enable, err := strconv.ParseBool(v)
+			if err != nil {
+				httpserver.Errorf(w, r, "cannot parse `enable` query arg: %s", err)
+				return true
+			}
+			rwURL := r.FormValue("url")
+			if rwURL == "" {
+				httpserver.Errorf(w, r, "missing `url` query arg; it must match the `url` label of vmagent_remotewrite_* metrics for the target -remoteWrite.url, "+
+					"or be set to `*` to target all the configured -remoteWrite.url destinations")
+				return true
+			}
+			if matched := remotewrite.SetMaintenanceMode(rwURL, enable); matched == 0 {
+				httpserver.Errorf(w, r, "no -remoteWrite.url destinations match `url=%s`", rwURL)
+				return true
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"maintenance": remotewrite.GetMaintenanceMode(),
+		})
+		return true
 	case "/ready":
 		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
@@ -831,6 +862,8 @@ var (
 	remoteWriteStatusURLRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-url-relabel-config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+
+	remoteWriteMaintenanceRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite/maintenance"}`)
 )
 
 func usage() {
@@ -841,3 +874,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remotewrite.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmagent/opentelemetry/request_handler.go

@@ -63,6 +63,7 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range tss {
@@ -82,7 +83,19 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	var metadataTotal int
 	if prommetadata.IsEnabled() {
-		ctx.WriteRequest.Metadata = mms
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: mm.AccountID,
+				ProjectID: mm.ProjectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
 		metadataTotal = len(mms)
 	}
 

app/vmagent/prometheusimport/request_handler_test.go

@@ -52,6 +52,7 @@ func setUp() {
 
 func tearDown() {
 	protoparserutil.StopUnmarshalWorkers()
+	remotewrite.Stop()
 	srv.Close()
 	logger.ResetOutputForTest()
 	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()

app/vmagent/remotewrite/client.go

@@ -97,6 +97,8 @@ type client struct {
 	useVMProto          atomic.Bool
 	canDowngradeVMProto atomic.Bool
 
+	maintenanceMode atomic.Bool
+
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
@@ -151,17 +153,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,
@@ -316,6 +324,12 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 	var block []byte
 	ch := make(chan bool, 1)
 	for {
+		if c.maintenanceMode.Load() {
+			if !c.waitForMaintenanceModeOff() {
+				return
+			}
+		}
+
 		block, ok = readBlock(block[:0])
 		if !ok {
 			return
@@ -333,6 +347,14 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 			}
 			// Return unsent block to the queue.
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			select {
+			case <-c.stopCh:
+				// c must be stopped.
+			default:
+				// sendBlock returned false because maintenance mode is enabled, not because
+				// c is stopping. Keep the worker alive so it resumes once maintenance mode is disabled.
+				continue
+			}
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
@@ -357,6 +379,23 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 	}
 }
 
+// waitForMaintenanceModeOff blocks while maintenance mode is enabled for c.
+//
+// It returns false only if c.stopCh is closed while waiting.
+func (c *client) waitForMaintenanceModeOff() bool {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for c.maintenanceMode.Load() {
+		select {
+		case <-t.C:
+		case <-c.stopCh:
+			return false
+		}
+	}
+	return true
+}
+
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
 	req, err := c.newRequest(url, body)
 	if err != nil {
@@ -415,7 +454,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
-// The function returns false only if c.stopCh is closed.
+// The function returns false if c.stopCh is closed or if maintenance mode is enabled for c.
 // Otherwise, it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
@@ -423,6 +462,10 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 	retriesCount := 0
 
 again:
+	if c.maintenanceMode.Load() {
+		return false
+	}
+
 	startTime := time.Now()
 	resp, err := c.doRequest(c.remoteWriteURL, block)
 	c.requestDuration.UpdateDuration(startTime)
@@ -505,12 +548,7 @@ again:
 	// Handle response
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
-	if err != nil {
-		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
-	} else {
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %s", len(block), c.sanitizedURL, retriesCount, statusCode, body, bt.CurrentDelay())
-	}
+	logUnexpectedStatusCode(block, c.sanitizedURL, statusCode, retriesCount, bt.CurrentDelay(), retryAfterHeader > 0, body, err)
 	if !bt.Wait(c.stopCh) {
 		return false
 	}
@@ -546,6 +584,26 @@ func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
+var remoteWriteUnexpectedStatusLogger = logger.WithThrottler("remoteWriteUnexpectedStatus", 5*time.Second)
+
+func logUnexpectedStatusCode(block []byte, sanitizedURL string, statusCode, retriesCount int, retryDelay time.Duration, isExpectedBackoff bool, body []byte, bodyErr error) {
+	if bodyErr != nil {
+		remoteWriteUnexpectedStatusLogger.Errorf("cannot read response body from %q during retry #%d: %s", sanitizedURL, retriesCount, bodyErr)
+		return
+	}
+
+	msg := fmt.Sprintf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
+		"re-sending the block in %s", len(block), sanitizedURL, retriesCount, statusCode, body, retryDelay)
+
+	if isExpectedBackoff {
+		// The remote storage explicitly signaled backoff duration via the Retry-After header,
+		// so this isn't an anomaly worth an ERROR log.
+		remoteWriteUnexpectedStatusLogger.Warnf("%s", msg)
+		return
+	}
+
+	remoteWriteUnexpectedStatusLogger.Errorf("%s", msg)
+}
 
 // repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
 //

app/vmagent/remotewrite/obfuscate.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+type obfuscateLabelsCtx struct {
+	labels []prompb.Label
+
+	// buf holds allocations for cached results below
+	buf []byte
+
+	// cacheResults maps original label values to their SHA-256 hex digests,
+	// avoiding redundant hashing of repeated values within a single batch.
+	cacheResults map[string]string
+}
+
+func (olctx *obfuscateLabelsCtx) reset() {
+	promrelabel.CleanLabels(olctx.labels)
+	olctx.labels = olctx.labels[:0]
+	clear(olctx.cacheResults)
+	olctx.buf = olctx.buf[:0]
+}
+
+var obfuscateLabelsCtxPool = &sync.Pool{
+	New: func() any {
+		return &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+	},
+}
+
+func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
+	return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
+}
+
+func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
+	ctx.reset()
+	obfuscateLabelsCtxPool.Put(ctx)
+}
+
+func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
+	if len(obfuscateLabels) == 0 || len(tss) == 0 {
+		return tss
+	}
+	labels := olctx.labels
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		found := false
+		for _, labelName := range obfuscateLabels {
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
+			if tmp == nil {
+				continue
+			}
+			found = true
+			if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
+				// fast path: the obfuscated result was calculated before
+				tmp.Value = obfuscatedValue
+				continue
+			}
+
+			digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
+			buf := olctx.buf
+			bufLen := len(buf)
+			buf = hex.AppendEncode(buf, digest[:])
+			obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			olctx.buf = buf
+			olctx.cacheResults[tmp.Value] = obfuscatedValue
+			tmp.Value = obfuscatedValue
+		}
+		if found {
+			ts.Labels = labels[labelsLen:]
+		} else {
+			labels = labels[:labelsLen]
+		}
+	}
+	olctx.labels = labels
+	return tss
+}
+
+func (rwctx *remoteWriteCtx) initObfuscateLabels() {
+	if len(*obfuscateLabels) == 0 {
+		return
+	}
+	idx := rwctx.idx
+	rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
+	rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
+
+	for _, label := range rwObfuscateLabelsList {
+		if label == "" {
+			continue
+		}
+		if !slices.Contains(rwctx.obfuscateLabels, label) {
+			rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
+		}
+	}
+}

app/vmagent/remotewrite/obfuscate_test.go

@@ -0,0 +1,186 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func TestRemoteWriteObfuscateLabels(t *testing.T) {
+	f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
+		t.Helper()
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		defer metrics.UnregisterAllMetrics()
+		originValue := *obfuscateLabels
+		defer func() {
+			*obfuscateLabels = originValue
+		}()
+		*obfuscateLabels = []string{obfuscateLabelList}
+		rwctx.initObfuscateLabels()
+
+		outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
+
+		if !reflect.DeepEqual(expectedTss, outputTss) {
+			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
+		}
+	}
+
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+
+	// 1. obfuscation is not set.
+	f("",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 1. obfuscation is set for another rwctx.
+	f(",ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 2. obfuscate the value of "ip" label
+	f("ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 3. obfuscate the values of "ip" and "instance"
+	f("ip^^instance",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: sha256Result("1234")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 4. duplicate label names in config must produce single SHA-256, not double
+	f("ip^^ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+}

app/vmagent/remotewrite/obfuscate_timing_test.go

@@ -0,0 +1,94 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
+	originValue := *obfuscateLabels
+	defer func() {
+		*obfuscateLabels = originValue
+	}()
+	*obfuscateLabels = []string{"ip^^instance"}
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+	expected := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("123")},
+				{Name: "instance", Value: sha256Result("12345")},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("1236")},
+				{Name: "instance", Value: sha256Result("some-long-instante-string")},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	defer metrics.UnregisterAllMetrics()
+
+	inputTss := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "123"},
+				{Name: "instance", Value: "12345"},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "1236"},
+				{Name: "instance", Value: "some-long-instante-string"},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		rwctx.initObfuscateLabels()
+		var localTss []prompb.TimeSeries
+		for pb.Next() {
+			// always make a shallow copy because obfuscate changes input
+			localTss = localTss[:0]
+			localTss = append(localTss, inputTss...)
+			olctx.reset()
+			outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
+			if !reflect.DeepEqual(expected, outputTss) {
+				b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
+			}
+		}
+	})
+
+}

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmagent/remotewrite/remotewrite.go

@@ -107,7 +107,13 @@ var (
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
 
 	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
 		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
+	obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
+		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
+		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
+	maintenanceAuthKey = flagutil.NewPassword("remoteWrite.maintenanceAuthKey", "Auth key for /remotewrite/maintenance http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 )
 
 var (
@@ -136,6 +142,42 @@ func MultitenancyEnabled() bool {
 	return *enableMultitenantHandlers
 }
 
+// SetMaintenanceMode enables or disables maintenance mode for the remote write destination(s)
+// identified by sanitizedURL, which must match the `url` label of the corresponding
+// vmagent_remotewrite_* metrics. Pass "*" to target all the configured -remoteWrite.url destinations.
+//
+// While a destination is in maintenance mode, vmagent doesn't attempt to send data to it at all -
+// it stops draining that destination's queue; buffering remains subject to the configured queue limits.
+//
+// It returns the number of destinations matched by sanitizedURL.
+func SetMaintenanceMode(sanitizedURL string, enabled bool) int {
+	matched := 0
+	for _, rwctx := range rwctxsGlobal {
+		if sanitizedURL != "*" && rwctx.c.sanitizedURL != sanitizedURL {
+			continue
+		}
+		rwctx.c.maintenanceMode.Store(enabled)
+		matched++
+	}
+	return matched
+}
+
+// GetMaintenanceMode returns the maintenance mode status for each configured -remoteWrite.url,
+// keyed by its sanitized URL.
+func GetMaintenanceMode() map[string]bool {
+	m := make(map[string]bool, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
+		m[rwctx.c.sanitizedURL] = rwctx.c.maintenanceMode.Load()
+	}
+	return m
+}
+
+// CheckMaintenanceAuthKey verifies the authKey query arg for the /remotewrite/maintenance http endpoint
+// against -remoteWrite.maintenanceAuthKey. See httpserver.CheckAuthFlag for the semantics of the return value.
+func CheckMaintenanceAuthKey(w http.ResponseWriter, r *http.Request) bool {
+	return httpserver.CheckAuthFlag(w, r, maintenanceAuthKey)
+}
+
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Pointer[relabelConfigs]
 
@@ -151,7 +193,8 @@ var maxQueues = cgroup.AvailableCPUs() * 16
 
 const persistentQueueDirname = "persistent-queue"
 
-// InitSecretFlags must be called after flag.Parse and before any logging.
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
@@ -240,6 +283,8 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
+	configReloaderStopCh = make(chan struct{})
+	configReloaderWG = sync.WaitGroup{}
 	configReloaderWG.Go(func() {
 		for {
 			select {
@@ -326,7 +371,7 @@ func initRemoteWriteCtxs(urls []string) {
 }
 
 var (
-	configReloaderStopCh = make(chan struct{})
+	configReloaderStopCh chan struct{}
 	configReloaderWG     sync.WaitGroup
 )
 
@@ -571,6 +616,14 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
 			mm.ProjectID = at.ProjectID
 		}
 	}
+	tmp := mms[:0]
+	for _, mm := range mms {
+		if timeserieslimits.IsMetricMetadataExceeding(&mm) {
+			continue
+		}
+		tmp = append(tmp, mm)
+	}
+	mms = tmp
 	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
 	// Since metadata is usually small and there is no guarantee that metadata can be sent to
 	// the same remote storage with the corresponding metrics.
@@ -873,6 +926,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
+	obfuscateLabels []string
+
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 	mdxRowsPreserved       *metrics.Counter
@@ -987,6 +1042,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
+	rwctx.initObfuscateLabels()
 
 	if enableMdx.GetOptionalArg(argIdx) {
 		mdxFilter := mdx.NewFilter()
@@ -1056,7 +1112,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 
 	copyTimeSeriesIfNeeded := func() {
 		if v == nil {
-			v := tssPool.Get().(*[]prompb.TimeSeries)
+			v = tssPool.Get().(*[]prompb.TimeSeries)
 			tss = append(*v, tss...)
 		}
 	}
@@ -1190,24 +1246,41 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
 func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
+	var olctx *obfuscateLabelsCtx
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if olctx != nil {
+			putObfuscateLabelsCtx(olctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
+	if len(rwctx.obfuscateLabels) != 0 {
+		copyTimeSeriesIfNeeded()
+		olctx = getObfuscateLabelsCtx()
+		tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
+	}
+
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )
@@ -282,7 +289,8 @@ func processFlags() {
 
 func setUp() {
 	const maxConcurrentRequests = 4
-	vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	maxQueueDuration := 5 * time.Second
+	vmstorage.Init(maxConcurrentRequests, maxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {

app/vmalert/datasource/client.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -267,7 +268,11 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
+		err = &httpserver.ErrorWithStatusCode{
+			StatusCode: resp.StatusCode,
+			Err:        fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
+		}
+		return nil, err
 	}
 	return resp, nil
 }

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -59,7 +60,8 @@ var (
 		`Only valid for VictoriaMetrics as the datasource.`)
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")
@@ -94,6 +96,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +128,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/main.go

@@ -88,10 +88,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remoteread.InitSecretFlags()
-	remotewrite.InitSecretFlags()
-	datasource.InitSecretFlags()
-	notifier.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -438,3 +435,12 @@ func getLastConfigError() error {
 	defer lastConfigErrMu.RUnlock()
 	return lastConfigErr
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
+	notifier.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmalert/notifier/alert.go

@@ -31,6 +31,8 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
+	// Interval contains the evaluation interval of the Alert's group
+	Interval time.Duration
 	// ActiveAt defines the moment of time when Alert has become active
 	ActiveAt time.Time
 	// Start defines the moment of time when Alert has become firing
@@ -84,6 +86,7 @@ type AlertTplData struct {
 	Labels    map[string]string
 	Value     float64
 	Expr      string
+	Interval  time.Duration
 	AlertID   uint64
 	GroupID   uint64
 	ActiveAt  time.Time
@@ -96,6 +99,7 @@ var tplHeaders = []string{
 	"{{ $type := .Type }}",
 	"{{ $labels := .Labels }}",
 	"{{ $expr := .Expr }}",
+	"{{ $interval := .Interval }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
 	"{{ $alertID := .AlertID }}",
@@ -115,6 +119,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		Type:     a.Type,
 		Labels:   labels,
 		Expr:     a.Expr,
+		Interval: a.Interval,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,

app/vmalert/notifier/alert_test.go

@@ -129,6 +129,17 @@ func TestAlertExecTemplate(t *testing.T) {
 		"exprEscapedHTML":   "vm_rows{"label"="bar"}<0",
 	})
 
+	// interval-template
+	f(&Alert{
+		Interval: 10 * time.Second,
+	}, map[string]string{
+		"interval":         "{{ .Interval }}",
+		"intervalVariable": "{{ $interval }}",
+	}, map[string]string{
+		"interval":         "10s",
+		"intervalVariable": "10s",
+	})
+
 	// query
 	f(&Alert{
 		Expr: `vm_rows{"label"="bar"}>0`,

app/vmalert/notifier/init.go

@@ -189,7 +189,8 @@ func Init(extLabels map[string]string, extURL string) error {
 	return nil
 }
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -54,7 +56,8 @@ var (
 	oauth2Scopes   = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")
@@ -76,7 +79,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +98,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmalert/remotewrite/init.go

@@ -57,7 +57,8 @@ var (
 	oauth2Scopes   = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")

app/vmalert/replay.go

@@ -25,11 +25,13 @@ var (
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
 		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
-		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+		"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, "+
+		"which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
@@ -73,7 +75,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 
 	for _, cfg := range groupsCfg {
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
-		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
+		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
 	}
 	logger.Infof("replay evaluation finished, generated %d samples", totalRows)
 	if err := rw.Close(); err != nil {

app/vmalert/replay_test.go

@@ -8,8 +8,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 type fakeReplayQuerier struct {
@@ -32,6 +34,14 @@ func (fc *fakeRWClient) Close() error {
 }
 
 func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
+	_, err = metricsql.Parse(q)
+	if err != nil {
+		return res, &httpserver.ErrorWithStatusCode{
+			StatusCode: 422,
+			Err:        err,
+		}
+	}
+
 	key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
 	dps, ok := fr.registry[q]
 	if !ok {
@@ -275,4 +285,28 @@ func TestReplay(t *testing.T) {
 			},
 		},
 	}, 10)
+
+	// rule with wrong expression won't break the other rule with continueWithExecutionErr
+	continueWithExecutionErrOld := *continueWithExecutionErr
+	defer func() {
+		*continueWithExecutionErr = continueWithExecutionErrOld
+	}()
+	*continueWithExecutionErr = true
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+		{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string][]datasource.Metric{
+			"sum(up)": {
+				"12:00:00+12:01:00": {
+					{
+						Timestamps: []int64{1, 2},
+						Values:     []float64{1, 2},
+					},
+				},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	}, 2)
 }

app/vmalert/rule/alerting.go

@@ -530,6 +530,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
 			a.Value = m.Values[0]
+			a.Interval = ar.EvalInterval
 			a.Annotations = annotations
 			a.KeepFiringSince = time.Time{}
 			continue
@@ -612,6 +613,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 		Type:      ar.Type.String(),
 		Labels:    ls.origin,
 		Expr:      ar.Expr,
+		Interval:  ar.EvalInterval,
 		AlertID:   hash(ls.processed),
 		GroupID:   ar.GroupID,
 		ActiveAt:  activeAt,
@@ -673,6 +675,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
 		Name:        ar.Name,
 		Type:        ar.Type.String(),
 		Expr:        ar.Expr,
+		Interval:    ar.EvalInterval,
 		For:         ar.For,
 		ActiveAt:    start,
 		Value:       m.Values[0],

app/vmalert/rule/alerting_test.go

@@ -662,6 +662,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -682,6 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "for-firing"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -703,6 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -759,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -771,6 +775,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -1134,7 +1139,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		fq.Add(metrics...)
 		fq.SetPartialResponse(isResponsePartial)
 
-		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
+		ts := time.Unix(3600, 0)
+		if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		for hash, expAlert := range alertsExpected {
@@ -1152,12 +1158,14 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 
 	f(&AlertingRule{
-		Name: "common",
+		Name:         "common",
+		EvalInterval: time.Hour,
 		Labels: map[string]string{
 			"region": "east",
 		},
 		Annotations: map[string]string{
-			"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":   `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1166,7 +1174,8 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "foo"`,
+				"summary":   `common: Too high connection number for "foo"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1176,7 +1185,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		},
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "bar"`,
+				"summary":   `common: Too high connection number for "bar"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1388,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
 		"pod":           "vmalert-0",
-		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1398,7 +1408,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/group.go

@@ -97,6 +97,7 @@ type groupMetrics struct {
 	iterationMissed   *metrics.Counter
 	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
+	iterationLimit    *metrics.Gauge
 }
 
 // merges group rule labels into result map
@@ -289,6 +290,8 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Headers = newGroup.Headers
 	g.NotifierHeaders = newGroup.NotifierHeaders
 	g.Labels = newGroup.Labels
+	g.EvalDelay = newGroup.EvalDelay
+	g.evalAlignment = newGroup.evalAlignment
 	g.Limit = newGroup.Limit
 	g.checksum = newGroup.checksum
 	g.Rules = newRules
@@ -336,6 +339,12 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
+	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_group_rule_results_limit{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		limit := g.Limit
+		g.mu.RUnlock()
+		return float64(limit)
+	})
 	for i := range g.Rules {
 		g.Rules[i].registerMetrics(g.metrics.set)
 	}
@@ -541,7 +550,7 @@ func (g *Group) infof(format string, args ...any) {
 }
 
 // Replay performs group replay
-func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
+func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	var total int
 	step := g.Interval * time.Duration(maxDataPoint)
 	ri := rangeIterator{start: start, end: end, step: step}
@@ -569,7 +578,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 			if !disableProgressBar {
 				bar = pb.StartNew(iterations)
 			}
-			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			if bar != nil {
 				bar.Finish()
 			}
@@ -591,7 +600,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 		rule := g.Rules[i]
 		sem <- struct{}{}
 		wg.Go(func() {
-			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			<-sem
 		})
 	}
@@ -611,7 +620,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	return total
 }
 
-func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
+func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
 	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
@@ -626,7 +635,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 		start := ri.s
 		end := ri.e
 		wg.Go(func() {
-			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
+			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}

app/vmalert/rule/group_test.go

@@ -78,6 +78,12 @@ func TestUpdateWith(t *testing.T) {
 		if g.Debug != expect.Debug {
 			t.Fatalf("expected to have debug %v; got %v", expect.Debug, g.Debug)
 		}
+		if !durationPtrEqual(g.EvalDelay, expect.EvalDelay) {
+			t.Fatalf("expected to have eval_delay %v; got %v", expect.EvalDelay, g.EvalDelay)
+		}
+		if !boolPtrEqual(g.evalAlignment, expect.evalAlignment) {
+			t.Fatalf("expected to have eval_alignment %v; got %v", expect.evalAlignment, g.evalAlignment)
+		}
 	}
 
 	// new rule
@@ -237,6 +243,37 @@ func TestUpdateWith(t *testing.T) {
 			{Alert: "foo1", Debug: &debug},
 		},
 	})
+
+	// update group evaluation settings
+	evalDelay := promutil.NewDuration(time.Minute)
+	evalAlignment := false
+	f(config.Group{
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "max(up)",
+		}},
+	}, config.Group{
+		EvalDelay:     evalDelay,
+		EvalAlignment: &evalAlignment,
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "min(up)",
+		}},
+	})
+}
+
+func durationPtrEqual(a, b *time.Duration) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	return *a == *b
+}
+
+func boolPtrEqual(a, b *bool) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	return *a == *b
 }
 
 func TestUpdateDuringRandSleep(t *testing.T) {

app/vmalert/rule/rule.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -118,7 +120,7 @@ func (s *ruleState) add(e StateEntry) {
 	s.entries[s.cur] = e
 }
 
-func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
+func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
 	var err error
 	var tss []prompb.TimeSeries
 	for i := range replayRuleRetryAttempts {
@@ -126,6 +128,22 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		if err == nil {
 			break
 		}
+		// retry request if possible to tolerate temporary network or datasource unavailability issues
+		var esc *httpserver.ErrorWithStatusCode
+		if errors.As(err, &esc) {
+			statusCode := esc.StatusCode
+			// if the status code is 400 or 422, the query failed due to reasons such as an expression syntax error or a resource limit being hit,
+			// rather than datasource unavailability.
+			// Continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity || statusCode == http.StatusBadRequest {
+				if continueWithExecutionErr {
+					logger.Errorf("rule %q: %s", r, err)
+					return 0, nil
+				} else {
+					return 0, err
+				}
+			}
+		}
 		logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
 		time.Sleep(time.Second)
 	}

app/vmalert/rule/web.go

@@ -75,10 +75,13 @@ type GroupAlerts struct {
 // ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type ApiRule struct {
-	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
+	// Rule state must be one of these under following scenarios:
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state. (only for alerting rules)
+	//  "firing": at least 1 alert in the rule in firing state. (only for alerting rules)
+	//  "inactive": rule's last evaluation was successful but no alert in the rule in firing or pending state. (only for alerting rules)
+	//  "unhealthy": rule's last evaluation was failed with error. (for both alerting and recording rules)
+	// 	"nomatch": rule's last evaluation was successful but no time series matched the rule's expression. (for both alerting and recording rules)
+	//  "ok": the recording rule's last evaluation was successful. (only for recording rules)
 	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
@@ -237,6 +240,7 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 }
 
 func (r *ApiRule) ExtendState() {
+	// if alerting rule already has alerts, then state is already set to either "pending" or "firing" and we don't need to change it
 	if len(r.Alerts) > 0 {
 		return
 	}

app/vmalert/web.go

@@ -115,7 +115,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		WriteRule(w, r, rule)
 		return true
-	// current used by old vmalert UI and Grafana Alerts
+	// used by old vmalert UI
 	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
@@ -128,6 +128,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			state = rf.states[0]
 			rf.states = rf.states[:1]
 		}
+		// enable extendedStates by default for vmalert UI
+		rf.extendedStates = true
 		lr := rh.groups(rf)
 		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
@@ -543,6 +545,8 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
+			// extendedStates is used by the vmalert UI to extend the rule state with values such as "nomatch" and "unhealthy".
+			// those states are not supported by Grafana and not part of the Prometheus API spec yet
 			if rf.extendedStates {
 				rule.ExtendState()
 			}

app/vmalert/web.qtpl

@@ -130,9 +130,12 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
-                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
+                                {% if g.States["inactive"] > 0 %}<span class="badge bg-light text-success border border-success" title="None of the alert instances is in a pending or firing state">{%d g.States["inactive"] %} inactive</span> {% endif %}
+                                {% if g.States["pending"] > 0 %}<span class="badge bg-warning text-dark" title="At least one alert instance is pending">{%d g.States["pending"] %} pending</span> {% endif %}
+                                {% if g.States["firing"] > 0 %}<span class="badge bg-danger" title="At least one alert instance is firing">{%d g.States["firing"] %} firing</span> {% endif %}
+                                {% if g.States["ok"] > 0 %}<span class="badge bg-success" title="Recording rule last evaluation succeeded">{%d g.States["ok"] %} ok</span>{% endif %}
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Last evaluation failed with an error">{%d g.States["unhealthy"] %} unhealthy</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Rule expression matched no time series">{%d g.States["nomatch"] %} nomatch</span> {% endif %}
                             </span>
                         </span>
                     </span>
@@ -169,7 +172,7 @@
                             <thead>
                                 <tr>
                                     <th scope="col" class="w-60">Rule</th>
-                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
+                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series Returned</th>
                                     <th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -188,8 +191,21 @@
                                                     {% else %}
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
-                                                    |
+                                                    {% if r.State == "inactive" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "pending" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "firing" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "ok" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "unhealthy" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "nomatch" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% endif %}
                                                     {%= seriesFetchedWarn(prefix, &r) %}
+                                                    |
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">

app/vmauth/auth_config.go

@@ -118,9 +118,10 @@ type AccessLogFilters struct {
 }
 
 func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
-	if ui.AccessLog == nil {
+	if ui == nil || ui.AccessLog == nil {
 		return
 	}
+
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -134,6 +135,17 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
 		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
+// hasAnyURLs reports whether ui has at least one backend URL route configured.
+// It is used only for unauthorized_user config section, since other users
+// must always have either URLPrefix or URLMaps set.
+func (ui *UserInfo) hasAnyURLs() bool {
+	if ui == nil {
+		return false
+	}
+
+	return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`
@@ -983,8 +995,11 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-		if err := ui.initURLs(); err != nil {
-			return nil, err
+
+		if ui.hasAnyURLs() {
+			if err := ui.initURLs(); err != nil {
+				return nil, err
+			}
 		}
 
 		metricLabels, err := ui.getMetricLabels()

app/vmauth/main.go

@@ -96,6 +96,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -175,11 +176,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
-		if ui != nil {
+		if ui.hasAnyURLs() {
 			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
+		ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 		handleMissingAuthorizationError(w)
 		return true
 	}
@@ -192,23 +194,24 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if tkn == nil {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
-		if !tkn.HasVMAccessClaim() && ui.JWT.DefaultVMAccessClaim == nil {
-			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		defer putToken(tkn)
+		// Call processUserRequest only if the token contains the vm_access claim
+		// or a default claim is configured; otherwise fall through to unauthorized_user.
+		if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
+			processUserRequest(w, r, ui, tkn)
 			return true
 		}
-		defer putToken(tkn)
-		processUserRequest(w, r, ui, tkn)
-		return true
 	}
 
 	uu := authConfig.Load().UnauthorizedUser
-	if uu != nil {
+	if uu.hasAnyURLs() {
 		processUserRequest(w, r, uu, nil)
 		return true
 	}
 
 	invalidAuthTokenRequests.Inc()
 	slowdownUnauthorizedResponse(r)
+	uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 	if *logInvalidAuthTokens {
 		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 		err = &httpserver.ErrorWithStatusCode{
@@ -384,7 +387,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 			return nil, &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
 					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
+				StatusCode: http.StatusRequestTimeout,
 			}
 		}
 
@@ -909,3 +912,8 @@ func slowdownUnauthorizedResponse(r *http.Request) {
 	}
 	timerpool.Put(t)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmauth/main_synctest_test.go

@@ -0,0 +1,54 @@
+//go:build synctest
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+func TestBufferRequestBody_Timeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+
+		defaultMaxQueueDuration := *maxQueueDuration
+		defer func() {
+			*maxQueueDuration = defaultMaxQueueDuration
+		}()
+
+		*maxQueueDuration = 100 * time.Millisecond
+
+		ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
+		defer cancel()
+
+		_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		esc, ok := err.(*httpserver.ErrorWithStatusCode)
+		if !ok {
+			t.Fatalf("unexpected error type: %s", err)
+		}
+		if esc.StatusCode != http.StatusRequestTimeout {
+			t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
+		}
+	})
+}
+
+type timeoutBody struct {
+	delay time.Duration
+}
+
+func (r *timeoutBody) Read(_ []byte) (int, error) {
+	time.Sleep(r.delay)
+	return 0, context.DeadlineExceeded
+}
+
+func (r *timeoutBody) Close() error {
+	return nil
+}

app/vmauth/main_test.go

@@ -785,7 +785,26 @@ statusCode=401
 Unauthorized`
 	f(simpleCfgStr, request, responseExpected)
 
-	// token without vm_access claim is accepted when it
+	// token without vm_access claim should fall through to unauthorized_user
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	responseExpected = `
+statusCode=200
+path: /bar/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+unauthorized_user:
+  url_prefix: {BACKEND}/bar
+users:
+- jwt:
+    public_keys:
+    - %q
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
+	// token without vm_access claim is accepted when default_vm_access_claim configured
 	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
 	request.Header.Set(`Authorization`, `Bearer `+roleToken)
 	responseExpected = `

app/vmbackup/main.go

@@ -47,9 +47,8 @@ func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
-	flagutil.RegisterSecretFlag("snapshot.createURL")
-	flagutil.RegisterSecretFlag("snapshot.deleteURL")
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -273,3 +272,10 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	flagutil.RegisterSecretFlag("snapshot.createURL")
+	flagutil.RegisterSecretFlag("snapshot.deleteURL")
+	pushmetrics.InitSecretFlags()
+}

app/vmctl/main.go

@@ -47,6 +47,7 @@ func main() {
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
 		flag.Parse()
+		initSecretFlags()
 		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
@@ -619,3 +620,8 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		Backoff:            bf,
 	}, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmctl/remoteread/remoteread.go

@@ -8,12 +8,14 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -234,9 +236,29 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
 	// shouldn't be accounted as an error.
 	for _, res := range readResp.Results {
 		for _, ts := range res.Timeseries {
-			vmTs := convertSamples(ts.Samples, ts.Labels)
-			if err := callback(vmTs); err != nil {
-				return err
+			// A series contains either float samples or native histogram samples.
+			// Both fields are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain samples of both types.
+			if len(ts.Samples) > 0 {
+				vmTs := convertSamples(ts.Samples, ts.Labels)
+				if err := callback(vmTs); err != nil {
+					return err
+				}
+			}
+			if len(ts.Histograms) > 0 {
+				hSamples := make([]histogramSample, 0, len(ts.Histograms))
+				for _, h := range ts.Histograms {
+					hSamples = append(hSamples, histogramSample{
+						timestamp: h.Timestamp,
+						fh:        h.ToFloatHistogram(),
+					})
+				}
+				for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
+					if err := callback(vmTs); err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -263,17 +285,45 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
 
 		for _, series := range res.ChunkedSeries {
 			samples := make([]prompb.Sample, 0)
+			var hSamples []histogramSample
 			for _, chunk := range series.Chunks {
-				s, err := parseSamples(chunk.Data)
-				if err != nil {
-					return err
+				switch chunk.Type {
+				case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
+					// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
+					// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
+					// MUST be set when using anything else than XOR". Senders before native
+					// histograms support (Prometheus < 2.40) do not set this field at all,
+					// so UNKNOWN chunks must be parsed as XOR ones.
+					s, err := parseSamples(chunk.Data)
+					if err != nil {
+						return err
+					}
+					samples = append(samples, s...)
+				case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
+					hs, err := parseHistograms(chunk.Type, chunk.Data)
+					if err != nil {
+						return err
+					}
+					hSamples = append(hSamples, hs...)
+				default:
+					return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
 				}
-				samples = append(samples, s...)
 			}
 
-			ts := convertSamples(samples, series.Labels)
-			if err := callback(ts); err != nil {
-				return err
+			// A series contains either XOR chunks or native histogram chunks.
+			// Both are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain chunks of both types.
+			if len(samples) > 0 {
+				ts := convertSamples(samples, series.Labels)
+				if err := callback(ts); err != nil {
+					return err
+				}
+			}
+			for _, ts := range convertHistograms(hSamples, series.Labels) {
+				if err := callback(ts); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -312,6 +362,151 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
 	return samples, it.Err()
 }
 
+// histogramSample represents a single native histogram sample.
+type histogramSample struct {
+	timestamp int64
+	fh        *histogram.FloatHistogram
+}
+
+func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
+	var enc chunkenc.Encoding
+	switch encoding {
+	case prompb.Chunk_HISTOGRAM:
+		enc = chunkenc.EncHistogram
+	case prompb.Chunk_FLOAT_HISTOGRAM:
+		enc = chunkenc.EncFloatHistogram
+	default:
+		return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
+	}
+	c, err := chunkenc.FromData(enc, chunk)
+	if err != nil {
+		return nil, fmt.Errorf("error read chunk: %w", err)
+	}
+
+	var hSamples []histogramSample
+	it := c.Iterator(nil)
+	for {
+		typ := it.Next()
+		if typ == chunkenc.ValNone {
+			break
+		}
+		switch typ {
+		case chunkenc.ValHistogram:
+			ts, h := it.AtHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        h.ToFloat(nil),
+			})
+		case chunkenc.ValFloatHistogram:
+			ts, fh := it.AtFloatHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        fh,
+			})
+		default:
+			// Skip unsupported values
+			continue
+		}
+	}
+	if err := it.Err(); err != nil {
+		return nil, fmt.Errorf("error iterate over chunks: %w", err)
+	}
+
+	return hSamples, nil
+}
+
+// convertHistograms converts native histogram samples into VictoriaMetrics histogram
+// time series in the same way as VictoriaMetrics converts native histograms
+// received via Prometheus remote write protocol: every native histogram sample
+// is converted into `<name>_count` and `<name>_sum` series plus a set of
+// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
+// The only difference is that for native histograms with custom buckets (NHCB)
+// bucket bounds are taken from the custom values, while the remote write protocol
+// parser ignores custom values and estimates the bounds with the exponential formula.
+// See https://prometheus.io/docs/specs/native_histograms/#data-model
+func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
+	if len(hSamples) == 0 {
+		return nil
+	}
+
+	labelPairs := make([]vm.LabelPair, 0, len(labels))
+	nameValue := ""
+	for _, label := range labels {
+		if label.Name == "__name__" {
+			nameValue = label.Value
+			continue
+		}
+		labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
+	}
+	// the metric has no name, skip it in the same way as VictoriaMetrics does
+	// when it receives a native histogram without the metric name via remote write protocol.
+	if nameValue == "" {
+		return nil
+	}
+
+	countSeries := &vm.TimeSeries{
+		Name:       nameValue + "_count",
+		LabelPairs: labelPairs,
+	}
+	sumSeries := &vm.TimeSeries{
+		Name:       nameValue + "_sum",
+		LabelPairs: labelPairs,
+	}
+	bucketSeries := make(map[string]*vm.TimeSeries)
+	// vmranges preserves the order of bucketSeries creation
+	// in order to get deterministic results.
+	var vmranges []string
+
+	for _, hs := range hSamples {
+		fh := hs.fh
+		countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
+		countSeries.Values = append(countSeries.Values, fh.Count)
+		sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
+		sumSeries.Values = append(sumSeries.Values, fh.Sum)
+
+		it := fh.AllBucketIterator()
+		for it.Next() {
+			b := it.At()
+			if b.Count <= 0 {
+				continue
+			}
+			vmrange := formatVmrange(b.Lower, b.Upper)
+			s := bucketSeries[vmrange]
+			if s == nil {
+				bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
+				copy(bucketLabelPairs, labelPairs)
+				bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
+				s = &vm.TimeSeries{
+					Name:       nameValue + "_bucket",
+					LabelPairs: bucketLabelPairs,
+				}
+				bucketSeries[vmrange] = s
+				vmranges = append(vmranges, vmrange)
+			}
+			s.Timestamps = append(s.Timestamps, hs.timestamp)
+			s.Values = append(s.Values, b.Count)
+		}
+	}
+
+	tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
+	tss = append(tss, countSeries, sumSeries)
+	for _, vmrange := range vmranges {
+		tss = append(tss, bucketSeries[vmrange])
+	}
+	return tss
+}
+
+// formatVmrange formats the given bucket bounds into `vmrange` label value
+// in the same way as VictoriaMetrics does for native histograms
+// received via Prometheus remote write protocol.
+func formatVmrange(lower, upper float64) string {
+	b := make([]byte, 0, 24)
+	b = strconv.AppendFloat(b, lower, 'e', 3, 64)
+	b = append(b, "..."...)
+	b = strconv.AppendFloat(b, upper, 'e', 3, 64)
+	return string(b)
+}
+
 type keyValue struct {
 	key   string
 	value string

app/vmctl/remoteread/remoteread_test.go

@@ -0,0 +1,334 @@
+package remoteread
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage/remote"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+)
+
+func testHistogram(mul int64) *histogram.Histogram {
+	return &histogram.Histogram{
+		Schema:          0,
+		Count:           uint64(10 * mul),
+		Sum:             25.5 * float64(mul),
+		ZeroThreshold:   0.001,
+		ZeroCount:       uint64(2 * mul),
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+		PositiveBuckets: []int64{1 * mul, 2 * mul},
+		NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		NegativeBuckets: []int64{4 * mul},
+	}
+}
+
+func TestConvertHistograms(t *testing.T) {
+	f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
+		t.Helper()
+
+		tss := convertHistograms(hSamples, labels)
+		if !reflect.DeepEqual(tss, expected) {
+			t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
+		}
+	}
+
+	// series without samples
+	f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
+
+	// series without the metric name must be skipped
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+	}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
+
+	// native histogram must be converted to _count, _sum and _bucket series
+	// in the same way as VictoriaMetrics does for Prometheus remote write protocol
+	labels := []prompb.Label{
+		{Name: "__name__", Value: "request_duration_seconds"},
+		{Name: "job", Value: "bar"},
+	}
+	jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
+	bucketLabels := func(vmrange string) []vm.LabelPair {
+		return []vm.LabelPair{
+			{Name: "job", Value: "bar"},
+			{Name: "vmrange", Value: vmrange},
+		}
+	}
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+		{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
+	}, labels, []*vm.TimeSeries{
+		{
+			Name:       "request_duration_seconds_count",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{10, 20},
+		},
+		{
+			Name:       "request_duration_seconds_sum",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{25.5, 51},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{4, 8},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{2, 4},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{1, 2},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{3, 6},
+		},
+	})
+}
+
+func TestParseHistograms(t *testing.T) {
+	c := chunkenc.NewHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse histogram chunk: %s", err)
+	}
+	if len(hSamples) != 2 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
+	}
+	for i, expected := range []struct {
+		timestamp int64
+		count     float64
+		sum       float64
+	}{
+		{timestamp: 1000, count: 10, sum: 25.5},
+		{timestamp: 2000, count: 20, sum: 51},
+	} {
+		if hSamples[i].timestamp != expected.timestamp {
+			t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
+		}
+		if hSamples[i].fh.Count != expected.count {
+			t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
+		}
+		if hSamples[i].fh.Sum != expected.sum {
+			t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
+		}
+	}
+
+	// unsupported chunk encoding must return error
+	if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
+		t.Fatalf("expecting non-nil error for unsupported chunk encoding")
+	}
+}
+
+func TestProcessResponse(t *testing.T) {
+	readResp := &prompb.ReadResponse{
+		Results: []*prompb.QueryResult{
+			{
+				Timeseries: []*prompb.TimeSeries{
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "cpu_usage"},
+							{Name: "job", Value: "bar"},
+						},
+						Samples: []prompb.Sample{
+							{Timestamp: 1000, Value: 1.5},
+						},
+					},
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "request_duration_seconds"},
+							{Name: "job", Value: "bar"},
+						},
+						Histograms: []prompb.Histogram{
+							prompb.FromIntHistogram(1000, testHistogram(1)),
+						},
+					},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(readResp)
+	if err != nil {
+		t.Fatalf("cannot marshal ReadResponse: %s", err)
+	}
+	compressed := snappy.Encode(nil, data)
+
+	var tss []*vm.TimeSeries
+	err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process response: %s", err)
+	}
+
+	// 1 float series + _count + _sum + 4 buckets
+	if len(tss) != 7 {
+		t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
+	}
+	if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[1])
+	}
+	if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[2])
+	}
+	for _, ts := range tss[3:] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+}
+
+type nopFlusher struct{}
+
+func (nopFlusher) Flush() {}
+
+func TestProcessStreamResponse(t *testing.T) {
+	// build a histogram chunk
+	hc := chunkenc.NewHistogramChunk()
+	hApp, err := hc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create histogram chunk appender: %s", err)
+	}
+	if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	// build a float chunk
+	xc := chunkenc.NewXORChunk()
+	xApp, err := xc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create xor chunk appender: %s", err)
+	}
+	xApp.Append(0, 1000, 1.5)
+
+	res := &prompb.ChunkedReadResponse{
+		ChunkedSeries: []*prompb.ChunkedSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "request_duration_seconds"},
+					{Name: "job", Value: "bar"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "cpu_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "memory_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					// the `type` field may be unset for XOR chunks,
+					// such chunks must be parsed as XOR ones
+					{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(res)
+	if err != nil {
+		t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
+	}
+	var buf bytes.Buffer
+	cw := remote.NewChunkedWriter(&buf, nopFlusher{})
+	if _, err := cw.Write(data); err != nil {
+		t.Fatalf("cannot write chunked response: %s", err)
+	}
+
+	var tss []*vm.TimeSeries
+	err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process stream response: %s", err)
+	}
+
+	// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
+	if len(tss) != 8 {
+		t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
+	}
+	if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[1])
+	}
+	for _, ts := range tss[2:6] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+	if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[6])
+	}
+	if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
+	}
+}
+
+func TestParseFloatHistograms(t *testing.T) {
+	c := chunkenc.NewFloatHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	fh := testHistogram(1).ToFloat(nil)
+	if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
+		t.Fatalf("cannot append float histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse float histogram chunk: %s", err)
+	}
+	if len(hSamples) != 1 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
+	}
+	if hSamples[0].timestamp != 1000 {
+		t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
+	}
+	if hSamples[0].fh.Count != 10 {
+		t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
+	}
+}

app/vmctl/vm/vm.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -111,7 +112,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 		if strings.Contains(dst, "?") {
 			separator = "&"
 		}
-		dst += fmt.Sprintf("%sextra_label=%s", separator, extraLabel)
+		dst += fmt.Sprintf("%sextra_label=%s", separator, url.QueryEscape(extraLabel))
 	}
 	return dst, nil
 }

app/vmctl/vm/vm_test.go

@@ -33,11 +33,14 @@ func TestAddExtraLabelsToImportPath_Success(t *testing.T) {
 	f("/api/v1/import", nil, "/api/v1/import")
 
 	// ok one extra label
-	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance=host-1")
+	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance%3Dhost-1")
 
 	// ok two extra labels
-	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
 
 	// ok two extra with exist param
-	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
+
+	// ok special chars in label value
+	f("/api/v1/import", []string{"team=a&b"}, "/api/v1/import?extra_label=team%3Da%26b")
 }

app/vminsert/common/insert_ctx.go

@@ -175,13 +175,19 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmpbs))
-	for idx, mmpb := range mmpbs {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmpbs {
+		if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
+			continue
+		}
+		mm := &mms[cnt]
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
 
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
@@ -201,14 +207,19 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmps))
-	for idx, mmpb := range mmps {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmps {
+		mm := &mms[cnt]
+		if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
+			continue
+		}
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
-
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{

app/vminsert/influx/request_handler.go

@@ -49,7 +49,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vminsert/main.go

@@ -68,9 +68,11 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented.")
+	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
+	maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
 )
 
 var (
@@ -106,7 +108,7 @@ func Init() {
 	promscrape.Init(func(_ *auth.Token, wr *prompb.WriteRequest) {
 		prompush.Push(wr)
 	})
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	timeserieslimits.MustInit(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 }
 
 // Stop stops vminsert.

app/vmrestore/main.go

@@ -38,6 +38,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -112,3 +113,8 @@ func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmselect/main.go

@@ -59,6 +59,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	initVMUIConfig()
 
 	vmalertproxy.Init(*vmalertProxyURL)
+
+}
+
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
+func InitSecretFlags() {
 	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 
@@ -369,6 +375,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -388,6 +398,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -541,7 +555,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmselect/prometheus/prometheus.go

@@ -516,7 +516,7 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	cp.deadline = searchutil.GetDeadlineForDelete(r, startTime)
 
 	if !cp.IsDefaultTimeRange() {
-		return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end)
+		return fmt.Errorf("delete API does not support specific time ranges using start and end args, the series can only be deleted completely")
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxDeleteSeries)
 	deletedCount, err := netstorage.DeleteSeries(nil, sq, cp.deadline)
@@ -540,11 +540,11 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 
@@ -584,7 +584,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 
 	cp, err := getCommonParams(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	cp.deadline = searchutil.GetDeadlineForStatusRequest(r, startTime)
 
@@ -596,7 +596,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		} else {
 			t, err := time.Parse("2006-01-02", dateStr)
 			if err != nil {
-				return fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err)
+				return httpserver.InvalidParamError(fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err))
 			}
 			date = uint64(t.Unix()) / secsPerDay
 		}
@@ -607,7 +607,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if len(topNStr) > 0 {
 		n, err := strconv.Atoi(topNStr)
 		if err != nil {
-			return fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err))
 		}
 		if n <= 0 {
 			n = 1
@@ -645,11 +645,11 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 	labels, err := netstorage.LabelNames(qt, sq, limit, cp.deadline)
@@ -671,10 +671,9 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
 func MetadataHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if limit < 0 {
 		limit = 0
@@ -734,11 +733,11 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, true)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
@@ -772,19 +771,19 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	mayCache := !httputil.GetBool(r, "nocache")
 	query := r.FormValue("query")
 	if len(query) == 0 {
-		return fmt.Errorf("missing `query` arg")
+		return httpserver.InvalidParamError(fmt.Errorf("missing `query` arg"))
 	}
 	start, err := httputil.GetTime(r, "time", ct)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	step, err := httputil.GetDuration(r, "step", lookbackDelta)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if step <= 0 {
 		step = defaultStep
@@ -792,16 +791,16 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 	maxLen := searchutil.GetMaxQueryLen()
 	if len(query) > maxLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
+		return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen))
 	}
 	etfs, err := searchutil.GetExtraTagFilters(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if childQuery, windowExpr, offsetExpr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
 		window, err := windowExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err))
 		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
@@ -815,7 +814,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 		tagFilterss, err := getTagFilterssFromMatches([]string{childQuery})
 		if err != nil {
-			return err
+			return httpserver.InvalidParamError(err)
 		}
 		filterss := searchutil.JoinTagFilterss(tagFilterss, etfs)
 
@@ -831,22 +830,25 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		return nil
 	}
 	if childQuery, windowExpr, stepExpr, offsetExpr := promql.IsRollup(query); childQuery != "" {
+		if len(childQuery) > maxLen {
+			return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(childQuery), maxLen))
+		}
 		newStep, err := stepExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse step in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse step in square brackets at %s: %w", query, err))
 		}
 		if newStep > 0 {
 			step = newStep
 		}
 		window, err := windowExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err))
 		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
 		end := start
 		start = end - window
-		if err := queryRangeHandler(qt, startTime, w, childQuery, start, end, step, r, ct, etfs); err != nil {
+		if err := queryRangeHandler(qt, startTime, w, childQuery, start, end, step, lookbackDelta, r, ct, etfs); err != nil {
 			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", childQuery, start, end, step, err)
 		}
 		return nil
@@ -854,7 +856,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 	queryOffset, err := getLatencyOffsetMilliseconds(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if !httputil.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
 		// Adjust start time only if `nocache` arg isn't set.
@@ -928,45 +930,43 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	ct := startTime.UnixNano() / 1e6
 	query := r.FormValue("query")
 	if len(query) == 0 {
-		return fmt.Errorf("missing `query` arg")
+		return httpserver.InvalidParamError(fmt.Errorf("missing `query` arg"))
+	}
+	maxLen := searchutil.GetMaxQueryLen()
+	if len(query) > maxLen {
+		return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen))
 	}
 	start, err := httputil.GetTime(r, "start", ct-defaultStep)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	end, err := httputil.GetTime(r, "end", ct)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	step, err := httputil.GetDuration(r, "step", defaultStep)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	etfs, err := searchutil.GetExtraTagFilters(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
-	if err := queryRangeHandler(qt, startTime, w, query, start, end, step, r, ct, etfs); err != nil {
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return httpserver.InvalidParamError(err)
+	}
+	if err := queryRangeHandler(qt, startTime, w, query, start, end, step, lookbackDelta, r, ct, etfs); err != nil {
 		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", query, start, end, step, err)
 	}
 	return nil
 }
 
 func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, query string,
-	start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
+	start, end, step, lookbackDelta int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	mayCache := !httputil.GetBool(r, "nocache")
 	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
-	lookbackDelta, err := getMaxLookback(r)
-	if err != nil {
-		return err
-	}
-
-	// Validate input args.
-	maxLen := searchutil.GetMaxQueryLen()
-	if len(query) > maxLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
-	}
 	if start > end {
 		end = start + defaultStep
 	}
@@ -1005,7 +1005,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if step < maxStepForPointsAdjustment.Milliseconds() {
 		queryOffset, err := getLatencyOffsetMilliseconds(r)
 		if err != nil {
-			return err
+			return httpserver.InvalidParamError(err)
 		}
 		if ct-queryOffset < end {
 			result = adjustLastPoints(result, ct-queryOffset, ct+step)
@@ -1156,13 +1156,13 @@ func QueryStatsHandler(w http.ResponseWriter, r *http.Request) error {
 	if len(topNStr) > 0 {
 		n, err := strconv.Atoi(topNStr)
 		if err != nil {
-			return fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err))
 		}
 		topN = n
 	}
 	maxLifetimeMsecs, err := httputil.GetDuration(r, "maxLifetime", 10*60*1000)
 	if err != nil {
-		return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err)
+		return httpserver.InvalidParamError(fmt.Errorf("cannot parse `maxLifetime` arg: %w", err))
 	}
 	maxLifetime := time.Duration(maxLifetimeMsecs) * time.Millisecond
 	w.Header().Set("Content-Type", "application/json")

app/vmselect/promql/binary_op.go

@@ -164,15 +164,21 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		switch true {
-		case metricsql.IsBinaryOpCmp(op):
-			// Do not remove empty series for comparison operations,
-			// since this may lead to missing result.
-		default:
+		isCmpOp := metricsql.IsBinaryOpCmp(op)
+		if !isCmpOp {
+			// Do not remove empty series for comparison operations, since NaN can be an
+			// explicitly present sample value. In particular, `value != NaN` is true.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -183,6 +189,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			logger.Panicf("BUG: len(left) must match len(right) and len(dst); got %d vs %d vs %d", len(left), len(right), len(dst))
 		}
 		isBool := bfa.be.Bool
+		fillLeft := bfa.be.FillLeft
+		fillRight := bfa.be.FillRight
+		// A NaN produced by a vector comparison denotes a filtered-out sample rather
+		// than an explicitly present NaN value. Drop it when this result is used as
+		// the right-hand side of another comparison.
+		// Note that filtered-out samples surviving as NaN inside other wrappers such as
+		// transform functions or arithmetic operations aren't detected, since such NaN
+		// is indistinguishable from an explicitly present NaN value there.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
+		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -193,6 +209,24 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			}
 			for j, a := range leftValues {
 				b := rightValues[j]
+				leftIsNaN := math.IsNaN(a)
+				rightIsNaN := math.IsNaN(b)
+				// Both sides are NaN.
+				if leftIsNaN && rightIsNaN {
+					dstValues[j] = bf(a, b, isBool)
+					continue
+				}
+				if dropNaNRight && rightIsNaN && fillRight == nil {
+					dstValues[j] = nan
+					continue
+				}
+				// Apply the fill value when either the left or right side is NaN, but not both.
+				if leftIsNaN && fillLeft != nil {
+					a = fillLeft.N
+				}
+				if rightIsNaN && fillRight != nil {
+					b = fillRight.N
+				}
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
@@ -202,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
+// isVectorComparisonExpr returns whether e is a comparison operation
+// with at least one non-scalar operand.
+func isVectorComparisonExpr(e metricsql.Expr) bool {
+	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
+		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
+		// the shape of the result. Subqueries are left as is, since rollup
+		// functions skip NaN values on their own.
+		e = re.Expr
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
+		return false
+	}
+	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
+}
+
+// isScalarLikeExpr returns whether e always evaluates to a scalar value.
+func isScalarLikeExpr(e metricsql.Expr) bool {
+	switch e := e.(type) {
+	case *metricsql.NumberExpr, *metricsql.DurationExpr:
+		return true
+	case *metricsql.BinaryOpExpr:
+		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
+	case *metricsql.FuncExpr:
+		switch strings.ToLower(e.Name) {
+		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
+			return true
+		}
+	}
+	return false
+}
+
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {
@@ -226,7 +292,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +305,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0])}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0])}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +370,28 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with NaN values for the fill_left()/fill_right()/fill() modifiers.
+// These NaN values will be replaced later with the fill value if needed.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = math.NaN()
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -424,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -451,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -459,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -470,42 +486,63 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
-	canPushdown := canPushdownCommonFilters(be)
-	if !canPushdown && !shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
-		// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
-		// from exprFirst to exprSecond.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
-		qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
-		defer qt.Done()
-		var wg sync.WaitGroup
-
-		var tssFirst []*timeseries
-		var errFirst error
-		qtFirst := qt.NewChild("expr1")
-		wg.Go(func() {
-			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
-			qtFirst.Done()
-		})
-
-		var tssSecond []*timeseries
-		var errSecond error
-		qtSecond := qt.NewChild("expr2")
-		wg.Go(func() {
-			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
-			qtSecond.Done()
-		})
-
-		wg.Wait()
-		if errFirst != nil {
-			return nil, nil, errFirst
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
 		}
-		if errSecond != nil {
-			return nil, nil, errSecond
-		}
-		return tssFirst, tssSecond, nil
+		return tssSecond, tssFirst, nil
 	}
-	if !canPushdown {
+	if canPushdownCommonFilters(be) {
+		// Execute binary operation in the following way:
+		//
+		// 1) execute the exprFirst
+		// 2) get common label filters for series returned at step 1
+		// 3) push down the found common label filters to exprSecond. This filters out unneeded series
+		//    during exprSecond execution instead of spending compute resources on extracting and processing these series
+		//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+		// 4) execute the exprSecond with possible additional filters found at step 3
+		//
+		// Typical use cases:
+		// - Kubernetes-related: show pod creation time with the node name:
+		//
+		//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
+		//
+		//   Without the optimization `kube_pod_info` would select and spend compute resources
+		//   for more time series than needed. The selected time series would be dropped later
+		//   when matching time series on the right and left sides of binary operand.
+		//
+		// - Generic alerting queries, which rely on `info` metrics.
+		//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
+		//
+		// - Queries, which get additional labels from `info` metrics.
+		//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
+		tssFirst, err := evalExpr(qt, ec, exprFirst)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
+			// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
+			// since the "exprFirst op exprSecond" would return an empty result in any case.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
+			return nil, nil, nil
+		}
+		lfs := getCommonLabelFilters(tssFirst)
+		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
+		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
+		tssSecond, err := evalExpr(qt, ec, exprSecond)
+		if err != nil {
+			return nil, nil, err
+		}
+		return sortResult(tssFirst, tssSecond)
+	}
+
+	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
+	// in exprFirst and exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
+	if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
 		qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
 		defer qt.Done()
 
@@ -525,47 +562,37 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		return tssFirst, tssSecond, nil
 	}
 
-	// Execute binary operation in the following way:
-	//
-	// 1) execute the exprFirst
-	// 2) get common label filters for series returned at step 1
-	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
-	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-	// 4) execute the exprSecond with possible additional filters found at step 3
-	//
-	// Typical use cases:
-	// - Kubernetes-related: show pod creation time with the node name:
-	//
-	//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
-	//
-	//   Without the optimization `kube_pod_info` would select and spend compute resources
-	//   for more time series than needed. The selected time series would be dropped later
-	//   when matching time series on the right and left sides of binary operand.
-	//
-	// - Generic alerting queries, which rely on `info` metrics.
-	//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
-	//
-	// - Queries, which get additional labels from `info` metrics.
-	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-	tssFirst, err := evalExpr(qt, ec, exprFirst)
-	if err != nil {
-		return nil, nil, err
+	// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
+	// from exprFirst to exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
+	qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
+	defer qt.Done()
+	var wg sync.WaitGroup
+
+	var tssFirst []*timeseries
+	var errFirst error
+	qtFirst := qt.NewChild("expr1")
+	wg.Go(func() {
+		tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+	})
+
+	var tssSecond []*timeseries
+	var errSecond error
+	qtSecond := qt.NewChild("expr2")
+	wg.Go(func() {
+		tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+	})
+
+	wg.Wait()
+	if errFirst != nil {
+		return nil, nil, errFirst
 	}
-	if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
-		// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
-		// since the "exprFirst op exprSecond" would return an empty result in any case.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
-		return nil, nil, nil
+	if errSecond != nil {
+		return nil, nil, errSecond
 	}
-	lfs := getCommonLabelFilters(tssFirst)
-	lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
-	exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
-	tssSecond, err := evalExpr(qt, ec, exprSecond)
-	if err != nil {
-		return nil, nil, err
-	}
-	return tssFirst, tssSecond, nil
+	return sortResult(tssFirst, tssSecond)
 }
 
 func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {
@@ -1687,10 +1714,6 @@ func assertInstantValues(tss []*timeseries) {
 
 var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
 
-var _ = metrics.NewGauge(`vm_max_memory_per_query`, func() float64 {
-	return float64(maxMemoryPerQuery.N)
-})
-
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
 	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
 ) ([]*timeseries, error) {

app/vmselect/promql/exec.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -46,15 +47,15 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 
 	e, err := parsePromQLWithCache(q)
 	if err != nil {
-		return nil, err
+		return nil, httpserver.InvalidParamError(err)
 	}
 
 	if *disableImplicitConversion || *logImplicitConversion {
 		isInvalid := metricsql.IsLikelyInvalid(e)
 		if isInvalid && *disableImplicitConversion {
 			// we don't add query=%q to err message as it will be added by the caller
-			return nil, fmt.Errorf("query requires implicit conversion and is rejected according to -search.disableImplicitConversion command-line flag. " +
-				"See https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details")
+			return nil, httpserver.InvalidParamError(fmt.Errorf("query requires implicit conversion and is rejected according to -search.disableImplicitConversion command-line flag. " +
+				"See https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details"))
 		}
 		if isInvalid && *logImplicitConversion {
 			logger.Warnf("query=%q requires implicit conversion, see https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details", e.AppendString(nil))

app/vmselect/promql/exec_test.go

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != NaN`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != 1200`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (1 > 2)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
+		// A missing sample on the left side results in NaN for every comparison
+		// operation, so no special handling is needed there.
+		t.Parallel()
+		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
+		// Prometheus drops the timestamps filtered out by the comparison on the right.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`
@@ -4006,6 +4136,275 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill() both sides NaN case`, func(t *testing.T) {
+		t.Parallel()
+		q := `(
+			label_set(time() <= 1200, "foo", "common")
+		) + fill(10) (
+			label_set(time() >= 1600, "foo", "common")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1010, 1210, nan, 1610, 1810, 2010},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`
@@ -4833,13 +5232,137 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`buckets_limit(zero)`, func(t *testing.T) {
+	// buckets that are consecutively empty at left and right ends will not be preserved.
+	t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
 		t.Parallel()
-		q := `buckets_limit(0, (
-			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
-			alias(label_set(50, "le", "120", "x", "y"), "metric"),
-		))`
-		resultExpected := []netstorage.Result{}
+		q := `sort(buckets_limit(3, (
+			alias(label_set(36, "le", "+Inf"), "metric"),
+			alias(label_set(36, "le", "25"), "metric"),
+			alias(label_set(36, "le", "21"), "metric"),
+			alias(label_set(36, "le", "19"), "metric"),
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+
+	// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
+	t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(buckets_limit(5, (
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("6"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("metric")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.MetricGroup = []byte("metric")
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
 		f(q, resultExpected)
 	})
 	t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6228,50 +6751,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
 		f(q, resultExpected)
 	})
-	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
-		t.Parallel()
-		q := `sort_by_label(
-			buckets_limit(
-				3,
-				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
-			), "le"
-		)`
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("+Inf"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("1.000e+00"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("2.448e+00"),
-			},
-		}
-		resultExpected := []netstorage.Result{r1, r2, r3}
-		f(q, resultExpected)
-	})
 	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

app/vmselect/promql/rollup.go

@@ -1055,7 +1055,7 @@ func newRollupHoltWinters(args []any) (rollupFunc, error) {
 			return nan
 		}
 
-		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing .
+		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29 .
 		// TODO: determine whether this shit really works.
 		s0 := rfa.prevValue
 		if math.IsNaN(s0) {

app/vmselect/promql/timeseries.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"strconv"
 	"sync"
@@ -288,6 +289,9 @@ func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
 }
 
 func marshalBytesFast(dst []byte, s []byte) []byte {
+	if len(s) > math.MaxUint16 {
+		logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
+	}
 	dst = encoding.MarshalUint16(dst, uint16(len(s)))
 	dst = append(dst, s...)
 	return dst

app/vmselect/promql/transform.go

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	if limit <= 0 {
-		return nil, nil
+		return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
 	}
 	if limit < 3 {
 		// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,6 +461,23 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 				prevValue = value
 			}
 		}
+
+		// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
+		epsilon := 1e-9
+		isEmptyBucket := func(hits float64) bool {
+			return !math.IsNaN(hits) && math.Abs(hits) < epsilon
+		}
+		l := 0
+		r := len(leGroup) - 1
+		for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
+			r--
+		}
+		for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
+			l++
+		}
+		leGroup = leGroup[l : r+1]
+
 		for len(leGroup) > limit {
 			// Preserve the first and the last bucket for better accuracy for min and max values
 			xxMinIdx := 1
@@ -1121,29 +1138,29 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 
 func fixBrokenBuckets(i int, xss []leTimeseries) {
 	// Buckets are already sorted by le, so their values must be in ascending order,
-	// since the next bucket includes all the previous buckets.
-	// If the next bucket has lower value than the current bucket,
-	// then the next bucket must be substituted with the current bucket value.
+	// since the upper bucket includes all the lower buckets.
+	// If the upper bucket has lower value than the current bucket,
+	// then the upper bucket must be substituted with the current bucket value.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102
 	if len(xss) < 2 {
 		return
 	}
 
-	vNext := xss[0].ts.Values[i]
+	vPrev := xss[0].ts.Values[i]
 	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
 	// compared with upper buckets in the loop below.
-	if math.IsNaN(vNext) {
-		vNext = 0
-		xss[0].ts.Values[i] = vNext
+	if math.IsNaN(vPrev) {
+		vPrev = 0
+		xss[0].ts.Values[i] = vPrev
 	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
-	// or are bigger than the lower bucket values.
+	// or are smaller than the lower bucket values.
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
-		if math.IsNaN(v) || vNext > v {
-			xss[j].ts.Values[i] = vNext
+		if math.IsNaN(v) || vPrev > v {
+			xss[j].ts.Values[i] = vPrev
 		} else {
-			vNext = v
+			vPrev = v
 		}
 	}
 }

app/vmselect/vmui/assets/MetricsQL-CQAtdgI6.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmselect/vmui/assets/rolldown-runtime-CNC7AqOf.js

@@ -0,0 +1 @@
+var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/assets/rolldown-runtime-Cyuzqnbw.js

@@ -1 +0,0 @@
-var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-CusQvJzs.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
+  <script type="module" crossorigin src="./assets/index-B1dXK3k7.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -30,6 +30,17 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	maxBackfillAge = flagutil.NewRetentionDuration("maxBackfillAge", "0", "The maximum allowed age for the ingested samples with historical timestamps. "+
+		"Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. "+
+		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
+		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
+	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
+		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -108,7 +119,7 @@ func DataPath() string {
 }
 
 // Init initializes vmstorage.
-func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
@@ -143,6 +154,7 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	opts := storage.OpenOptions{
 		Retention:                   retentionPeriod.Duration(),
 		FutureRetention:             futureRetention.Duration(),
+		MaxBackfillAge:              maxBackfillAge.Duration(),
 		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
 		MaxHourlySeries:             getMaxHourlySeries(),
 		MaxDailySeries:              getMaxDailySeries(),
@@ -169,6 +181,21 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
 
+	if *vmselectAddr != "" {
+		var err error
+		limits := vmselectapi.Limits{
+			MaxConcurrentRequests:         vmselectMaxConcurrentRequests,
+			MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
+			MaxQueueDuration:              vmselectMaxQueueDuration,
+			MaxQueueDurationFlagName:      "search.maxQueueDuration",
+		}
+		api := newVMStorageWithTenantID(vmStorage)
+		vmselectSrv, err = vmselectapi.NewServer(*vmselectAddr, api, limits, *vmselectDisableRPCCompression)
+		if err != nil {
+			logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
+		}
+	}
+
 	VMInsertAPI = vmStorage
 	VMSelectAPI = vmStorage
 	GetSearch = vmStorage.GetSearch
@@ -191,6 +218,8 @@ var (
 
 	// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
 	DebugFlush func()
+
+	vmselectSrv *vmselectapi.Server
 )
 
 // Stop stops the vmstorage
@@ -201,6 +230,10 @@ func Stop() {
 
 	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime := time.Now()
+
+	if vmselectSrv != nil {
+		vmselectSrv.MustStop()
+	}
 	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -443,8 +476,10 @@ func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/inmemory"}`, tm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/small"}`, tm.SmallSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/big"}`, tm.BigSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/metaindex"}`, tm.MetaindexSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/metaindex"}`, idbm.MetaindexSizeBytes)
 
 	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)

app/vmstorage/vmstorage.go

@@ -164,6 +164,10 @@ func (vms *VMStorage) IsReadOnly() bool {
 }
 
 func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	return vms.initSearch(qt, sq, marshalDefault, deadline)
+}
+
+func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, marshal marshalFunc, deadline uint64) (vmselectapi.BlockIterator, error) {
 	vms.wg.Add(1)
 
 	tr := sq.GetTimeRange()
@@ -178,6 +182,7 @@ func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery
 		return nil, fmt.Errorf("missing tag filters")
 	}
 	bi := getBlockIterator()
+	bi.marshal = marshal
 	bi.wgDone = vms.wg.Done
 	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
 	if err := bi.sr.Error(); err != nil {
@@ -198,11 +203,19 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
 	return searchQueryLimit
 }
 
+type marshalFunc func(dst []byte, src *storage.MetricBlock) []byte
+
+// marshalDefault is the default implementation of the MetricBlock marshaling.
+func marshalDefault(dst []byte, src *storage.MetricBlock) []byte {
+	return src.Marshal(dst)
+}
+
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
-	sr     storage.Search
-	mb     storage.MetricBlock
-	wgDone func()
+	sr      storage.Search
+	mb      storage.MetricBlock
+	marshal marshalFunc
+	wgDone  func()
 }
 
 var blockIteratorsPool sync.Pool
@@ -228,10 +241,10 @@ func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	if !bi.sr.NextMetricBlock() {
 		return dst, false
 	}
-	mb := bi.mb
+	mb := &bi.mb
 	mb.MetricName = bi.sr.MetricBlockRef.MetricName
 	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	dst = mb.Marshal(dst[:0])
+	dst = bi.marshal(dst[:0], mb)
 	return dst, true
 }
 

app/vmstorage/vmstorage_with_tenant_id.go

@@ -0,0 +1,282 @@
+package vmstorage
+
+import (
+	"flag"
+	"fmt"
+	"math"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	accountID = flag.Uint64("accountID", 0, "The accountID of the stored data")
+	projectID = flag.Uint64("projectID", 0, "The projectID of the stored data")
+)
+
+func newVMStorageWithTenantID(vms *VMStorage) *VMStorageWithTenantID {
+	if *accountID > math.MaxUint32 {
+		logger.Fatalf("-accountID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *accountID)
+	}
+	if *projectID > math.MaxUint32 {
+		logger.Fatalf("-projectID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *projectID)
+	}
+	return &VMStorageWithTenantID{
+		vms:       vms,
+		accountID: uint32(*accountID),
+		projectID: uint32(*projectID),
+	}
+}
+
+// VMStorageWithTenantID is a thin wrapper around VMStorage type that overrides
+// its methods to properly serve requests coming from a vmselect (require
+// tenantID).
+//
+// A new instance of this type should be created using
+// newVMStorageWithTenantID(). The created instance does not require closing.
+// The instance also does not take ownership of vms and it is the responsibility
+// of the caller to close vms.
+type VMStorageWithTenantID struct {
+	vms *VMStorage
+
+	accountID uint32
+	projectID uint32
+}
+
+// InitSearch initializes a storage search for a request initiated by a
+// vmselect.
+//
+// The search is initialized only if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+// Otherwise, the method returns an interator that will return no data.
+//
+// The method also overrides the data format of the data returned by the
+// iterator by prepending accountID and projectID bytes to the metric name and
+// the data block (a format used in vmcluster).
+func (vmst *VMStorageWithTenantID) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return emptyBI, nil
+	}
+	return vmst.vms.initSearch(qt, sq, vmst.marshalMetricBlock, deadline)
+}
+
+var emptyBI = &emptyBlockIterator{}
+
+// emptyBlockIterator is an implementation of vmselectapi.BlockIterator that
+// always returns no data.
+type emptyBlockIterator struct{}
+
+func (*emptyBlockIterator) MustClose() {}
+
+func (*emptyBlockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	return dst, false
+}
+
+func (*emptyBlockIterator) Error() error {
+	return nil
+}
+
+// marshalMetricBlock serializes a metric block in the format expected by
+// vmselect.
+//
+// vmselect expects metric names and data blocks to have the tenantID but
+// vmsingle does not have it. Therefore the tenantID needs to be included to
+// every metric name and block.
+func (vmst *VMStorageWithTenantID) marshalMetricBlock(dst []byte, src *storage.MetricBlock) []byte {
+	// Marshal metric name:
+	// 1. Marshal metric name length + accountID length + projectID length (in
+	//    bytes).
+	// 2. append accountID and projectID bytes
+	// 3. Finally append metric name bytes
+	dst = encoding.MarshalVarUint64(dst, uint64(len(src.MetricName))+8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = append(dst, src.MetricName...)
+
+	// Marshal data block.
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = storage.MarshalBlock(dst, &src.Block)
+
+	return dst
+}
+
+// SearchMetricNames searches the storage for metric names that match the query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+//
+// Found metric names are prepended with accountID and projectID bytes (a format
+// used in vmcluster).
+func (vmst *VMStorageWithTenantID) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+
+	metricNames, err := vmst.vms.SearchMetricNames(qt, sq, deadline)
+	if err != nil {
+		return nil, err
+	}
+
+	// vmselect expects metric names to have the tenantID but vmsingle does not
+	// have it. Therefore the tenantID needs to be prepended to every metric
+	// name.
+	dst := make([]byte, 0, 8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	tenantID := string(dst)
+
+	for i, metricName := range metricNames {
+		metricNames[i] = tenantID + metricName
+	}
+	return metricNames, nil
+}
+
+// LabelValues searches the storage for label values that match the query and
+// correspond to a label whose name is `labelName`. The returned result
+// will contain not more than `maxLabelValues`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
+}
+
+// TagValueSuffixes searches the storage for Graphite tag value suffixes. The
+// returned result will contain not more than `maxSuffixes`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return nil, nil
+	}
+	return vmst.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+}
+
+// LabelNames searches the storage for label names that match the query.
+// The returned result will contain not more than `maxLabelNames`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelNames(qt, sq, maxLabelNames, deadline)
+}
+
+// SeriesCount returns the total number of metrics stored in the database.
+//
+// The method may return inflated numbers. How inflated the count depends
+// on the churn rate and the retention period. For example, if a metric lasts
+// for 2 months, it will be counted twice.
+//
+// The method also counts the deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return 0.
+func (vmst *VMStorageWithTenantID) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return 0, nil
+	}
+	return vmst.vms.SeriesCount(qt, accountID, projectID, deadline)
+}
+
+// Tenants returns just one tenant consisting of the -accountID and -projectID
+// flag values.
+func (vmst *VMStorageWithTenantID) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	tenantID := fmt.Sprintf("%d:%d", vmst.accountID, vmst.projectID)
+	return []string{tenantID}, nil
+}
+
+// TSDBStatus retrieves the status for metrics that match to the search query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return empty
+// status.
+func (vmst *VMStorageWithTenantID) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return &storage.TSDBStatus{}, nil
+	}
+	return vmst.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
+}
+
+// DeleteSeries marks as deleted metrics that match the search query.
+// The method returns the number of deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, no metrics will be deleted
+// and the method will return 0.
+func (vmst *VMStorageWithTenantID) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return 0, nil
+	}
+	return vmst.vms.DeleteSeries(qt, sq, deadline)
+}
+
+// RegisterMetricNames registers metric names in the index, the sample values
+// and timestamps are ignored.
+func (vmst *VMStorageWithTenantID) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
+	return vmst.vms.RegisterMetricNames(qt, mrs, deadline)
+}
+
+// GetMetricNamesUsageStats retrieves the usage stats for metrics whose name
+// matches the pattern.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return metricnamestats.StatsResult{}, nil
+	}
+	return vmst.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
+}
+
+// ResetMetricNamesUsageStats resets the metric name usage stats.
+func (vmst *VMStorageWithTenantID) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
+	return vmst.vms.ResetMetricNamesUsageStats(qt, deadline)
+}
+
+// GetMetadataRecords retrieves the metadata for the metricName.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return nil, nil
+	}
+	return vmst.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
+}
+
+// hasValidTenantID returns true if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) hasValidTenantID(sq *storage.SearchQuery) bool {
+	return sq.IsMultiTenant || vmst.isValidTenantID(sq.AccountID, sq.ProjectID)
+}
+
+// isValidTenantToken returns true if the TenantToken is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantToken(tt *storage.TenantToken) bool {
+	return tt == nil || vmst.isValidTenantID(tt.AccountID, tt.ProjectID)
+}
+
+// isValidTenantID returns true if the accountID and projectID match -accountID
+// and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantID(accountID, projectID uint32) bool {
+	return accountID == vmst.accountID && projectID == vmst.projectID
+}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.4 AS build-web-stage
+FROM golang:1.26.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/assets/favicon.svg

@@ -0,0 +1,5 @@
+<svg width="48" height="48" fill="#020202" xmlns="http://www.w3.org/2000/svg">
+    <path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
+    <path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
+    <path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
+</svg>

app/vmui/packages/vmui/index.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg"/>
-  <link rel="apple-touch-icon" href="/favicon.svg"/>
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
+  <link id="favicon" rel="icon" href="/assets/favicon.svg" />
+  <link rel="apple-touch-icon" href="/assets/favicon.svg" />
+  <link id="mask-icon" rel="mask-icon" href="/assets/favicon.svg?no-inline" color="#000000">
 
   <meta name="robots" content="noindex">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.21",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.5",
-    "preact": "^10.29.2",
-    "qs": "^6.15.2",
+    "marked": "^18.0.6",
+    "preact": "^10.29.7",
+    "qs": "^6.15.3",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.17.0",
+    "react-router-dom": "^7.18.1",
     "uplot": "^1.6.32",
-    "vite": "^8.0.16",
+    "vite": "^8.1.4",
     "web-vitals": "^5.3.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.5",
+    "@eslint/eslintrc": "^3.3.6",
     "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.5",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.9.2",
+    "@types/node": "^26.1.1",
     "@types/qs": "^6.15.1",
     "@types/react": "^19.2.17",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.61.0",
-    "@typescript-eslint/parser": "^8.61.0",
+    "@typescript-eslint/eslint-plugin": "^8.64.0",
+    "@typescript-eslint/parser": "^8.64.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "globals": "^17.6.0",
-    "http-proxy-middleware": "^4.1.0",
+    "globals": "^17.7.0",
+    "http-proxy-middleware": "^4.2.0",
     "jsdom": "^29.1.1",
-    "postcss": "^8.5.15",
+    "postcss": "^8.5.19",
     "sass-embedded": "^1.100.0",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.10"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/public/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vmui/packages/vmui/public/manifest.json

@@ -3,7 +3,7 @@
   "name": "vmui",
   "icons": [
     {
-      "src": "favicon.svg",
+      "src": "./assets/favicon.svg",
       "sizes": "any",
       "type": "image/svg+xml"
     }

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/BrowserTabController.tsx

@@ -0,0 +1,61 @@
+import { FC, useMemo } from "preact/compat";
+import "./style.scss";
+import { createFaviconUrl } from "../../../../utils/favicon";
+import classNames from "classnames";
+import { useBrowserTabSync } from "./hooks/useBrowserTabSync";
+import { CloseIcon } from "../../../Main/Icons";
+import { faviconColors } from "../../../../constants/faviconColors";
+
+const BrowserTabController: FC = () => {
+  const { faviconColor, changeFaviconColor } = useBrowserTabSync();
+
+  const faviconUrl = useMemo(() => {
+    return createFaviconUrl(faviconColor);
+  }, [faviconColor]);
+
+  const createHandlerClick = (color?: string) => () => {
+    changeFaviconColor(color);
+  };
+
+  return (
+    <div className="vm-browser-tab-controller">
+      <p className="vm-server-configurator__title">Favicon color</p>
+
+      <div className="vm-browser-tab-controller-palette">
+        <div className="vm-browser-tab-controller-palette-list">
+          <button
+            className="vm-browser-tab-controller-palette-list__item vm-browser-tab-controller-palette-list__item_reset"
+            type="button"
+            onClick={createHandlerClick()}
+            aria-label="Reset favicon color"
+          >
+            <CloseIcon />
+          </button>
+
+          {faviconColors.map(color => (
+            <button
+              className={classNames({
+                "vm-browser-tab-controller-palette-list__item": true,
+                "vm-browser-tab-controller-palette-list__item_selected": faviconColor === color
+              })}
+              key={color}
+              type="button"
+              style={{ color }}
+              onClick={createHandlerClick(color)}
+              aria-label={`Set favicon color to ${color}`}
+              aria-pressed={faviconColor === color}
+            />
+          ))}
+        </div>
+
+        <img
+          className="vm-browser-tab-controller-palette__preview"
+          src={faviconUrl}
+          alt="Favicon preview"
+        />
+      </div>
+    </div>
+  );
+};
+
+export default BrowserTabController;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/hooks/useBrowserTabSync.ts

@@ -0,0 +1,41 @@
+import useEventListener from "../../../../../hooks/useEventListener";
+import { useEffect, useState } from "preact/compat";
+import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../../utils/storage";
+import { getFaviconStorageKey, updateFaviconColor } from "../../../../../utils/favicon";
+
+const storageKey = `FAVICON_COLOR:${getFaviconStorageKey()}` as const;
+
+const getColorFromStorage = () => {
+  return getFromStorage(storageKey) as string | undefined;
+};
+
+export const useBrowserTabSync = () => {
+  const [faviconColor, setFaviconColor] = useState(getColorFromStorage);
+
+  const handleUpdateColor = () => {
+    setFaviconColor(getColorFromStorage());
+  };
+
+  const changeFaviconColor = (color?: string) => {
+    if (color) {
+      saveToStorage(storageKey, color);
+    } else {
+      removeFromStorage([storageKey]);
+    }
+  };
+
+  useEffect(() => {
+    handleUpdateColor();
+  }, []);
+
+  useEffect(() => {
+    updateFaviconColor(faviconColor);
+  }, [faviconColor]);
+
+  useEventListener("storage", handleUpdateColor);
+
+  return {
+    faviconColor,
+    changeFaviconColor,
+  };
+};

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/style.scss

@@ -0,0 +1,71 @@
+@use "src/styles/variables" as *;
+
+$color-item-size: 28px;
+$outline-width: 2px;
+$outline-offset: 2px;
+$outline-space: $outline-width + $outline-offset;
+
+.vm-browser-tab-controller {
+  .vm-server-configurator__title {
+    padding-bottom: 2px;
+  }
+
+  &-palette {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: calc($padding-large * 2);
+
+    &-list {
+      display: flex;
+      align-items: center;
+      justify-content: flex-start;
+      flex-wrap: wrap;
+      gap: $padding-small;
+
+      &__item {
+        width: $color-item-size;
+        height: $color-item-size;
+        aspect-ratio: 1;
+        border-radius: 50%;
+        background-color: currentColor;
+        cursor: pointer;
+        transition-property: transform;
+        transition-duration: 0.15s;
+        transition-timing-function: linear;
+
+        &:hover {
+          transform: scale(1.1);
+        }
+
+        &:focus-visible {
+          transform: scale(1.1);
+        }
+
+        &_reset {
+          display: flex;
+          align-items: center;
+          justify-content: center;
+          background: transparent;
+          border: $border-divider;
+          color: $color-text-disabled;
+          padding: calc($padding-small / 2);
+        }
+
+        &_selected {
+          width: $color-item-size - 2 * $outline-space;
+          height: $color-item-size - 2 * $outline-space;
+          margin: $outline-space;
+          outline: $outline-width solid currentColor;
+          outline-offset: $outline-offset;
+          pointer-events: none;
+        }
+      }
+    }
+
+    &__preview {
+      width: $color-item-size;
+      height: auto;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -8,10 +8,12 @@ import Tooltip from "../../Main/Tooltip/Tooltip";
 import LimitsConfigurator from "./LimitsConfigurator/LimitsConfigurator";
 import { getAppModeEnable } from "../../../utils/app-mode";
 import classNames from "classnames";
-import Timezones from "./Timezones/Timezones";
+import TimezonesPicker from "./Timezones/TimezonesPicker";
 import ThemeControl from "../ThemeControl/ThemeControl";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
+import BrowserTabController from "./BrowserTabController/BrowserTabController";
+import LegendCollapseController from "./LegendCollapseController/LegendCollapseController";
 
 const title = "Settings";
 
@@ -26,7 +28,6 @@ const GlobalSettings: FC = () => {
 
   const serverSettingRef = useRef<ChildComponentHandle>(null);
   const limitsSettingRef = useRef<ChildComponentHandle>(null);
-  const timezoneSettingRef = useRef<ChildComponentHandle>(null);
 
   const {
     value: open,
@@ -37,7 +38,6 @@ const GlobalSettings: FC = () => {
   const handleApply = () => {
     serverSettingRef.current && serverSettingRef.current.handleApply();
     limitsSettingRef.current && limitsSettingRef.current.handleApply();
-    timezoneSettingRef.current && timezoneSettingRef.current.handleApply();
     handleClose();
   };
 
@@ -49,6 +49,10 @@ const GlobalSettings: FC = () => {
         onClose={handleClose}
       />
     },
+    {
+      show: true,
+      component: <TimezonesPicker/>
+    },
     {
       show: true,
       component: <LimitsConfigurator
@@ -58,12 +62,16 @@ const GlobalSettings: FC = () => {
     },
     {
       show: true,
-      component: <Timezones ref={timezoneSettingRef}/>
+      component: <LegendCollapseController/>
     },
     {
       show: !appModeEnable,
       component: <ThemeControl/>
-    }
+    },
+    {
+      show: true,
+      component: <BrowserTabController/>
+    },
   ].filter(control => control.show);
 
   return <>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LegendCollapseController/LegendCollapseController.tsx

@@ -0,0 +1,30 @@
+import { FC, useEffect, useState } from "preact/compat";
+import { getFromStorage, saveToStorage } from "../../../../utils/storage";
+import Switch from "../../../Main/Switch/Switch";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import "./style.scss";
+
+const LegendCollapseController: FC = () => {
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
+  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
+  useEffect(() => {
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
+  }, [legendCollapse]);
+
+  return (
+    <div className="vm-legend-collapse-controller">
+      <Switch
+        fullWidth
+        color="neutral"
+        value={legendCollapse}
+        onChange={setLegendCollapse}
+        label={<span className="vm-server-configurator__title">Auto-collapse legend</span>}
+      />
+      <span className="vm-legend-collapse-controller__description">
+        Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
+      </span>
+    </div>);
+};
+
+export default LegendCollapseController;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LegendCollapseController/style.scss

@@ -0,0 +1,19 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-collapse-controller {
+  background-color: $color-hover-black;
+  border-radius: $border-radius-medium;
+  padding: $padding-large;
+  border: $border-divider;
+
+  .vm-graph-settings-row__label {
+    margin: 0;
+  }
+
+  &__description {
+    padding-top: $padding-global;
+    font-size: $font-size-small;
+    line-height: 1.3;
+    text-wrap: pretty;
+  }
+}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -1,23 +1,21 @@
-import { forwardRef, useCallback, useEffect, useImperativeHandle, useState } from "preact/compat";
+import { forwardRef, useCallback, useImperativeHandle, useState } from "preact/compat";
 import { DisplayType, ErrorTypes } from "../../../../types";
 import TextField from "../../../Main/TextField/TextField";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import { InfoIcon, RestartIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
-import { DEFAULT_MAX_SERIES, LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../../state/customPanel/CustomPanelStateContext";
-import Switch from "../../../Main/Switch/Switch";
-import { getFromStorage, saveToStorage } from "../../../../utils/storage";
 
 interface ServerConfiguratorProps {
-  onClose: () => void
+  onClose: () => void;
 }
 
-const fields: {label: string, type: DisplayType}[] = [
+const fields: { label: string, type: DisplayType }[] = [
   { label: "Graph", type: DisplayType.chart },
   { label: "JSON", type: DisplayType.code },
   { label: "Table", type: DisplayType.table }
@@ -29,8 +27,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
-  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
-  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
 
   const [limits, setLimits] = useState(seriesLimits);
   const [error, setError] = useState({
@@ -43,7 +40,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     setLimits(DEFAULT_MAX_SERIES);
   };
 
-  const createChangeHandler = (type: DisplayType) =>  (val: string) => {
+  const createChangeHandler = (type: DisplayType) => (val: string) => {
     const value = val || "";
     setError(prev => ({ ...prev, [type]: +value < 0 ? ErrorTypes.positiveNumber : "" }));
     setLimits({
@@ -57,10 +54,6 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     onClose();
   }, [limits]);
 
-  useEffect(() => {
-    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
-  }, [legendCollapse]);
-
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);
 
   return (
@@ -106,19 +99,6 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
           </div>
         ))}
       </div>
-
-      <div className="vm-graph-settings-row">
-        <span className="vm-graph-settings-row__label">Auto-collapse legend</span>
-        <Switch
-          value={legendCollapse}
-          onChange={setLegendCollapse}
-          label={legendCollapse ? "Enabled" : "Disabled"}
-          fullWidth={isMobile}
-        />
-        <span className="vm-legend-configs-item__info">
-          Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
-        </span>
-      </div>
     </div>
   );
 });

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/Timezones.tsx

@@ -1,183 +0,0 @@
-import { FC, forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "preact/compat";
-import { getBrowserTimezone, getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
-import { ArrowDropDownIcon } from "../../../Main/Icons";
-import classNames from "classnames";
-import Popper from "../../../Main/Popper/Popper";
-import Accordion from "../../../Main/Accordion/Accordion";
-import TextField from "../../../Main/TextField/TextField";
-import { Timezone } from "../../../../types";
-import "./style.scss";
-import useDeviceDetect from "../../../../hooks/useDeviceDetect";
-import useBoolean from "../../../../hooks/useBoolean";
-import WarningTimezone from "./WarningTimezone";
-import { useTimeDispatch, useTimeState } from "../../../../state/time/TimeStateContext";
-
-interface PinnedTimezone extends Timezone {
-  title: string;
-  isInvalid?: boolean;
-}
-
-const browserTimezone = getBrowserTimezone();
-
-const Timezones: FC = forwardRef((props, ref) => {
-  const { isMobile } = useDeviceDetect();
-  const timezones = getTimezoneList();
-
-  const { timezone: stateTimezone, defaultTimezone } = useTimeState();
-  const timeDispatch = useTimeDispatch();
-
-  const [timezone, setTimezone] = useState(stateTimezone);
-  const [search, setSearch] = useState("");
-  const targetRef = useRef<HTMLDivElement>(null);
-
-  const {
-    value: openList,
-    toggle: toggleOpenList,
-    setFalse: handleCloseList,
-  } = useBoolean(false);
-
-  const pinnedTimezones = useMemo(() => [
-    {
-      title: `Default time (${defaultTimezone})`,
-      region: defaultTimezone,
-      utc: defaultTimezone ? getUTCByTimezone(defaultTimezone) : "UTC"
-    },
-    {
-      title: browserTimezone.title,
-      region: browserTimezone.region,
-      utc: getUTCByTimezone(browserTimezone.region),
-      isInvalid: !browserTimezone.isValid
-    },
-    {
-      title: "UTC (Coordinated Universal Time)",
-      region: "UTC",
-      utc: "UTC"
-    },
-  ].filter(t => t.region) as PinnedTimezone[], [defaultTimezone]);
-
-  const searchTimezones = useMemo(() => {
-    if (!search) return timezones;
-    try {
-      return getTimezoneList(search);
-    } catch (e) {
-      return {};
-    }
-  }, [search, timezones]);
-
-  const timezonesGroups = useMemo(() => Object.keys(searchTimezones), [searchTimezones]);
-
-  const activeTimezone = useMemo(() => ({
-    region: timezone,
-    utc: getUTCByTimezone(timezone)
-  }), [timezone]);
-
-  const handleChangeSearch = (val: string) => {
-    setSearch(val);
-  };
-
-  const handleSetTimezone = (val: Timezone) => {
-    setTimezone(val.region);
-    setSearch("");
-    handleCloseList();
-  };
-
-  const createHandlerSetTimezone = (val: Timezone) => () => {
-    handleSetTimezone(val);
-  };
-
-  useEffect(() => {
-    setTimezone(stateTimezone);
-  }, [stateTimezone]);
-
-  useImperativeHandle(ref, () => ({
-    handleApply: () => {
-      timeDispatch({ type: "SET_TIMEZONE", payload: timezone });
-    }
-  }), [timezone]);
-
-  return (
-    <div className="vm-timezones">
-      <div className="vm-server-configurator__title">
-        Time zone
-      </div>
-      <div
-        className="vm-timezones-item vm-timezones-item_selected"
-        onClick={toggleOpenList}
-        ref={targetRef}
-      >
-        <div className="vm-timezones-item__title">{activeTimezone.region}</div>
-        <div className="vm-timezones-item__utc">{activeTimezone.utc}</div>
-        <div
-          className={classNames({
-            "vm-timezones-item__icon": true,
-            "vm-timezones-item__icon_open": openList
-          })}
-        >
-          <ArrowDropDownIcon/>
-        </div>
-      </div>
-      <Popper
-        open={openList}
-        buttonRef={targetRef}
-        placement="bottom-left"
-        onClose={handleCloseList}
-        fullWidth
-        title={isMobile ? "Time zone" : undefined}
-      >
-        <div
-          className={classNames({
-            "vm-timezones-list": true,
-            "vm-timezones-list_mobile": isMobile,
-          })}
-        >
-          <div className="vm-timezones-list-header">
-            <div className="vm-timezones-list-header__search">
-              <TextField
-                autofocus
-                label="Search"
-                value={search}
-                onChange={handleChangeSearch}
-              />
-            </div>
-            {pinnedTimezones.map((t, i) => t && (
-              <div
-                key={`${i}_${t.region}`}
-                className="vm-timezones-item vm-timezones-list-group-options__item"
-                onClick={createHandlerSetTimezone(t)}
-              >
-                <div className="vm-timezones-item__title">{t.title}{t.isInvalid && <WarningTimezone/>}</div>
-                <div className="vm-timezones-item__utc">{t.utc}</div>
-              </div>
-            ))}
-          </div>
-          {timezonesGroups.map(t => (
-            <div
-              className="vm-timezones-list-group"
-              key={t}
-            >
-              <Accordion
-                defaultExpanded={true}
-                title={<div className="vm-timezones-list-group__title">{t}</div>}
-              >
-                <div className="vm-timezones-list-group-options">
-                  {searchTimezones[t] && searchTimezones[t].map(item => (
-                    <div
-                      className="vm-timezones-item vm-timezones-list-group-options__item"
-                      onClick={createHandlerSetTimezone(item)}
-                      key={item.search}
-                    >
-                      <div className="vm-timezones-item__title">{item.region}</div>
-                      <div className="vm-timezones-item__utc">{item.utc}</div>
-                    </div>
-                  ))}
-                </div>
-              </Accordion>
-            </div>
-          ))}
-        </div>
-      </Popper>
-    </div>
-  );
-});
-
-export default Timezones;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/TimezonesList.tsx

@@ -0,0 +1,129 @@
+import { FC, useMemo, useState } from "preact/compat";
+import { getBrowserTimezone, getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
+import classNames from "classnames";
+import Accordion from "../../../Main/Accordion/Accordion";
+import TextField from "../../../Main/TextField/TextField";
+import { Timezone } from "../../../../types";
+import "./style.scss";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import WarningTimezone from "./WarningTimezone";
+import { useTimeState } from "../../../../state/time/TimeStateContext";
+
+interface PinnedTimezone extends Timezone {
+  title: string;
+  isInvalid?: boolean;
+}
+
+type Props = {
+  onChange: (tz: Timezone) => void;
+}
+
+const browserTimezone = getBrowserTimezone();
+
+const TimezonesList: FC<Props> = ({ onChange }) => {
+  const { isMobile } = useDeviceDetect();
+  const { defaultTimezone } = useTimeState();
+
+  const timezones = useMemo(() => getTimezoneList(), []);
+
+  const [search, setSearch] = useState("");
+
+  const pinnedTimezones = useMemo(() => [
+    {
+      title: `Default time (${defaultTimezone})`,
+      region: defaultTimezone,
+      utc: defaultTimezone ? getUTCByTimezone(defaultTimezone) : "UTC"
+    },
+    {
+      title: browserTimezone.title,
+      region: browserTimezone.region,
+      utc: getUTCByTimezone(browserTimezone.region),
+      isInvalid: !browserTimezone.isValid
+    },
+    {
+      title: "UTC (Coordinated Universal Time)",
+      region: "UTC",
+      utc: "UTC"
+    },
+  ].filter(t => t.region) as PinnedTimezone[], [defaultTimezone]);
+
+  const searchTimezones = useMemo(() => {
+    if (!search) return timezones;
+    try {
+      return getTimezoneList(search);
+    } catch (e) {
+      return {};
+    }
+  }, [search, timezones]);
+
+  const timezonesGroups = useMemo(() => Object.keys(searchTimezones), [searchTimezones]);
+
+  const handleChangeSearch = (val: string) => {
+    setSearch(val);
+  };
+
+  const handleSetTimezone = (tz: Timezone) => {
+    onChange(tz);
+    setSearch("");
+  };
+
+  const createHandlerSetTimezone = (val: Timezone) => () => {
+    handleSetTimezone(val);
+  };
+
+  return (
+    <div
+      className={classNames({
+        "vm-list": true,
+        "vm-timezones-list": true,
+        "vm-timezones-list_mobile": isMobile,
+      })}
+    >
+      <div className="vm-timezones-list-header">
+        <div className="vm-timezones-list-header__search">
+          <TextField
+            label="Search"
+            value={search}
+            onChange={handleChangeSearch}
+          />
+        </div>
+      </div>
+      {pinnedTimezones.map((t, i) => t && (
+        <div
+          key={`${i}_${t.region}`}
+          className="vm-list-item vm-timezones-item vm-timezones-list-group-options__item"
+          onClick={createHandlerSetTimezone(t)}
+        >
+          <div className="vm-timezones-item__title">{t.title}{t.isInvalid && <WarningTimezone/>}</div>
+          <div className="vm-timezones-item__utc">{t.utc}</div>
+        </div>
+      ))}
+      {timezonesGroups.map(t => (
+        <div
+          className="vm-timezones-list-group"
+          key={t}
+        >
+          <Accordion
+            defaultExpanded={true}
+            title={<div className="vm-timezones-list-group__title">{t}</div>}
+          >
+            <div className="vm-timezones-list-group-options">
+              {searchTimezones[t] && searchTimezones[t].map(item => (
+                <div
+                  className="vm-list-item vm-timezones-item vm-timezones-list-group-options__item"
+                  onClick={createHandlerSetTimezone(item)}
+                  key={item.search}
+                >
+                  <div className="vm-timezones-item__title">{item.region}</div>
+                  <div className="vm-timezones-item__utc">{item.utc}</div>
+                </div>
+              ))}
+            </div>
+          </Accordion>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+export default TimezonesList;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/TimezonesPicker.tsx

@@ -0,0 +1,71 @@
+import { FC, useMemo, useRef } from "preact/compat";
+import { getUTCByTimezone } from "../../../../utils/time";
+import { ArrowDropDownIcon } from "../../../Main/Icons";
+import classNames from "classnames";
+import { Timezone } from "../../../../types";
+import "./style.scss";
+import useBoolean from "../../../../hooks/useBoolean";
+import { useTimeDispatch, useTimeState } from "../../../../state/time/TimeStateContext";
+import TimezonesList from "./TimezonesList";
+import Popper from "../../../Main/Popper/Popper";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+
+const TimezonesPicker: FC = () => {
+  const { isMobile } = useDeviceDetect();
+  const { timezone: stateTimezone } = useTimeState();
+  const timeDispatch = useTimeDispatch();
+
+  const triggerRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: isOpenList,
+    toggle: toggleOpenList,
+    setFalse: handleCloseList,
+  } = useBoolean(false);
+
+  const activeTimezone = useMemo(() => ({
+    region: stateTimezone,
+    utc: getUTCByTimezone(stateTimezone)
+  }), [stateTimezone]);
+
+  const handleSetTimezone = (tz: Timezone) => {
+    timeDispatch({ type: "SET_TIMEZONE", payload: tz.region });
+    handleCloseList();
+  };
+
+  return (
+    <div className="vm-timezones">
+      <div className="vm-server-configurator__title">
+        Time zone
+      </div>
+      <div
+        className="vm-timezones-item vm-timezones-item_selected"
+        onClick={toggleOpenList}
+        ref={triggerRef}
+      >
+        <div className="vm-timezones-item__title">{activeTimezone.region}</div>
+        <div className="vm-timezones-item__utc">{activeTimezone.utc}</div>
+        <div
+          className={classNames({
+            "vm-timezones-item__icon": true,
+            "vm-timezones-item__icon_open": isOpenList
+          })}
+        >
+          <ArrowDropDownIcon/>
+        </div>
+      </div>
+      <Popper
+        open={isOpenList}
+        buttonRef={triggerRef}
+        placement="bottom-left"
+        onClose={handleCloseList}
+        fullWidth
+        title={isMobile ? "Time zone" : undefined}
+      >
+        <TimezonesList onChange={handleSetTimezone}/>
+      </Popper>
+    </div>
+  );
+};
+
+export default TimezonesPicker;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/style.scss

@@ -16,6 +16,7 @@
     }
 
     &__title {
+      flex-grow: 1;
       display: flex;
       align-items: center;
       gap: $padding-small;
@@ -34,6 +35,7 @@
       background-color: $color-hover-black;
       padding: calc($padding-small/2);
       border-radius: $border-radius-small;
+      font-size: $font-size-small;
     }
 
     &__icon {
@@ -54,9 +56,11 @@
   }
 
   &-list {
+    padding-top: 0;
     max-height: 300px;
     background-color: $color-background-block;
     border-radius: $border-radius-medium;
+    font-size: $font-size-small;
     overflow: auto;
 
     &_mobile {
@@ -72,10 +76,9 @@
       top: 0;
       background-color: $color-background-block;
       z-index: 2;
-      border-bottom: $border-divider;
 
       &__search {
-        padding: $padding-small;
+        padding: $padding-small $padding-small calc($padding-small / 2);
       }
     }
 
@@ -91,6 +94,7 @@
         font-weight: bold;
         color: $color-text-secondary;
         padding: $padding-small $padding-global;
+        font-size: $font-size-small;
       }
 
       &-options {
@@ -98,7 +102,7 @@
         align-items: flex-start;
 
         &__item {
-          padding: $padding-small $padding-global;
+          padding: calc($padding-small / 2) $padding-global;
           transition: background-color 200ms ease;
 
           &:hover {

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -4,9 +4,9 @@
   display: flex;
   flex-direction: column;
   align-items: center;
-  gap: $padding-large;
+  gap: calc($padding-global * 2);
   width: 600px;
-  padding-bottom: $padding-medium;
+  padding-inline: $padding-large;
 
   &_mobile {
     grid-auto-rows: min-content;
@@ -62,6 +62,7 @@
     justify-content: flex-end;
     gap: $padding-small;
     width: 100%;
+    padding-block: $padding-global;
   }
 
   &_mobile &-footer {

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -22,12 +22,10 @@ const StepConfigurator: FC = () => {
   const { isMobile } = useDeviceDetect();
 
   const { customStep: value, isHistogram } = useGraphState();
-  const { period: { step, end, start } } = useTimeState();
+  const { period: { end, start } } = useTimeState();
   const graphDispatch = useGraphDispatch();
   const { displayType } = useCustomPanelState();
 
-  const prevDuration = usePrevious(end - start);
-
   const defaultStep = useMemo(() => {
     return getStepFromDuration(end - start, isHistogram, displayType);
   }, [end, start, isHistogram, displayType]);
@@ -106,16 +104,14 @@ const StepConfigurator: FC = () => {
   }, [defaultStep]);
 
   useEffect(() => {
-    const dur = end - start;
-    if (dur === prevDuration || !prevDuration || value !== prevDefaultStep) return;
-    if (defaultStep) {
-      handleApply(defaultStep);
-    }
-  }, [prevDuration, defaultStep]);
+    if (!prevDefaultStep) return;
+    if (value !== prevDefaultStep) return;
+    if (value === defaultStep) return;
 
-  useEffect(() => {
-    if (step === value || step === defaultStep) handleApply(defaultStep);
-  }, [isHistogram, displayType]);
+    graphDispatch({ type: "SET_CUSTOM_STEP", payload: defaultStep });
+    setCustomStep(defaultStep);
+    setError("");
+  }, [defaultStep, prevDefaultStep, value, graphDispatch]);
 
   return (
     <div
@@ -183,7 +179,7 @@ const StepConfigurator: FC = () => {
           <div className="vm-step-control-popper-info">
             <p>
               <code>step</code> - the <Hyperlink
-                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
+                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations"
                 text="interval"
               /> between datapoints, which must be returned from the range query.
               The <code>query</code> is executed