all: update Docker image tag for VictoriaMetrics components from v1.97.1 to v1.98.0

See https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.98.0

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/workflows/check-licenses.yml

@@ -25,7 +25,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
         if: ${{ matrix.language == 'go' }}
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -75,7 +75,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +86,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +100,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -102,7 +102,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -115,6 +115,6 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.txt

.gitignore

@@ -22,3 +22,4 @@ Gemfile.lock
 /_site
 _site
 *.tmp
+/docs/.jekyll-metadata

Makefile

@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
+MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
@@ -178,7 +178,8 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64
+	victoria-metrics-openbsd-amd64 \
+	victoria-metrics-windows-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
@@ -465,7 +466,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.20
+	go mod tidy -compat=1.22
 	go mod vendor
 
 app-local:

SECURITY.md

@@ -5,8 +5,8 @@
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
+| v1.97.x LTS release | :white_check_mark: |
 | v1.93.x LTS release | :white_check_mark: |
-| v1.87.x LTS release | :white_check_mark: |
 | other releases  | :x:                |
 
 ## Reporting a Vulnerability

app/victoria-logs/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -22,11 +21,10 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -34,18 +32,21 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
 
-	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":9428"}
+	}
+	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -53,9 +54,9 @@ func main() {
 	logger.Infof("received signal %s", sig)
 	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-metrics/main.go

@@ -26,8 +26,8 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
@@ -66,7 +66,11 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8428"}
+	}
+	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
@@ -76,7 +80,7 @@ func main() {
 
 	startSelfScraper()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -86,9 +90,9 @@ func main() {
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -124,6 +128,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -180,7 +181,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddr, false, requestHandler)
+	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -226,7 +227,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(*httpListenAddrs); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -413,6 +414,7 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
+
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -497,3 +499,73 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
+
+func TestImportJSONLines(t *testing.T) {
+	f := func(labelsCount, labelLen int) {
+		t.Helper()
+
+		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
+		line := generateJSONLine(labelsCount, labelLen)
+		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
+		}
+		if resp.StatusCode != 204 {
+			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
+		}
+	}
+
+	// labels with various lengths
+	for i := 0; i < 500; i++ {
+		f(10, i*5)
+	}
+
+	// Too many labels
+	f(1000, 100)
+
+	// Too long labels
+	f(1, 100_000)
+	f(10, 100_000)
+	f(10, 10_000)
+}
+
+func generateJSONLine(labelsCount, labelLen int) string {
+	m := make(map[string]string, labelsCount)
+	m["__name__"] = generateSizedRandomString(labelLen)
+	for j := 1; j < labelsCount; j++ {
+		labelName := generateSizedRandomString(labelLen)
+		labelValue := generateSizedRandomString(labelLen)
+		m[labelName] = labelValue
+	}
+
+	type jsonLine struct {
+		Metric     map[string]string `json:"metric"`
+		Values     []float64         `json:"values"`
+		Timestamps []int64           `json:"timestamps"`
+	}
+	line := &jsonLine{
+		Metric:     m,
+		Values:     []float64{1.34},
+		Timestamps: []int64{time.Now().UnixNano() / 1e6},
+	}
+	data, err := json.Marshal(&line)
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal JSON: %w", err))
+	}
+	data = append(data, '\n')
+	return string(data)
+}
+
+const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
+
+func generateSizedRandomString(size int) string {
+	dst := make([]byte, size)
+	for i := range dst {
+		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
+	}
+	return string(dst)
+}

app/victoria-metrics/self_scraper.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -49,16 +50,8 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	var currentTimestamp int64
-	for {
-		select {
-		case <-selfScraperStopCh:
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			currentTimestamp = currentTime.UnixNano() / 1e6
-		}
+	f := func(currentTime time.Time, sendStaleMarkers bool) {
+		currentTimestamp := currentTime.UnixNano() / 1e6
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -83,12 +76,27 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			mr.Value = r.Value
+			if sendStaleMarkers {
+				mr.Value = decimal.StaleNaN
+			} else {
+				mr.Value = r.Value
+			}
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
+	for {
+		select {
+		case <-selfScraperStopCh:
+			f(time.Now(), true)
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			f(currentTime, false)
+		}
+	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.d1313636.css",
-    "main.js": "./static/js/main.1919fefe.js",
+    "main.css": "./static/css/main.1e22ee10.css",
+    "main.js": "./static/js/main.92cf3903.js",
     "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.48b7b7105a48d7775f01.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d1313636.css",
-    "static/js/main.1919fefe.js"
+    "static/css/main.1e22ee10.css",
+    "static/js/main.92cf3903.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.92cf3903.js"></script><link href="./static/css/main.1e22ee10.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/media/MetricsQL.48b7b7105a48d7775f01.md

@@ -810,7 +810,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -819,7 +819,7 @@ This function is supported by PromQL. See also [timestamp_with_name](#timestamp_
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
@@ -828,7 +828,7 @@ See also [timestamp](#timestamp).
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +837,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +852,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +862,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 

app/vmagent/datadogsketches/request_handler.go

@@ -0,0 +1,95 @@
+package datadogsketches
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
+		return insertRows(at, sketches, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for _, sketch := range sketches {
+		ms := sketch.ToSummary()
+		for _, m := range ms {
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: m.Name,
+			})
+			for _, label := range m.Labels {
+				labels = append(labels, prompbmarshal.Label{
+					Name:  label.Name,
+					Value: label.Value,
+				})
+			}
+			for _, tag := range sketch.Tags {
+				name, value := datadogutils.SplitTag(tag)
+				if name == "host" {
+					name = "exported_host"
+				}
+				labels = append(labels, prompbmarshal.Label{
+					Name:  name,
+					Value: value,
+				})
+			}
+			labels = append(labels, extraLabels...)
+			samplesLen := len(samples)
+			for _, p := range m.Points {
+				samples = append(samples, prompbmarshal.Sample{
+					Timestamp: p.Timestamp,
+					Value:     p.Value,
+				})
+			}
+			rowsTotal += len(m.Points)
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[samplesLen:],
+			})
+		}
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/main.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
@@ -45,10 +46,10 @@ import (
 )
 
 var (
-	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -69,7 +70,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -118,7 +120,11 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8429"}
+	}
+	logger.Infof("starting vmagent at %q...", listenAddrs)
 	startTime := time.Now()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
@@ -141,9 +147,7 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	if len(*httpListenAddr) > 0 {
-		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
-	}
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -152,13 +156,11 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	if len(*httpListenAddr) > 0 {
-		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-		if err := httpserver.Stop(*httpListenAddr); err != nil {
-			logger.Fatalf("cannot stop the webservice: %s", err)
-		}
-		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -367,6 +369,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "/datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -421,7 +432,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -430,7 +441,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -440,6 +451,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -599,6 +613,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -655,6 +678,9 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
+	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)

app/vmagent/remotewrite/client.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -34,6 +35,7 @@ var (
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
+	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -121,7 +123,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	tr := &http.Transport{
 		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 10 * time.Second,
+		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -395,7 +397,8 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
-	retryDuration := time.Second
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:
@@ -405,8 +408,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > time.Minute {
-			retryDuration = time.Minute
+		if retryDuration > maxRetryDuration {
+			retryDuration = maxRetryDuration
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -452,8 +455,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > time.Minute {
-		retryDuration = time.Minute
+	if retryDuration > maxRetryDuration {
+		retryDuration = maxRetryDuration
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()

app/vmagent/remotewrite/pendingseries.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -15,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -69,7 +71,8 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	ticker := time.NewTicker(*flushInterval)
+	d := timeutil.AddJitterToDuration(*flushInterval)
+	ticker := time.NewTicker(d)
 	defer ticker.Stop()
 	for {
 		select {
@@ -107,11 +110,12 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss []prompbmarshal.TimeSeries
-
+	tss     []prompbmarshal.TimeSeries
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-	buf     []byte
+
+	// buf holds labels data
+	buf []byte
 }
 
 func (wr *writeRequest) reset() {
@@ -222,33 +226,45 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
+// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
+var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
+
 func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
+
+	marshalConcurrencyCh <- struct{}{}
+
 	bb := writeRequestBufPool.Get()
 	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := snappyBufPool.Get()
+		zb := compressBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
+
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			if !tryPushBlock(zb.B) {
-				return false
+			zbLen := len(zb.B)
+			ok := tryPushBlock(zb.B)
+			compressBufPool.Put(zb)
+			if ok {
+				blockSizeRows.Update(float64(len(wr.Timeseries)))
+				blockSizeBytes.Update(float64(zbLen))
 			}
-			blockSizeRows.Update(float64(len(wr.Timeseries)))
-			blockSizeBytes.Update(float64(len(zb.B)))
-			snappyBufPool.Put(zb)
-			return true
+			return ok
 		}
-		snappyBufPool.Put(zb)
+		compressBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -294,5 +310,7 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var writeRequestBufPool bytesutil.ByteBufferPool
-var snappyBufPool bytesutil.ByteBufferPool
+var (
+	writeRequestBufPool bytesutil.ByteBufferPool
+	compressBufPool     bytesutil.ByteBufferPool
+)

app/vmalert-tool/unittest/unittest.go

@@ -25,6 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -184,7 +185,8 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
+	var ab flagutil.ArrayBool
+	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
 		switch r.URL.Path {
 		case "/prometheus/api/v1/query":
 			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
@@ -225,7 +227,7 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop(httpListenAddr); err != nil {
+	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
 		logger.Errorf("cannot stop the webservice: %s", err)
 	}
 	vmstorage.Stop()

app/vmalert/Makefile

@@ -68,6 +68,7 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -22,6 +22,7 @@ groups:
               {{ . | first | value }}
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
+          link: http://localhost:3000/d/wNf0q_kZk?viewPanel=51&from={{($activeAt.Add (parseDurationTime "1h")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime "-1h")).UnixMilli}}
       - alert: ExampleAlertAlwaysFiring
         update_entries_limit: -1
         expr: sum by(job)

app/vmalert/datasource/init.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -93,7 +94,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
 	}
 
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/main.go

@@ -59,8 +59,8 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -178,15 +178,19 @@ func main() {
 
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8880"}
+	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
 	pushmetrics.Stop()
 
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
@@ -248,7 +252,13 @@ func newManager(ctx context.Context) (*manager, error) {
 func getExternalURL(customURL string) (*url.URL, error) {
 	if customURL == "" {
 		// use local hostname as external URL
-		return getHostnameAsExternalURL(*httpListenAddr, httpserver.IsTLS())
+		listenAddr := ":8880"
+		if len(*httpListenAddrs) > 0 {
+			listenAddr = (*httpListenAddrs)[0]
+		}
+		isTLS := httpserver.IsTLS(0)
+
+		return getHostnameAsExternalURL(listenAddr, isTLS)
 	}
 	u, err := url.Parse(customURL)
 	if err != nil {
@@ -260,13 +270,13 @@ func getExternalURL(customURL string) (*url.URL, error) {
 	return u, nil
 }
 
-func getHostnameAsExternalURL(httpListenAddr string, isSecure bool) (*url.URL, error) {
+func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
 	hname, err := os.Hostname()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get hostname: %w", err)
 	}
 	port := ""
-	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+	if ipport := strings.Split(addr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]
 	}
 	schema := "http://"

app/vmalert/manager.go

@@ -156,11 +156,14 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
+			// cancel evaluation so the Update will be applied as fast as possible.
+			// it is important to call InterruptEval before the update, because cancel fn
+			// can be re-assigned during the update.
+			item.old.InterruptEval()
 			go func(old *rule.Group, new *rule.Group) {
 				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
-			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alertmanager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -127,7 +128,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remoteread/init.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -60,7 +61,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/debug_client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -30,7 +30,7 @@ func NewDebugClient() (*DebugClient, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/init.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -64,7 +65,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/rule/alerting.go

@@ -324,16 +324,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 
-		// if alert is instant, For: 0
-		if ar.For == 0 {
-			a.State = notifier.StateFiring
-			for i := range s.Values {
-				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
-			}
-			continue
-		}
-
-		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
@@ -354,6 +344,10 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.Start = at
 			}
 			prevT = at
+			if ar.For == 0 {
+				// rules with `for: 0` are always firing when they have Value
+				a.State = notifier.StateFiring
+			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
 			// save alert's state on last iteration, so it can be used on the next execRange call
@@ -446,14 +440,13 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, start, qFn)
+		a, err := ar.newAlert(m, ls, ts, qFn)
 		if err != nil {
 			curState.Err = fmt.Errorf("failed to create alert: %w", err)
 			return nil, curState.Err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
-		a.ActiveAt = ts
 		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
@@ -479,7 +472,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -559,9 +552,9 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 }
 
 const (
-	// alertMetricName is the metric name for synthetic alert timeseries.
+	// alertMetricName is the metric name for time series reflecting the alert state.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for 'for' state of alert.
+	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -576,12 +569,10 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(a, timestamp))
-	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(a, timestamp))
+	return []prompbmarshal.TimeSeries{
+		alertToTimeSeries(a, timestamp),
+		alertForToTimeSeries(a, timestamp),
 	}
-	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {

app/vmalert/rule/alerting_test.go

@@ -28,20 +28,26 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				"job":      "foo",
-				"instance": "bar",
-			}},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					"job":      "foo",
+					"instance": "bar",
+				}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -49,19 +55,33 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+						"job":      "foo",
+						"instance": "bar",
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				alertStateLabel: "foo",
-				"__name__":      "bar",
-			}},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					alertStateLabel: "foo",
+					"__name__":      "bar",
+				}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__":      alertForStateMetricName,
+						alertStateLabel: "foo",
+					}),
 			},
 		},
 		{
@@ -308,14 +328,17 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &datasource.FakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
+			ts := time.Now()
 			for i, step := range tc.steps {
 				fq.Reset()
 				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
+				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-				// artificial delay between applying steps
-				time.Sleep(defaultStep)
+
+				// shift the execution timestamp before the next iteration
+				ts = ts.Add(defaultStep)
+
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -367,7 +390,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 			nil,
 		},
@@ -378,8 +401,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels: map[string]string{"name": "foo"},
-					State:  notifier.StateFiring,
+					Labels:   map[string]string{"name": "foo"},
+					State:    notifier.StateFiring,
+					ActiveAt: time.Unix(1, 0),
 				},
 			},
 			nil,
@@ -390,9 +414,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
 			},
 			nil,
 		},
@@ -460,6 +484,20 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				For:         time.Second,
 			}},
 		},
+		{
+			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+			},
+			nil,
+		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
 			[]datasource.Metric{
@@ -534,21 +572,25 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					}},
 				//
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					}},
 			},
 			nil,
 		},
@@ -1095,6 +1137,12 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 	return &rule
 }
 
+func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
+	rule := newTestAlertingRule(name, waitFor)
+	rule.EvalInterval = evalInterval
+	return rule
+}
+
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor

app/vmalert/web.go

@@ -12,11 +12,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+
 var (
 	apiLinks = [][2]string{
 		// api links are relative since they can be used by external clients,
@@ -84,7 +88,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -95,12 +102,20 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		data, err := rh.listGroups()
+		var data []byte
+		var err error
+
+		rf := extractRulesFilter(r)
+		data, err = rh.listGroups(rf)
+
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -108,6 +123,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
+
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
@@ -151,6 +167,9 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -201,26 +220,90 @@ type listGroupsResponse struct {
 	} `json:"data"`
 }
 
-func (rh *requestHandler) groups() []apiGroup {
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
+type rulesFilter struct {
+	files         []string
+	groupNames    []string
+	ruleNames     []string
+	ruleType      string
+	excludeAlerts bool
+}
+
+func extractRulesFilter(r *http.Request) rulesFilter {
+	rf := rulesFilter{}
+
+	var ruleType string
+	ruleTypeParam := r.URL.Query().Get("type")
+	// for some reason, `type` in filter doesn't match `type` in response,
+	// so we use this matching here
+	if ruleTypeParam == "alert" {
+		ruleType = ruleTypeAlerting
+	} else if ruleTypeParam == "record" {
+		ruleType = ruleTypeRecording
+	}
+	rf.ruleType = ruleType
+
+	rf.excludeAlerts = httputils.GetBool(r, "exclude_alerts")
+	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
+	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
+	rf.files = append([]string{}, r.Form["file[]"]...)
+	return rf
+}
+
+func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]apiGroup, 0)
-	for _, g := range rh.m.groups {
-		groups = append(groups, groupToAPI(g))
+	isInList := func(list []string, needle string) bool {
+		if len(list) < 1 {
+			return true
+		}
+		for _, i := range list {
+			if i == needle {
+				return true
+			}
+		}
+		return false
 	}
 
-	// sort list of alerts for deterministic output
+	groups := make([]apiGroup, 0)
+	for _, group := range rh.m.groups {
+		if !isInList(rf.groupNames, group.Name) {
+			continue
+		}
+		if !isInList(rf.files, group.File) {
+			continue
+		}
+
+		g := groupToAPI(group)
+		// the returned list should always be non-nil
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
+		filteredRules := make([]apiRule, 0)
+		for _, r := range g.Rules {
+			if rf.ruleType != "" && rf.ruleType != r.Type {
+				continue
+			}
+			if !isInList(rf.ruleNames, r.Name) {
+				continue
+			}
+			if rf.excludeAlerts {
+				r.Alerts = nil
+			}
+			filteredRules = append(filteredRules, r)
+		}
+		g.Rules = filteredRules
+		groups = append(groups, g)
+	}
+	// sort list of groups for deterministic output
 	sort.Slice(groups, func(i, j int) bool {
 		return groups[i].Name < groups[j].Name
 	})
-
 	return groups
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups()
+	lr.Data.Groups = rh.groups(rf)
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{

app/vmalert/web_test.go

@@ -23,6 +23,7 @@ func TestHandler(t *testing.T) {
 	})
 	g := &rule.Group{
 		Name:        "group",
+		File:        "rules.yaml",
 		Concurrency: 1,
 	}
 	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
@@ -165,6 +166,74 @@ func TestHandler(t *testing.T) {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
+
+	t.Run("/api/v1/rules&filters", func(t *testing.T) {
+		check := func(url string, expGroups, expRules int) {
+			t.Helper()
+			lr := listGroupsResponse{}
+			getResp(ts.URL+url, &lr, 200)
+			if length := len(lr.Data.Groups); length != expGroups {
+				t.Errorf("expected %d groups got %d", expGroups, length)
+			}
+			if len(lr.Data.Groups) < 1 {
+				return
+			}
+			var rulesN int
+			for _, gr := range lr.Data.Groups {
+				rulesN += len(gr.Rules)
+			}
+			if rulesN != expRules {
+				t.Errorf("expected %d rules got %d", expRules, rulesN)
+			}
+		}
+
+		check("/api/v1/rules?type=alert", 1, 1)
+		check("/api/v1/rules?type=record", 1, 1)
+
+		check("/vmalert/api/v1/rules?type=alert", 1, 1)
+		check("/vmalert/api/v1/rules?type=record", 1, 1)
+
+		// no filtering expected due to bad params
+		check("/api/v1/rules?type=badParam", 1, 2)
+		check("/api/v1/rules?foo=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
+	})
+	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
+		// check if response returns active alerts by default
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml", &lr, 200)
+		activeAlerts := 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts == 0 {
+			t.Fatalf("expected at least 1 active alert in response; got 0")
+		}
+
+		// disable returning alerts via param
+		lr = listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml&exclude_alerts=true", &lr, 200)
+		activeAlerts = 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts != 0 {
+			t.Fatalf("expected to get 0 active alert in response; got %d", activeAlerts)
+		}
+	})
 }
 
 func TestEmptyResponse(t *testing.T) {

app/vmalert/web_types.go

@@ -193,10 +193,15 @@ func ruleToAPI(r interface{}) apiRule {
 	return apiRule{}
 }
 
+const (
+	ruleTypeRecording = "recording"
+	ruleTypeAlerting  = "alerting"
+)
+
 func recordingToAPI(rr *rule.RecordingRule) apiRule {
 	lastState := rule.GetLastEntry(rr)
 	r := apiRule{
-		Type:              "recording",
+		Type:              ruleTypeRecording,
 		DatasourceType:    rr.Type.String(),
 		Name:              rr.Name,
 		Query:             rr.Expr,
@@ -224,7 +229,7 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 func alertingToAPI(ar *rule.AlertingRule) apiRule {
 	lastState := rule.GetLastEntry(ar)
 	r := apiRule{
-		Type:              "alerting",
+		Type:              ruleTypeAlerting,
 		DatasourceType:    ar.Type.String(),
 		Name:              ar.Name,
 		Query:             ar.Expr,

app/vmauth/auth_config.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,12 +17,13 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
@@ -41,6 +43,9 @@ var (
 type AuthConfig struct {
 	Users            []UserInfo `yaml:"users,omitempty"`
 	UnauthorizedUser *UserInfo  `yaml:"unauthorized_user,omitempty"`
+
+	// ms holds all the metrics for the given AuthConfig
+	ms *metrics.Set
 }
 
 // UserInfo is user information read from authConfigPath
@@ -60,12 +65,15 @@ type UserInfo struct {
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
+	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
+	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -266,8 +274,10 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 		if bu.isBroken() {
 			continue
 		}
-		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
+		if atomic.LoadInt32(&bu.concurrentRequests) == 0 {
 			// Fast path - return the backend with zero concurrently executed requests.
+			// Do not use atomic.CompareAndSwapInt32(), since it is much slower on systems with many CPU cores.
+			atomic.AddInt32(&bu.concurrentRequests, 1)
 			return bu
 		}
 	}
@@ -462,17 +472,19 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var authConfig atomic.Pointer[AuthConfig]
-var authUsers atomic.Pointer[map[string]*UserInfo]
-var authConfigWG sync.WaitGroup
-var stopCh chan struct{}
+var (
+	authConfig   atomic.Pointer[AuthConfig]
+	authUsers    atomic.Pointer[map[string]*UserInfo]
+	authConfigWG sync.WaitGroup
+	stopCh       chan struct{}
+)
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
 func loadAuthConfig() (bool, error) {
-	data, err := fs.ReadFileOrHTTP(*authConfigPath)
+	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
@@ -494,6 +506,11 @@ func loadAuthConfig() (bool, error) {
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
+	prevAc := authConfig.Load()
+	if prevAc != nil {
+		metrics.UnregisterSet(prevAc.ms)
+	}
+	metrics.RegisterSet(ac.ms)
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
@@ -506,10 +523,13 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-	var ac AuthConfig
-	if err = yaml.UnmarshalStrict(data, &ac); err != nil {
+	ac := &AuthConfig{
+		ms: metrics.NewSet(),
+	}
+	if err = yaml.UnmarshalStrict(data, ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
+
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		if ui.Username != "" {
@@ -527,23 +547,30 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
-		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
+		}
+		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
+		ui.concurrencyLimitReached = ac.ms.NewCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
 		ui.httpTransport = tr
 	}
-	return &ac, nil
+	return ac, nil
 }
 
 func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
@@ -554,43 +581,47 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		if ui.BearerToken == "" && ui.Username == "" {
-			return nil, fmt.Errorf("either bearer_token or username must be set")
+		if ui.Username != "" && ui.Password == "" {
+			// Do not allow setting username without password if there are other auth configs exist.
+			// This should prevent from typical mis-configuration when access by username without password
+			// remains open if other authorization schemes are defined.
+			if ui.BearerToken != "" {
+				return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+			}
 		}
-		if ui.BearerToken != "" && ui.Username != "" {
-			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+		ats := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if len(ats) == 0 {
+			return nil, fmt.Errorf("one of bearer_token, username or mtls must be set")
 		}
-		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[at1] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
-		}
-		if byAuthToken[at2] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
+		for _, at := range ats {
+			if uiOld := byAuthToken[at]; uiOld != nil {
+				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
+					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
+			}
 		}
 
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
-		name := ui.name()
-		if ui.BearerToken != "" {
-			if ui.Password != "" {
-				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			}
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		if ui.BearerToken != "" && ui.Password != "" {
+			return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 		}
-		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
+		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -600,12 +631,36 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		}
 		ui.httpTransport = tr
 
-		byAuthToken[at1] = ui
-		byAuthToken[at2] = ui
+		for _, at := range ats {
+			byAuthToken[at] = ui
+		}
 	}
 	return byAuthToken, nil
 }
 
+var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
+
+func (ui *UserInfo) getMetricLabels() (string, error) {
+	name := ui.name()
+	if len(name) == 0 && len(ui.MetricLabels) == 0 {
+		// fast path
+		return "", nil
+	}
+	labels := make([]string, 0, len(ui.MetricLabels)+1)
+	if len(name) > 0 {
+		labels = append(labels, fmt.Sprintf(`username=%q`, name))
+	}
+	for k, v := range ui.MetricLabels {
+		if !labelNameRegexp.MatchString(k) {
+			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
+		}
+		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
+	}
+	sort.Strings(labels)
+	labelsStr := "{" + strings.Join(labels, ",") + "}"
+	return labelsStr, nil
+}
+
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
@@ -676,29 +731,51 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		return "bearer_token"
+		h := xxhash.Sum64([]byte(ui.BearerToken))
+		return fmt.Sprintf("bearer_token:hash:%016X", h)
 	}
 	return ""
 }
 
-func getAuthTokens(bearerToken, username, password string) (string, string) {
+func getAuthTokens(bearerToken, username, password string) []string {
+	var ats []string
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getAuthToken(bearerToken, "", "")
-		at2 := getAuthToken("", bearerToken, "")
-		return at1, at2
+		at1 := getHTTPAuthBearerToken(bearerToken)
+		at2 := getHTTPAuthBasicToken(bearerToken, "")
+		ats = append(ats, at1, at2)
+	} else if username != "" {
+		at := getHTTPAuthBasicToken(username, password)
+		ats = append(ats, at)
 	}
-	at := getAuthToken("", username, password)
-	return at, at
+	return ats
 }
 
-func getAuthToken(bearerToken, username, password string) string {
-	if bearerToken != "" {
-		return "Bearer " + bearerToken
-	}
+func getHTTPAuthBearerToken(bearerToken string) string {
+	return "http_auth:Bearer " + bearerToken
+}
+
+func getHTTPAuthBasicToken(username, password string) string {
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "Basic " + token64
+	return "http_auth:Basic " + token64
+}
+
+func getAuthTokensFromRequest(r *http.Request) []string {
+	var ats []string
+
+	ah := r.Header.Get("Authorization")
+	if ah == "" {
+		return ats
+	}
+	if strings.HasPrefix(ah, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		ah = strings.Replace(ah, "Token", "Bearer", 1)
+	}
+	at := "http_auth:" + ah
+	ats = append(ats, at)
+	return ats
 }
 
 func (up *URLPrefix) sanitize() error {

app/vmauth/auth_config_test.go

@@ -229,6 +229,14 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
+`)
+	// Invalid metric label name
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  metric_labels:
+    not-prometheus-compatible: value
 `)
 }
 
@@ -259,7 +267,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -282,7 +290,7 @@ users:
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
@@ -304,11 +312,11 @@ users:
 - username: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "bar", ""): {
+		getHTTPAuthBasicToken("bar", ""): {
 			Username:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
@@ -328,7 +336,7 @@ users:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -357,7 +365,7 @@ users:
 				},
 			},
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -387,7 +395,7 @@ users:
 			},
 		},
 	})
-	// Multiple users with the same name
+	// Multiple users with the same name - this should work, since these users have different passwords
 	f(`
 users:
 - username: foo-same
@@ -397,17 +405,18 @@ users:
   password: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo-same", "baz"): {
+		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "foo-same", "bar"): {
+		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
+
 	// with default url
 	f(`
 users:
@@ -424,7 +433,7 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -456,7 +465,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -489,7 +498,41 @@ users:
 			}),
 		},
 	})
-
+	// With metric_labels
+	f(`
+users:
+- username: foo-same
+  password: baz
+  url_prefix: http://foo
+  metric_labels:
+    dc: eu
+    team: dev
+- username: foo-same
+  password: bar
+  url_prefix: https://bar/x///
+  metric_labels:
+    backend_env: test
+    team: accounting
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo-same", "baz"): {
+			Username:  "foo-same",
+			Password:  "baz",
+			URLPrefix: mustParseURL("http://foo"),
+			MetricLabels: map[string]string{
+				"dc":   "eu",
+				"team": "dev",
+			},
+		},
+		getHTTPAuthBasicToken("foo-same", "bar"): {
+			Username:  "foo-same",
+			Password:  "bar",
+			URLPrefix: mustParseURL("https://bar/x"),
+			MetricLabels: map[string]string{
+				"backend_env": "test",
+				"team":        "accounting",
+			},
+		},
+	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -516,7 +559,7 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getAuthToken("", "foo", "bar")]
+	ui := m[getHTTPAuthBasicToken("foo", "bar")]
 	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
@@ -526,6 +569,86 @@ unauthorized_user:
 	}
 }
 
+func TestUserInfoGetMetricLabels(t *testing.T) {
+	t.Run("empty-labels", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-username", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-name", func(t *testing.T) {
+		ui := &UserInfo{
+			Name:        "user1",
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-bearer-token", func(t *testing.T) {
+		ui := &UserInfo{
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("invalid-label", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "foo",
+			MetricLabels: map[string]string{
+				",{": "aaaa",
+			},
+		}
+		_, err := ui.getMetricLabels()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	})
+}
+
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false

app/vmauth/example_config.yml

@@ -10,6 +10,11 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Adds labels to the exported metrics for given user section
+  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
+  metric_labels:
+    backend_dc: eu
+    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.

app/vmauth/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -33,8 +33,8 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
@@ -45,7 +45,7 @@ var (
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
@@ -65,10 +65,14 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	logger.Infof("starting vmauth at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8427"}
+	}
+	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -77,8 +81,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -89,7 +93,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, *reloadAuthKey, "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -97,8 +101,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
-	authToken := r.Header.Get("Authorization")
-	if authToken == "" {
+
+	ats := getAuthTokensFromRequest(r)
+	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
@@ -110,18 +115,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
-	if strings.HasPrefix(authToken, "Token ") {
-		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-		// See https://docs.influxdata.com/influxdb/v2.0/api/
-		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
-	}
 
-	ac := *authUsers.Load()
-	ui := ac[authToken]
+	ui := getUserInfoByAuthTokens(ats)
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
+			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
@@ -137,6 +136,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+func getUserInfoByAuthTokens(ats []string) *UserInfo {
+	ac := *authUsers.Load()
+	for _, at := range ats {
+		ui := ac[at]
+		if ui != nil {
+			return ui
+		}
+	}
+	return nil
+}
+
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
@@ -150,12 +160,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if err := ui.beginConcurrencyLimit(); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
+
+			// Requests failed because of concurrency limit must be counted as errors,
+			// since this usually means the backend cannot keep up with the current load.
+			ui.backendErrors.Inc()
 			return
 		}
 	default:
 		concurrentRequestsLimitReached.Inc()
 		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
 		handleConcurrencyLimitError(w, r, err)
+
+		// Requests failed because of concurrency limit must be counted as errors,
+		// since this usually means the backend cannot keep up with the current load.
+		ui.backendErrors.Inc()
 		return
 	}
 	processRequest(w, r, ui)
@@ -201,7 +219,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -213,15 +231,16 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
+	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	req.Host = targetURL.Host
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := transport.RoundTrip(req)
+	res, err := ui.httpTransport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -229,15 +248,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				ui.backendErrors.Inc()
+			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
+				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -247,7 +271,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
 		return false
 	}
-	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
+	if hasInt(retryStatusCodes, res.StatusCode) {
+		_ = res.Body.Close()
+		if !rtbOK || !rtb.canRetry() {
+			// If we get an error from the retry_status_codes list, but cannot execute retry,
+			// we consider such a request an error as well.
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+					res.StatusCode, targetURL),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
+			return true
+		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
@@ -266,6 +303,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
 	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
 	copyBufPool.Put(copyBuf)
+	_ = res.Body.Close()
 	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
@@ -393,8 +431,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var transportMap = make(map[string]*http.Transport)
-var transportMapLock sync.Mutex
+var (
+	transportMap     = make(map[string]*http.Transport)
+	transportMapLock sync.Mutex
+)
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)
@@ -422,7 +462,7 @@ func newTransport(insecureSkipVerify bool, caFile string) (*http.Transport, erro
 		tlsCfg.ClientSessionCache = tls.NewLRUClientSessionCache(0)
 		tlsCfg.InsecureSkipVerify = insecureSkipVerify
 		if caFile != "" {
-			data, err := fs.ReadFileOrHTTP(caFile)
+			data, err := fscore.ReadFileOrHTTP(caFile)
 			if err != nil {
 				return nil, fmt.Errorf("cannot read tls_ca_file: %w", err)
 			}

app/vmbackup/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
 )
 
 var (
@@ -93,7 +94,8 @@ func main() {
 		}
 	}
 
-	go httpserver.Serve(*httpListenAddr, false, nil)
+	listenAddrs := []string{*httpListenAddr}
+	go httpserver.Serve(listenAddrs, nil, nil)
 
 	pushmetrics.Init()
 	err := makeBackup()
@@ -104,8 +106,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
@@ -168,7 +170,7 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
 }
 
 func newSrcFS() (*fslocal.FS, error) {
-	if err := snapshot.Validate(*snapshotName); err != nil {
+	if err := snapshotutil.Validate(*snapshotName); err != nil {
 		return nil, fmt.Errorf("invalid -snapshotName=%q: %w", *snapshotName, err)
 	}
 	snapshotPath := filepath.Join(*storageDataPath, "snapshots", *snapshotName)

app/vmctl/backoff/backoff_test.go

@@ -15,7 +15,6 @@ func TestRetry_Do(t *testing.T) {
 		backoffFactor      float64
 		backoffMinDuration time.Duration
 		retryableFunc      retryableFunc
-		ctx                context.Context
 		cancelTimeout      time.Duration
 		want               uint64
 		wantErr            bool
@@ -25,7 +24,6 @@ func TestRetry_Do(t *testing.T) {
 			retryableFunc: func() error {
 				return ErrBadRequest
 			},
-			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -35,7 +33,6 @@ func TestRetry_Do(t *testing.T) {
 				time.Sleep(time.Millisecond * 100)
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -58,7 +55,6 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    1,
 			wantErr: false,
 		},
@@ -75,7 +71,6 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    5,
 			wantErr: true,
 		},
@@ -85,14 +80,8 @@ func TestRetry_Do(t *testing.T) {
 			backoffFactor:      1.7,
 			backoffMinDuration: time.Millisecond * 10,
 			retryableFunc: func() error {
-				t := time.NewTicker(time.Millisecond * 5)
-				defer t.Stop()
-				for range t.C {
-					return fmt.Errorf("got some error")
-				}
-				return nil
+				return fmt.Errorf("got some error")
 			},
-			ctx:           context.Background(),
 			cancelTimeout: time.Millisecond * 40,
 			want:          3,
 			wantErr:       true,
@@ -101,12 +90,13 @@ func TestRetry_Do(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			r := &Backoff{retries: tt.backoffRetries, factor: tt.backoffFactor, minDuration: tt.backoffMinDuration}
+			ctx := context.Background()
 			if tt.cancelTimeout != 0 {
-				newCtx, cancelFn := context.WithTimeout(tt.ctx, tt.cancelTimeout)
-				tt.ctx = newCtx
+				newCtx, cancelFn := context.WithTimeout(context.Background(), tt.cancelTimeout)
+				ctx = newCtx
 				defer cancelFn()
 			}
-			got, err := r.Retry(tt.ctx, tt.retryableFunc)
+			got, err := r.Retry(ctx, tt.retryableFunc)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
 				return

app/vminsert/datadogsketches/request_handler.go

@@ -0,0 +1,85 @@
+package datadogsketches
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="datadogsketches"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="datadogsketches"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
+		return insertRows(sketches, extraLabels)
+	})
+}
+
+func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsLen := 0
+	for _, sketch := range sketches {
+		rowsLen += sketch.RowsCount()
+	}
+	ctx.Reset(rowsLen)
+	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
+	for _, sketch := range sketches {
+		ms := sketch.ToSummary()
+		for _, m := range ms {
+			ctx.Labels = ctx.Labels[:0]
+			ctx.AddLabel("", m.Name)
+			for _, label := range m.Labels {
+				ctx.AddLabel(label.Name, label.Value)
+			}
+			for _, tag := range sketch.Tags {
+				name, value := datadogutils.SplitTag(tag)
+				if name == "host" {
+					name = "exported_host"
+				}
+				ctx.AddLabel(name, value)
+			}
+			for j := range extraLabels {
+				label := &extraLabels[j]
+				ctx.AddLabel(label.Name, label.Value)
+			}
+			if hasRelabeling {
+				ctx.ApplyRelabeling()
+			}
+			if len(ctx.Labels) == 0 {
+				// Skip metric without labels.
+				continue
+			}
+			ctx.SortLabelsIfNeeded()
+			var metricNameRaw []byte
+			var err error
+			for _, p := range m.Points {
+				metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, p.Timestamp, p.Value)
+				if err != nil {
+					return err
+				}
+			}
+			rowsTotal += len(m.Points)
+		}
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/main.go

@@ -13,6 +13,7 @@ import (
 
 	vminsertCommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -29,6 +30,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
@@ -62,7 +64,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )
@@ -269,6 +272,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "/datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -315,7 +327,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -324,7 +336,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -334,9 +346,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
-		w.WriteHeader(http.StatusNoContent)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
 		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
@@ -389,6 +404,9 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
+	datadogsketchesWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+
 	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake", protocol="datadog"}`)

app/vmrestore/main.go

@@ -37,7 +37,8 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	go httpserver.Serve(*httpListenAddr, false, nil)
+	listenAddrs := []string{*httpListenAddr}
+	go httpserver.Serve(listenAddrs, nil, nil)
 
 	srcFS, err := newSrcFS()
 	if err != nil {
@@ -62,8 +63,8 @@ func main() {
 	dstFS.MustStop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())

app/vmselect/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
@@ -29,13 +30,13 @@ import (
 )
 
 var (
-	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
+	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	resetCacheAuthKey    = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
+	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
@@ -148,8 +149,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
 					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
+				StatusCode: http.StatusTooManyRequests,
 			}
+			w.Header().Add("Retry-After", "10")
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -170,7 +172,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	if path == "/internal/resetRollupResultCache" {
-		if !httpserver.CheckAuthFlag(w, r, *resetCacheAuthKey, "resetCacheAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, resetCacheAuthKey.Get(), "resetCacheAuthKey") {
 			return true
 		}
 		promql.ResetRollupResultCache()
@@ -367,7 +369,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
-		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
 			return true
 		}
 		graphiteTagsDelSeriesRequests.Inc()
@@ -386,7 +388,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
-		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
 			return true
 		}
 		deleteRequests.Inc()
@@ -425,6 +427,14 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		}
 		return true
 	}
+	if path == "/vmui/timezone" {
+		httpserver.EnableCORS(w, r)
+		if err := handleVMUITimezone(w); err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	}
 	if strings.HasPrefix(path, "/vmui/") {
 		if strings.HasPrefix(path, "/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.

app/vmselect/netstorage/netstorage.go

@@ -5,10 +5,12 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"reflect"
 	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -88,30 +90,6 @@ type timeseriesWork struct {
 	rowsProcessed int
 }
 
-func (tsw *timeseriesWork) reset() {
-	tsw.mustStop = nil
-	tsw.rss = nil
-	tsw.pts = nil
-	tsw.f = nil
-	tsw.err = nil
-	tsw.rowsProcessed = 0
-}
-
-func getTimeseriesWork() *timeseriesWork {
-	v := tswPool.Get()
-	if v == nil {
-		v = &timeseriesWork{}
-	}
-	return v.(*timeseriesWork)
-}
-
-func putTimeseriesWork(tsw *timeseriesWork) {
-	tsw.reset()
-	tswPool.Put(tsw)
-}
-
-var tswPool sync.Pool
-
 func (tsw *timeseriesWork) do(r *Result, workerID uint) error {
 	if atomic.LoadUint32(tsw.mustStop) != 0 {
 		return nil
@@ -270,22 +248,20 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	maxWorkers := MaxWorkers()
 	if maxWorkers == 1 || tswsLen == 1 {
 		// It is faster to process time series in the current goroutine.
-		tsw := getTimeseriesWork()
+		var tsw timeseriesWork
 		tmpResult := getTmpResult()
 		rowsProcessedTotal := 0
 		var err error
 		for i := range rss.packedTimeseries {
-			initTimeseriesWork(tsw, &rss.packedTimeseries[i])
+			initTimeseriesWork(&tsw, &rss.packedTimeseries[i])
 			err = tsw.do(&tmpResult.rs, 0)
 			rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 			rowsProcessedTotal += tsw.rowsProcessed
 			if err != nil {
 				break
 			}
-			tsw.reset()
 		}
 		putTmpResult(tmpResult)
-		putTimeseriesWork(tsw)
 
 		return rowsProcessedTotal, err
 	}
@@ -295,11 +271,9 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// which reduces the scalability on systems with many CPU cores.
 
 	// Prepare the work for workers.
-	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
+	tsws := make([]timeseriesWork, len(rss.packedTimeseries))
 	for i := range rss.packedTimeseries {
-		tsw := getTimeseriesWork()
-		initTimeseriesWork(tsw, &rss.packedTimeseries[i])
-		tsws[i] = tsw
+		initTimeseriesWork(&tsws[i], &rss.packedTimeseries[i])
 	}
 
 	// Prepare worker channels.
@@ -314,9 +288,9 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	}
 
 	// Spread work among workers.
-	for i, tsw := range tsws {
+	for i := range tsws {
 		idx := i % len(workChs)
-		workChs[idx] <- tsw
+		workChs[idx] <- &tsws[i]
 	}
 	// Mark worker channels as closed.
 	for _, workCh := range workChs {
@@ -339,14 +313,14 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// Collect results.
 	var firstErr error
 	rowsProcessedTotal := 0
-	for _, tsw := range tsws {
+	for i := range tsws {
+		tsw := &tsws[i]
 		if tsw.err != nil && firstErr == nil {
 			// Return just the first error, since other errors are likely duplicate the first error.
 			firstErr = tsw.err
 		}
 		rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 		rowsProcessedTotal += tsw.rowsProcessed
-		putTimeseriesWork(tsw)
 	}
 	return rowsProcessedTotal, firstErr
 }
@@ -1044,7 +1018,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 			for xw := range workCh {
 				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
 					errGlobalLock.Lock()
-					if errGlobal != nil {
+					if errGlobal == nil {
 						errGlobal = err
 						atomic.StoreUint32(&mustStop, 1)
 					}
@@ -1169,15 +1143,44 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
-		brsPrealloc [4]blockRef
-		brs         []blockRef
+		brs []blockRef
 	}
-	m := make(map[string]*blockRefs, maxSeriesCount)
-	orderedMetricNames := make([]string, 0, maxSeriesCount)
+
 	blocksRead := 0
 	samples := 0
 	tbf := getTmpBlocksFile()
 	var buf []byte
+	var metricNamePrev []byte
+
+	// metricNamesBuf is used for holding all the loaded unique metric names at m and orderedMetricNames.
+	// It should reduce pressure on Go GC by reducing the number of string allocations
+	// when constructing metricName string from byte slice.
+	metricNamesBufCap := maxSeriesCount * 100
+	if metricNamesBufCap > maxFastAllocBlockSize {
+		metricNamesBufCap = maxFastAllocBlockSize
+	}
+	metricNamesBuf := make([]byte, 0, metricNamesBufCap)
+
+	// brssPool is used for holding all the blockRefs objects across all the loaded time series.
+	// It should reduce pressure on Go GC by reducing the number of blockRefs allocations.
+	brssPool := make([]blockRefs, 0, maxSeriesCount)
+
+	// brsPool is used for holding the most of blockRefs.brs slices across all the loaded time series.
+	// It should reduce pressure on Go GC by reducing the number of allocations for blockRefs.brs slices.
+	brsPoolCap := uintptr(maxSeriesCount)
+	if brsPoolCap > maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) {
+		brsPoolCap = maxFastAllocBlockSize / unsafe.Sizeof(blockRef{})
+	}
+	brsPool := make([]blockRef, 0, brsPoolCap)
+
+	// m maps from metricName to the index of blockRefs inside brssPool
+	m := make(map[string]int, maxSeriesCount)
+
+	// orderedMetricNames contains the list of loaded unique metric names
+	// in the load order. This order is important for triggering sequential data reading.
+	orderedMetricNames := make([]string, 0, maxSeriesCount)
+
+	var brsIdx int
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
@@ -1190,8 +1193,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		if *maxSamplesPerQuery > 0 && samples > *maxSamplesPerQuery {
 			putTmpBlocksFile(tbf)
 			putStorageSearch(sr)
-			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: to increase the -search.maxSamplesPerQuery; to reduce time range for the query; to use more specific label filters in order to select lower number of series", *maxSamplesPerQuery)
+			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: to increase the -search.maxSamplesPerQuery; "+
+				"to reduce time range for the query; to use more specific label filters in order to select lower number of series", *maxSamplesPerQuery)
 		}
+
 		buf = br.Marshal(buf[:0])
 		addr, err := tbf.WriteBlockRefData(buf)
 		if err != nil {
@@ -1199,24 +1204,59 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-		// Do not intern mb.MetricName, since it leads to increased memory usage.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
+
 		metricName := sr.MetricBlockRef.MetricName
-		brs := m[string(metricName)]
-		if brs == nil {
-			brs = &blockRefs{}
-			brs.brs = brs.brsPrealloc[:0]
+		if metricNamePrev == nil || string(metricName) != string(metricNamePrev) {
+			idx, ok := m[string(metricName)]
+			if !ok {
+				if cap(brssPool) > len(brssPool) {
+					brssPool = brssPool[:len(brssPool)+1]
+				} else {
+					brssPool = append(brssPool, blockRefs{})
+				}
+				idx = len(brssPool) - 1
+			}
+			brsIdx = idx
+			metricNamePrev = append(metricNamePrev[:0], metricName...)
+		}
+
+		brs := &brssPool[brsIdx]
+		partRef := br.PartRef()
+		if uintptr(cap(brsPool)) >= maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) && len(brsPool) == cap(brsPool) {
+			// Allocate a new brsPool in order to avoid slow allocation of an object
+			// bigger than maxFastAllocBlockSize bytes at append() below.
+			brsPool = make([]blockRef, 0, maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}))
+		}
+		if canAppendToBlockRefPool(brsPool, brs.brs) {
+			// It is safe appending blockRef to brsPool, since there are no other items added there yet.
+			brsPool = append(brsPool, blockRef{
+				partRef: partRef,
+				addr:    addr,
+			})
+			brs.brs = brsPool[len(brsPool)-len(brs.brs)-1 : len(brsPool) : len(brsPool)]
+		} else {
+			// It is unsafe appending blockRef to brsPool, since there are other items added there.
+			// So just append it to brs.brs.
+			brs.brs = append(brs.brs, blockRef{
+				partRef: partRef,
+				addr:    addr,
+			})
 		}
-		brs.brs = append(brs.brs, blockRef{
-			partRef: br.PartRef(),
-			addr:    addr,
-		})
 		if len(brs.brs) == 1 {
-			metricNameStr := string(metricName)
+			if cap(metricNamesBuf) >= maxFastAllocBlockSize && len(metricNamesBuf)+len(metricName) > cap(metricNamesBuf) {
+				// Allocate a new metricNamesBuf in order to avoid slow allocation of byte slice
+				// bigger than maxFastAllocBlockSize bytes at append() below.
+				metricNamesBuf = make([]byte, 0, maxFastAllocBlockSize)
+			}
+			metricNamesBufLen := len(metricNamesBuf)
+			metricNamesBuf = append(metricNamesBuf, metricName...)
+			metricNameStr := bytesutil.ToUnsafeString(metricNamesBuf[metricNamesBufLen:])
+
 			orderedMetricNames = append(orderedMetricNames, metricNameStr)
-			m[metricNameStr] = brs
+			m[metricNameStr] = brsIdx
 		}
 	}
+
 	if err := sr.Error(); err != nil {
 		putTmpBlocksFile(tbf)
 		putStorageSearch(sr)
@@ -1239,7 +1279,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	for i, metricName := range orderedMetricNames {
 		pts[i] = packedTimeseries{
 			metricName: metricName,
-			brs:        m[metricName].brs,
+			brs:        brssPool[m[metricName]].brs,
 		}
 	}
 	rss.packedTimeseries = pts
@@ -1255,6 +1295,25 @@ type blockRef struct {
 	addr    tmpBlockAddr
 }
 
+// canAppendToBlockRefPool returns true if a points to the pool and the last item in a is the last item in the pool.
+//
+// In this case it is safe appending an item to the pool and then updating the a, so it refers to the extended slice.
+//
+// True is also returned if a is nil, since in this case it is safe appending an item to the pool and pointing a
+// to the last item in the pool.
+func canAppendToBlockRefPool(pool, a []blockRef) bool {
+	if a == nil {
+		return true
+	}
+	if len(a) > len(pool) {
+		// a doesn't belong to pool
+		return false
+	}
+	shPool := (*reflect.SliceHeader)(unsafe.Pointer(&pool))
+	shA := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	return shPool.Data+uintptr(shPool.Len)*unsafe.Sizeof(blockRef{}) == shA.Data+uintptr(shA.Len)*unsafe.Sizeof(blockRef{})
+}
+
 func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]storage.TagFilter, maxMetrics int, deadline searchutils.Deadline) ([]*storage.TagFilters, error) {
 	tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
 	for _, tagFilters := range tagFilterss {
@@ -1300,3 +1359,8 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 	}
 	return dst, nil
 }
+
+// Go uses fast allocations for block sizes up to 32Kb.
+//
+// See https://github.com/golang/go/blob/704401ffa06c60e059c9e6e4048045b4ff42530a/src/runtime/malloc.go#L11
+const maxFastAllocBlockSize = 32 * 1024

app/vmselect/netstorage/tmp_blocks_file.go

@@ -136,21 +136,25 @@ func (tbf *tmpBlocksFile) Finalize() error {
 		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
 	}
 	tbf.buf = tbf.buf[:0]
-	r := fs.MustOpenReaderAt(fname)
+	r := fs.NewReaderAt(tbf.f)
+
 	// Hint the OS that the file is read almost sequentiallly.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
+
 	// Collect local stats in order to improve performance on systems with big number of CPU cores.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966
 	r.SetUseLocalStats()
+
 	tbf.r = r
+	tbf.f = nil
 	return nil
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
 	var buf []byte
-	if tbf.f == nil {
+	if tbf.r == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
 	} else {
 		bb := tmpBufPool.Get()
@@ -169,21 +173,45 @@ func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBl
 var tmpBufPool bytesutil.ByteBufferPool
 
 func (tbf *tmpBlocksFile) MustClose() {
-	if tbf.f == nil {
+	if tbf.f != nil {
+		// tbf.f could be non-nil if Finalize wasn't called.
+		// In this case tbf.r must be nil.
+		if tbf.r != nil {
+			logger.Panicf("BUG: tbf.r must be nil when tbf.f!=nil")
+		}
+
+		// Try removing the file before closing it in order to prevent from flushing the in-memory data
+		// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
+		// Gracefully handle this case by attempting to remove the file after closing it.
+		fname := tbf.f.Name()
+		errRemove := os.Remove(fname)
+		if err := tbf.f.Close(); err != nil {
+			logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+		}
+		if errRemove != nil {
+			if err := os.Remove(fname); err != nil {
+				logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+			}
+		}
+		tbf.f = nil
 		return
 	}
-	if tbf.r != nil {
-		// tbf.r could be nil if Finalize wasn't called.
-		tbf.r.MustClose()
-	}
-	fname := tbf.f.Name()
 
-	if err := tbf.f.Close(); err != nil {
-		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+	if tbf.r == nil {
+		// Nothing to do
+		return
 	}
-	// We cannot remove unclosed at non-posix filesystems, like windows
-	if err := os.Remove(fname); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+
+	// Try removing the file before closing it in order to prevent from flushing the in-memory data
+	// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
+	// Gracefully handle this case by attempting to remove the file after closing it.
+	fname := tbf.r.Path()
+	errRemove := os.Remove(fname)
+	tbf.r.MustClose()
+	if errRemove != nil {
+		if err := os.Remove(fname); err != nil {
+			logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+		}
 	}
-	tbf.f = nil
+	tbf.r = nil
 }

app/vmselect/prometheus/expand-with-exprs.qtpl

@@ -82,7 +82,7 @@ textarea { margin: 1em }
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -146,7 +146,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/expand-with-exprs.qtpl.go

@@ -195,7 +195,7 @@ func streamwithExprsTutorial(qw422016 *qt422016.Writer) {
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -259,7 +259,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/prometheus.go

@@ -321,7 +321,9 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonPara
 		firstLineSent := uint32(0)
 		writeLineFunc = func(xb *exportBlock, workerID uint) error {
 			bb := sw.getBuffer(workerID)
-			if atomic.CompareAndSwapUint32(&firstLineOnce, 0, 1) {
+			// Use atomic.LoadUint32() in front of atomic.CompareAndSwapUint32() in order to avoid slow inter-CPU synchronization
+			// in fast path after the first line has been already sent.
+			if atomic.LoadUint32(&firstLineOnce) == 0 && atomic.CompareAndSwapUint32(&firstLineOnce, 0, 1) {
 				// Send the first line to sw.bw
 				WriteExportPromAPILine(bb, xb)
 				_, err := sw.bw.Write(bb.B)
@@ -497,7 +499,10 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 	if err != nil {
 		return err
 	}
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
+	// Do not limit the number of unique time series, which could be scanned
+	// during the search for matching label values, since users expect this API
+	// must always work.
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, -1)
 	labelValues, err := netstorage.LabelValues(qt, labelName, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
@@ -594,7 +599,10 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	if err != nil {
 		return err
 	}
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
+	// Do not limit the number of unique time series, which could be scanned
+	// during the search for matching label values, since users expect this API
+	// must always work.
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, -1)
 	labels, err := netstorage.LabelNames(qt, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain labels: %w", err)
@@ -718,9 +726,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		start -= offset
 		end := start
 		start = end - window
-		if start < 0 {
-			start = 0
-		}
 		// Do not include data point with a timestamp matching the lower boundary of the window as Prometheus does.
 		start++
 		if end < start {

app/vmselect/promql/aggr.go

@@ -111,41 +111,48 @@ func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) [
 	modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
 	m := aggrPrepareSeries(argOrig, modifier, maxSeries, keepOriginal)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tss := range m {
-		rv := afe(tss, modifier)
+	for _, tssl := range m {
+		rv := afe(tssl.tss, modifier)
 		rvs = append(rvs, rv...)
 	}
 	return rvs, nil
 }
 
-func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string][]*timeseries {
+func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string]*tssList {
 	// Remove empty time series, e.g. series with all NaN samples,
 	// since such series are ignored by aggregate functions.
 	argOrig = removeEmptySeries(argOrig)
 	arg := copyTimeseriesMetricNames(argOrig, keepOriginal)
 
 	// Perform grouping.
-	m := make(map[string][]*timeseries)
+	m := make(map[string]*tssList)
 	bb := bbPool.Get()
 	for i, ts := range arg {
 		removeGroupTags(&ts.MetricName, modifier)
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		k := string(bb.B)
+		k := bb.B
 		if keepOriginal {
 			ts = argOrig[i]
 		}
-		tss := m[k]
-		if tss == nil && maxSeries > 0 && len(m) >= maxSeries {
-			// We already reached time series limit after grouping. Skip other time series.
-			continue
+		tssl := m[string(k)]
+		if tssl == nil {
+			if maxSeries > 0 && len(m) >= maxSeries {
+				// We already reached time series limit after grouping. Skip other time series.
+				continue
+			}
+			tssl = &tssList{}
+			m[string(k)] = tssl
 		}
-		tss = append(tss, ts)
-		m[k] = tss
+		tssl.tss = append(tssl.tss, ts)
 	}
 	bbPool.Put(bb)
 	return m
 }
 
+type tssList struct {
+	tss []*timeseries
+}
+
 func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	tss, err := getAggrTimeseries(afa.args)
 	if err != nil {
@@ -626,8 +633,8 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 
 	m := aggrPrepareSeries(args[1], &afa.ae.Modifier, afa.ae.Limit, false)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tss := range m {
-		rv, err := afe(tss, modifier)
+	for _, tssl := range m {
+		rv, err := afe(tssl.tss, modifier)
 		if err != nil {
 			return nil, err
 		}

app/vmselect/promql/aggr_incremental.go

@@ -111,8 +111,8 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 	removeGroupTags(&ts.MetricName, &iafc.ae.Modifier)
 	bb := bbPool.Get()
 	bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-	k := string(bb.B)
-	iac := m[k]
+	k := bb.B
+	iac := m[string(k)]
 	if iac == nil {
 		if iafc.ae.Limit > 0 && len(m) >= iafc.ae.Limit {
 			// Skip this time series, since the limit on the number of output time series has been already reached.
@@ -131,7 +131,7 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 			ts:     tsAggr,
 			values: make([]float64, len(ts.Values)),
 		}
-		m[k] = iac
+		m[string(k)] = iac
 	}
 	bbPool.Put(bb)
 	iafc.callbacks.updateAggrFunc(iac, ts.Values)

app/vmselect/promql/binary_op.go

@@ -257,7 +257,7 @@ func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft,
 			var tsCopy timeseries
 			tsCopy.CopyFromShallowTimestamps(tsLeft)
 			tsCopy.MetricName.SetTags(joinTags, joinPrefix, skipTags, &tsRight.MetricName)
-			bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
+			bb.B = marshalMetricNameSorted(bb.B[:0], &tsCopy.MetricName)
 			pair, ok := m[string(bb.B)]
 			if !ok {
 				m[string(bb.B)] = &tsPair{
@@ -522,7 +522,9 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 		mn := storage.GetMetricName()
 		for _, ts := range arg {
 			mn.CopyFrom(&ts.MetricName)
-			mn.ResetMetricGroup()
+			if !be.KeepMetricNames {
+				mn.ResetMetricGroup()
+			}
 			switch groupOp {
 			case "on":
 				mn.RemoveTagsOn(groupTags)
@@ -531,7 +533,7 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 			default:
 				logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
 			}
-			bb.B = marshalMetricTagsSorted(bb.B[:0], mn)
+			bb.B = marshalMetricNameSorted(bb.B[:0], mn)
 			k := string(bb.B)
 			m[k] = append(m[k], ts)
 		}

app/vmselect/promql/eval.go

@@ -29,7 +29,8 @@ import (
 )
 
 var (
-	disableCache                   = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling")
+	disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful when ingesting historical data. "+
+		"See https://docs.victoriametrics.com/#backfilling . See also -search.resetRollupResultCacheOnStartup")
 	maxPointsSubqueryPerTimeseries = flag.Int("search.maxPointsSubqueryPerTimeseries", 100e3, "The maximum number of points per series, which can be generated by subquery. "+
 		"See https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3")
 	maxMemoryPerQuery = flagutil.NewBytes("search.maxMemoryPerQuery", 0, "The maximum amounts of memory a single query may consume. "+
@@ -1265,7 +1266,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains bigger values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1327,7 +1328,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains smaller values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1392,7 +1393,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		return tss, nil
 	default:
 		qt.Printf("instant rollup optimization isn't implemented for %s()", funcName)
@@ -1419,8 +1420,8 @@ func hasDuplicateSeries(tss []*timeseries) bool {
 	return false
 }
 
-func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	getMin := func(a, b float64) float64 {
@@ -1429,13 +1430,13 @@ func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMin)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMin)
 	qt.Printf("resulting series=%d; ok=%v", len(tss), ok)
 	return tss, ok
 }
 
-func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	getMax := func(a, b float64) float64 {
@@ -1444,12 +1445,12 @@ func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMax)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMax)
 	qt.Printf("resulting series=%d", len(tss))
 	return tss, ok
 }
 
-func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, f func(a, b float64) float64) ([]*timeseries, bool) {
+func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, timestamp int64, f func(a, b float64) float64) ([]*timeseries, bool) {
 	assertInstantValues(tssCached)
 	assertInstantValues(tssStart)
 	assertInstantValues(tssEnd)
@@ -1500,12 +1501,16 @@ func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, f func(a,
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
+
+	setInstantTimestamp(rvs, timestamp)
+
 	return rvs, true
 }
 
-// getSumInstantValues calculates tssCached + tssStart - tssEnd
-func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) []*timeseries {
-	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+// getSumInstantValues aggregates tssCached, tssStart, tssEnd time series
+// into a new time series with value = tssCached + tssStart - tssEnd
+func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) []*timeseries {
+	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	assertInstantValues(tssCached)
@@ -1550,10 +1555,19 @@ func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
+
+	setInstantTimestamp(rvs, timestamp)
+
 	qt.Printf("resulting series=%d", len(rvs))
 	return rvs
 }
 
+func setInstantTimestamp(tss []*timeseries, timestamp int64) {
+	for _, ts := range tss {
+		ts.Timestamps[0] = timestamp
+	}
+}
+
 func assertInstantValues(tss []*timeseries) {
 	for _, ts := range tss {
 		if len(ts.Values) != 1 {
@@ -1681,9 +1695,6 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	} else {
 		minTimestamp -= ec.Step
 	}
-	if minTimestamp < 0 {
-		minTimestamp = 0
-	}
 	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss, ec.MaxSeries)
 	rss, err := netstorage.ProcessSearchQuery(qt, sq, ec.Deadline)
 	if err != nil {

app/vmselect/promql/eval_test.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -95,3 +96,77 @@ func TestQueryStats_addSeriesFetched(t *testing.T) {
 		t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched)
 	}
 }
+
+func TestGetSumInstantValues(t *testing.T) {
+	f := func(cached, start, end []*timeseries, timestamp int64, expectedResult []*timeseries) {
+		t.Helper()
+
+		result := getSumInstantValues(nil, cached, start, end, timestamp)
+		if !reflect.DeepEqual(result, expectedResult) {
+			t.Errorf("unexpected result; got\n%v\nwant\n%v", result, expectedResult)
+		}
+	}
+	ts := func(name string, timestamp int64, value float64) *timeseries {
+		return &timeseries{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte(name),
+			},
+			Timestamps: []int64{timestamp},
+			Values:     []float64{value},
+		}
+	}
+
+	// start - end + cached = 1
+	f(
+		nil,
+		[]*timeseries{ts("foo", 42, 1)},
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+
+	// start - end + cached = 0
+	f(
+		nil,
+		[]*timeseries{ts("foo", 100, 1)},
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 0)},
+	)
+
+	// start - end + cached = 2
+	f(
+		[]*timeseries{ts("foo", 10, 1)},
+		[]*timeseries{ts("foo", 100, 1)},
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 2)},
+	)
+
+	// start - end + cached = 1
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		[]*timeseries{ts("foo", 100, 1)},
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+
+	// start - end + cached = 0
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		nil,
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 0)},
+	)
+
+	// start - end + cached = 1
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		nil,
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+}

app/vmselect/promql/exec_test.go

@@ -3706,7 +3706,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `(
 		  (label_set(time(), "t1", "v1", "__name__", "q1") or label_set(10, "t2", "v2", "__name__", "q2"))
 		    +
-		  (label_set(100, "t1", "v1", "__name__", "q3") or label_set(time(), "t2", "v3"))
+		  (label_set(100, "t1", "v1", "__name__", "q1") or label_set(time(), "t2", "v3"))
 		) keep_metric_names`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -6108,6 +6108,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`sum_gt_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_gt_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5.9, 5.2, 8.5, 5.1, 4.9, 4.5},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum_le_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_le_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4.2, 4.9, 3.2, 5.8, 4.1, 5.3},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum_eq_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_eq_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increases_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `increases_over_time(rand(0)[200s:10s])`
@@ -8050,10 +8083,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate()`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s])`
+		q := `rollup_rate((2200-time())[600s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -8062,7 +8095,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -8071,7 +8104,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{4, 3, 2, 1, 0, -1},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
@@ -8083,10 +8116,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "max")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "max")`
+		q := `rollup_rate((2200-time())[600s], "max")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -8094,10 +8127,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "avg")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "avg")`
+		q := `rollup_rate((2200-time())[600s], "avg")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}

app/vmselect/promql/rollup.go

@@ -79,12 +79,15 @@ var rollupFuncs = map[string]newRollupFunc{
 	"rollup_rate":             newRollupFuncOneOrTwoArgs(rollupFake), // + rollupFuncsRemoveCounterResets
 	"rollup_scrape_interval":  newRollupFuncOneOrTwoArgs(rollupFake),
 	"scrape_interval":         newRollupFuncOneArg(rollupScrapeInterval),
+	"share_eq_over_time":      newRollupShareEQ,
 	"share_gt_over_time":      newRollupShareGT,
 	"share_le_over_time":      newRollupShareLE,
-	"share_eq_over_time":      newRollupShareEQ,
 	"stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples),
 	"stddev_over_time":        newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":        newRollupFuncOneArg(rollupStdvar),
+	"sum_eq_over_time":        newRollupSumEQ,
+	"sum_gt_over_time":        newRollupSumGT,
+	"sum_le_over_time":        newRollupSumLE,
 	"sum_over_time":           newRollupFuncOneArg(rollupSum),
 	"sum2_over_time":          newRollupFuncOneArg(rollupSum2),
 	"tfirst_over_time":        newRollupFuncOneArg(rollupTfirst),
@@ -859,6 +862,11 @@ func removeCounterResets(values []float64) {
 		}
 		prevValue = v
 		values[i] = v + correction
+		// Check again, there could be precision error in float operations,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571
+		if i > 0 && values[i] < values[i-1] {
+			values[i] = values[i-1]
+		}
 	}
 }
 
@@ -1084,59 +1092,89 @@ func newRollupDurationOverTime(args []interface{}) (rollupFunc, error) {
 }
 
 func newRollupShareLE(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterLE)
+	return newRollupAvgFilter(args, countFilterLE)
 }
 
-func countFilterLE(values []float64, le float64) int {
+func countFilterLE(values []float64, le float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v <= le {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
 func newRollupShareGT(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterGT)
+	return newRollupAvgFilter(args, countFilterGT)
 }
 
-func countFilterGT(values []float64, gt float64) int {
+func countFilterGT(values []float64, gt float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v > gt {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
 func newRollupShareEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterEQ)
+	return newRollupAvgFilter(args, countFilterEQ)
 }
 
-func countFilterEQ(values []float64, eq float64) int {
+func sumFilterEQ(values []float64, eq float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v == eq {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func sumFilterLE(values []float64, le float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v <= le {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func sumFilterGT(values []float64, gt float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v > gt {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func countFilterEQ(values []float64, eq float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v == eq {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
-func countFilterNE(values []float64, ne float64) int {
+func countFilterNE(values []float64, ne float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v != ne {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
-func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
-	rf, err := newRollupCountFilter(args, countFilter)
+func newRollupAvgFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
+	rf, err := newRollupFilter(args, f)
 	if err != nil {
 		return nil, err
 	}
@@ -1146,23 +1184,35 @@ func newRollupShareFilter(args []interface{}, countFilter func(values []float64,
 	}, nil
 }
 
+func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, countFilterEQ)
+}
+
 func newRollupCountLE(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterLE)
+	return newRollupFilter(args, countFilterLE)
 }
 
 func newRollupCountGT(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterGT)
-}
-
-func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterEQ)
+	return newRollupFilter(args, countFilterGT)
 }
 
 func newRollupCountNE(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterNE)
+	return newRollupFilter(args, countFilterNE)
 }
 
-func newRollupCountFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
+func newRollupSumEQ(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterEQ)
+}
+
+func newRollupSumLE(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterLE)
+}
+
+func newRollupSumGT(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterGT)
+}
+
+func newRollupFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 2); err != nil {
 		return nil, err
 	}
@@ -1178,7 +1228,7 @@ func newRollupCountFilter(args []interface{}, countFilter func(values []float64,
 			return nan
 		}
 		limit := limits[rfa.idx]
-		return float64(countFilter(values, limit))
+		return f(values, limit)
 	}
 	return rf, nil
 }
@@ -1911,6 +1961,10 @@ func rollupChangesPrometheus(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values[1:] {
 		if v != prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 			prevValue = v
 		}
@@ -1934,6 +1988,10 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 	}
 	for _, v := range values {
 		if v != prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 			prevValue = v
 		}
@@ -1962,6 +2020,10 @@ func rollupIncreases(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v > prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 		}
 		prevValue = v
@@ -1993,6 +2055,10 @@ func rollupResets(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v < prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 		}
 		prevValue = v

app/vmselect/promql/rollup_result_cache.go

@@ -30,6 +30,8 @@ var (
 		"due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset")
 	disableAutoCacheReset = flag.Bool("search.disableAutoCacheReset", false, "Whether to disable automatic response cache reset if a sample with timestamp "+
 		"outside -search.cacheTimestampOffset is inserted into VictoriaMetrics")
+	resetRollupResultCacheOnStartup = flag.Bool("search.resetRollupResultCacheOnStartup", false, "Whether to reset rollup result cache on startup. "+
+		"See https://docs.victoriametrics.com/#rollup-result-cache . See also -search.disableCache")
 )
 
 // ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
@@ -43,6 +45,11 @@ func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
 		rollupResultResetMetricRowSample.Store(&storage.MetricRow{})
 		go checkRollupResultCacheReset()
 	})
+	if atomic.LoadUint32(&needRollupResultCacheReset) != 0 {
+		// The cache has been already instructed to reset.
+		return
+	}
+
 	minTimestamp := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds() + checkRollupResultCacheResetInterval.Milliseconds()
 	needCacheReset := false
 	for i := range mrs {
@@ -112,7 +119,12 @@ func InitRollupResultCache(cachePath string) {
 	cacheSize := getRollupResultCacheSize()
 	var c *workingsetcache.Cache
 	if len(rollupResultCachePath) > 0 {
-		logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
+		if *resetRollupResultCacheOnStartup {
+			logger.Infof("removing rollupResult cache at %q becasue -search.resetRollupResultCacheOnStartup command-line flag is set", rollupResultCachePath)
+			fs.MustRemoveAll(rollupResultCachePath)
+		} else {
+			logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
+		}
 		c = workingsetcache.Load(rollupResultCachePath, cacheSize)
 		mustLoadRollupResultCacheKeyPrefix(rollupResultCachePath)
 	} else {

app/vmselect/promql/rollup_test.go

@@ -125,7 +125,7 @@ func TestRemoveCounterResets(t *testing.T) {
 	// removeCounterResets doesn't expect negative values, so it doesn't work properly with them.
 	values = []float64{-100, -200, -300, -400}
 	removeCounterResets(values)
-	valuesExpected = []float64{-100, -300, -600, -1000}
+	valuesExpected = []float64{-100, -100, -100, -100}
 	timestampsExpected := []int64{0, 1, 2, 3}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
@@ -136,6 +136,17 @@ func TestRemoveCounterResets(t *testing.T) {
 	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
+
+	// verify results always increase monotonically with possible float operations precision error
+	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
+	removeCounterResets(values)
+	var prev float64
+	for i, v := range values {
+		if v < prev {
+			t.Fatalf("error: unexpected value keep getting bigger %d; cur %v; pre %v\n", i, v, prev)
+		}
+		prev = v
+	}
 }
 
 func TestDeltaValues(t *testing.T) {
@@ -396,6 +407,75 @@ func TestRollupCountNEOverTime(t *testing.T) {
 	f(12, 11)
 }
 
+func TestRollupSumLEOverTime(t *testing.T) {
+	f := func(le, vExpected float64) {
+		t.Helper()
+		les := []*timeseries{{
+			Values:     []float64{le},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
+		testRollupFunc(t, "sum_le_over_time", args, vExpected)
+	}
+
+	f(-123, 0)
+	f(0, 0)
+	f(10, 0)
+	f(12, 12)
+	f(30, 33)
+	f(50, 289)
+	f(100, 442)
+	f(123, 565)
+	f(1000, 565)
+}
+
+func TestRollupSumGTOverTime(t *testing.T) {
+	f := func(le, vExpected float64) {
+		t.Helper()
+		les := []*timeseries{{
+			Values:     []float64{le},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
+		testRollupFunc(t, "sum_gt_over_time", args, vExpected)
+	}
+
+	f(-123, 565)
+	f(0, 565)
+	f(10, 565)
+	f(12, 553)
+	f(30, 532)
+	f(50, 276)
+	f(100, 123)
+	f(123, 0)
+	f(1000, 0)
+}
+
+func TestRollupSumEQOverTime(t *testing.T) {
+	f := func(le, vExpected float64) {
+		t.Helper()
+		les := []*timeseries{{
+			Values:     []float64{le},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
+		testRollupFunc(t, "sum_eq_over_time", args, vExpected)
+	}
+
+	f(-123, 0)
+	f(0, 0)
+	f(10, 0)
+	f(12, 12)
+	f(30, 0)
+	f(50, 0)
+	f(100, 0)
+	f(123, 123)
+	f(1000, 0)
+}
+
 func TestRollupQuantileOverTime(t *testing.T) {
 	f := func(phi, vExpected float64) {
 		t.Helper()

app/vmselect/vmui.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
@@ -14,6 +15,8 @@ import (
 var (
 	vmuiCustomDashboardsPath = flag.String("vmui.customDashboardsPath", "", "Optional path to vmui dashboards. "+
 		"See https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards")
+	vmuiDefaultTimezone = flag.String("vmui.defaultTimezone", "", "The default timezone to be used in vmui. "+
+		"Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local")
 )
 
 // dashboardSettings represents dashboard settings file struct.
@@ -65,6 +68,16 @@ func handleVMUICustomDashboards(w http.ResponseWriter) error {
 	return nil
 }
 
+func handleVMUITimezone(w http.ResponseWriter) error {
+	tz, err := time.LoadLocation(*vmuiDefaultTimezone)
+	if err != nil {
+		return fmt.Errorf("cannot load timezone %q: %w", *vmuiDefaultTimezone, err)
+	}
+	response := fmt.Sprintf(`{"timezone": %q}`, tz)
+	writeSuccessResponse(w, []byte(response))
+	return nil
+}
+
 func writeSuccessResponse(w http.ResponseWriter, data []byte) {
 	w.WriteHeader(http.StatusOK)
 	w.Header().Set("Content-Type", "application/json")

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.fb353c1e.css",
-    "main.js": "./static/js/main.5bcddddc.js",
+    "main.css": "./static/css/main.be4fee7a.css",
+    "main.js": "./static/js/main.fd9d9e16.js",
     "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.b64c4dbf91f4fa581621.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8a01ddf56e4e6bc1ccf1.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.fb353c1e.css",
-    "static/js/main.5bcddddc.js"
+    "static/css/main.be4fee7a.css",
+    "static/js/main.fd9d9e16.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.5bcddddc.js"></script><link href="./static/css/main.fb353c1e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.fd9d9e16.js"></script><link href="./static/css/main.be4fee7a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmselect/vmui/static/media/MetricsQL.8a01ddf56e4e6bc1ccf1.md

@@ -70,7 +70,7 @@ The list of MetricsQL features on top of PromQL:
   It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
 * Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
-  selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
+  selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
 * Support for `group_left(*)` and `group_right(*)` for copying all the labels from time series on the `one` side
   of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches).
@@ -243,7 +243,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_eq_over_time](#share_eq_over_time).
 
 #### count_gt_over_time
 
@@ -253,7 +253,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
 
@@ -263,7 +263,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
 
@@ -743,7 +743,7 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_le_over_time](#share_le_over_time).
+See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
 
@@ -756,7 +756,7 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_gt_over_time](#share_gt_over_time).
+See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
 
@@ -766,6 +766,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_eq_over_time](#count_eq_over_time).
+
 #### stale_samples_over_time
 
 `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
@@ -792,6 +794,33 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
 
+#### sum_eq_over_time
+
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values equal to `eq`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
+
+#### sum_gt_over_time
+
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values bigger than `gt`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
+
+#### sum_le_over_time
+
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values smaller or equal to `le`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
+
 #### sum_over_time
 
 `sum_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the sum of raw sample values
@@ -810,7 +839,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -819,7 +848,7 @@ This function is supported by PromQL. See also [timestamp_with_name](#timestamp_
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
@@ -828,7 +857,7 @@ See also [timestamp](#timestamp).
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +866,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +881,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +891,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -1029,7 +1058,7 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. This function is supported by PromQL. See also [acosh](#acosh).
+This function is supported by PromQL. See also [acosh](#acosh).
 
 #### day_of_month
 
@@ -1049,6 +1078,15 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+#### day_of_year
+
+`day_of_year(q)` is a [transform function](#transform-functions), which returns the day of year for every point of every time series returned by `q`.
+It is expected that `q` returns unix timestamps. The returned values are in the range `[1...365]` for non-leap years, and `[1 to 366]` in leap years.
+
+Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+This function is supported by PromQL.
+
 #### days_in_month
 
 `days_in_month(q)` is a [transform function](#transform-functions), which returns the number of days in the month identified

app/vmstorage/main.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -20,14 +22,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
-	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 var (
 	retentionPeriod       = flagutil.NewDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
-	snapshotAuthKey       = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
-	forceMergeAuthKey     = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
-	forceFlushAuthKey     = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages")
+	snapshotAuthKey       = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages")
+	forceMergeAuthKey     = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages")
+	forceFlushAuthKey     = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages")
 	snapshotsMaxAge       = flagutil.NewDuration("snapshotsMaxAge", "0", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
 	snapshotCreateTimeout = flag.Duration("snapshotCreateTimeout", 0, "The timeout for creating new snapshot. If set, make sure that timeout is lower than backup period")
 
@@ -36,13 +38,10 @@ var (
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
-	finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
-		"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
-		"Zero value disables final merge")
-	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing. Please use -smallMergeConcurrency "+
-		"for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage")
-	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . "+
-		"It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries")
+	_ = flag.Duration("finalMergeDelay", 0, "Deprecated: this flag does nothing")
+	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing")
+	_ = flag.Int("smallMergeConcurrency", 0, "Deprecated: this flag does nothing")
+
 	retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+
 		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
 		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
@@ -94,8 +93,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
 	storage.SetLogNewSeries(*logNewSeries)
-	storage.SetFinalMergeDelay(*finalMergeDelay)
-	storage.SetMergeWorkersCount(*smallMergeConcurrency)
 	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
@@ -259,7 +256,7 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, *forceMergeAuthKey, "forceMergeAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey.Get(), "forceMergeAuthKey") {
 			return true
 		}
 		// Run force merge in background
@@ -277,7 +274,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if path == "/internal/force_flush" {
-		if !httpserver.CheckAuthFlag(w, r, *forceFlushAuthKey, "forceFlushAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey.Get(), "forceFlushAuthKey") {
 			return true
 		}
 		logger.Infof("flushing storage to make pending data available for reading")
@@ -293,7 +290,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if !strings.HasPrefix(path, "/snapshot") {
 		return false
 	}
-	if !httpserver.CheckAuthFlag(w, r, *snapshotAuthKey, "snapshotAuthKey") {
+	if !httpserver.CheckAuthFlag(w, r, snapshotAuthKey.Get(), "snapshotAuthKey") {
 		return true
 	}
 	path = path[len("/snapshot"):]
@@ -400,7 +397,8 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverWG.Add(1)
 	go func() {
 		defer staleSnapshotsRemoverWG.Done()
-		t := time.NewTicker(11 * time.Second)
+		d := timeutil.AddJitterToDuration(time.Second * 11)
+		t := time.NewTicker(d)
 		defer t.Stop()
 		for {
 			select {
@@ -493,11 +491,8 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_composite_filter_success_conversions_total`, idbm.CompositeFilterSuccessConversions)
 	metrics.WriteCounterUint64(w, `vm_composite_filter_missing_conversions_total`, idbm.CompositeFilterMissingConversions)
 
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/inmemory"}`, tm.InmemoryAssistedMerges)
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/small"}`, tm.SmallAssistedMerges)
-
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryAssistedMerges)
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/file"}`, idbm.FileAssistedMerges)
+	// vm_assisted_merges_total name is used for backwards compatibility.
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryPartsLimitReachedCount)
 
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_total`, idbm.ItemsAdded)
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_size_bytes_total`, idbm.ItemsAddedSizeBytes)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.21.6 as build-web-stage
+FROM golang:1.22.0 as build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/README.md

@@ -7,6 +7,7 @@ Web UI for VictoriaMetrics
 * [Updating vmui embedded into VictoriaMetrics](#updating-vmui-embedded-into-victoriametrics)
 * [Predefined dashboards](#predefined-dashboards)
 * [App mode config options](#app-mode-config-options)
+* [Timezone configuration](#timezone-configuration)
 
 ----
 
@@ -246,3 +247,39 @@ vmui can be used to paste into other applications
 ```html
 <div id="root" data-params='{"serverURL":"http://localhost:8428","useTenantID":true,"headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
 ```
+
+----
+
+## Timezone configuration
+
+vmui's timezone setting offers flexibility in displaying time data. It can be set through a configuration flag and is adjustable within the vmui interface. This feature caters to various user preferences and time zones.
+
+### Default Timezone Setting
+
+#### Via Configuration Flag
+
+- Set the default timezone using the `--vmui.defaultTimezone` flag.
+- Accepts a valid IANA Time Zone string (e.g., `America/New_York`, `Europe/Berlin`, `Etc/GMT+3`).
+- If the flag is unset or invalid, vmui defaults to the browser's local timezone.
+
+#### User Interface Adjustments
+
+- Users can change the timezone in the vmui interface.
+- Any changed setting in the interface overrides the flag's default, persisting for the user.
+- The timezone specified in the `--vmui.defaultTimezone` flag is included in the vmui's timezone selection dropdown, aiding user choice.
+
+### Key Points
+
+- **Fallback to Browser's Local Timezone**: If the flag is not set or an invalid timezone is specified, vmui uses the local timezone of the user's browser.
+- **User Preference Priority**: User-selected timezones in vmui take precedence over the default set by the flag.
+- **Cluster Consistency**: Ensure uniform timezone settings across cluster nodes, but individual user interface selections will always override these defaults.
+
+### Examples
+
+Setting a default timezone, with user options to change:
+
+```
+./victoria-metrics --vmui.defaultTimezone="America/New_York"
+```
+
+In this scenario, if a user in Berlin accesses vmui without changing settings, it will default to their browser's local timezone (CET). If they select a different timezone in vmui, this choice will override the `"America/New_York"` setting for that user.

app/vmui/packages/vmui/src/App.tsx

@@ -14,6 +14,7 @@ import PreviewIcons from "./components/Main/Icons/PreviewIcons";
 import WithTemplate from "./pages/WithTemplate";
 import Relabel from "./pages/Relabel";
 import ActiveQueries from "./pages/ActiveQueries";
+import QueryAnalyzer from "./pages/QueryAnalyzer";
 
 const App: FC = () => {
   const [loadedTheme, setLoadedTheme] = useState(false);
@@ -49,6 +50,10 @@ const App: FC = () => {
                   path={router.trace}
                   element={<TracePage/>}
                 />
+                <Route
+                  path={router.queryAnalyzer}
+                  element={<QueryAnalyzer/>}
+                />
                 <Route
                   path={router.dashboards}
                   element={<DashboardsLayout/>}

app/vmui/packages/vmui/src/api/query-range.ts

@@ -4,4 +4,4 @@ export const getQueryRangeUrl = (server: string, query: string, period: TimePara
   `${server}/api/v1/query_range?query=${encodeURIComponent(query)}&start=${period.start}&end=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
 
 export const getQueryUrl = (server: string, query: string, period: TimeParams, nocache: boolean, queryTracing: boolean): string =>
-  `${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
+  `${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -70,7 +70,7 @@ The list of MetricsQL features on top of PromQL:
   It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
 * Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
-  selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
+  selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
 * Support for `group_left(*)` and `group_right(*)` for copying all the labels from time series on the `one` side
   of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches).
@@ -243,7 +243,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_eq_over_time](#share_eq_over_time).
 
 #### count_gt_over_time
 
@@ -253,7 +253,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
 
@@ -263,7 +263,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
 
@@ -743,7 +743,7 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_le_over_time](#share_le_over_time).
+See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
 
@@ -756,7 +756,7 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_gt_over_time](#share_gt_over_time).
+See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
 
@@ -766,6 +766,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_eq_over_time](#count_eq_over_time).
+
 #### stale_samples_over_time
 
 `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
@@ -792,6 +794,33 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
 
+#### sum_eq_over_time
+
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values equal to `eq`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
+
+#### sum_gt_over_time
+
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values bigger than `gt`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
+
+#### sum_le_over_time
+
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values smaller or equal to `le`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
+
 #### sum_over_time
 
 `sum_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the sum of raw sample values
@@ -810,7 +839,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -819,7 +848,7 @@ This function is supported by PromQL. See also [timestamp_with_name](#timestamp_
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
@@ -828,7 +857,7 @@ See also [timestamp](#timestamp).
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +866,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +881,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +891,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -29,7 +29,7 @@ const GlobalSettings: FC = () => {
 
   const appModeEnable = getAppModeEnable();
   const { serverUrl: stateServerUrl, theme } = useAppState();
-  const { timezone: stateTimezone } = useTimeState();
+  const { timezone: stateTimezone, defaultTimezone } = useTimeState();
   const { seriesLimits } = useCustomPanelState();
 
   const dispatch = useAppDispatch();
@@ -78,6 +78,10 @@ const GlobalSettings: FC = () => {
     setServerUrl(stateServerUrl);
   }, [stateServerUrl]);
 
+  useEffect(() => {
+    setTimezone(stateTimezone);
+  }, [stateTimezone]);
+
   const controls = [
     {
       show: !appModeEnable && !isLogsApp,
@@ -100,6 +104,7 @@ const GlobalSettings: FC = () => {
       show: true,
       component: <Timezones
         timezoneState={timezone}
+        defaultTimezone={defaultTimezone}
         onChange={setTimezone}
       />
     },

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -51,6 +51,12 @@ const ServerConfigurator: FC<ServerConfiguratorProps> = ({
     }
   }, [enabledStorage]);
 
+  useEffect(() => {
+    if (enabledStorage) {
+      saveToStorage("SERVER_URL", serverUrl);
+    }
+  }, [serverUrl]);
+
   return (
     <div>
       <div className="vm-server-configurator__title">

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/Timezones.tsx

@@ -1,22 +1,30 @@
 import React, { FC, useMemo, useRef, useState } from "preact/compat";
-import { getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
+import { getBrowserTimezone, getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
 import { ArrowDropDownIcon } from "../../../Main/Icons";
 import classNames from "classnames";
 import Popper from "../../../Main/Popper/Popper";
 import Accordion from "../../../Main/Accordion/Accordion";
-import dayjs from "dayjs";
 import TextField from "../../../Main/TextField/TextField";
 import { Timezone } from "../../../../types";
 import "./style.scss";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import useBoolean from "../../../../hooks/useBoolean";
+import WarningTimezone from "./WarningTimezone";
 
 interface TimezonesProps {
-  timezoneState: string
-  onChange: (val: string) => void
+  timezoneState: string;
+  defaultTimezone?: string;
+  onChange: (val: string) => void;
 }
 
-const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
+interface PinnedTimezone extends Timezone {
+  title: string;
+  isInvalid?: boolean;
+}
+
+const browserTimezone = getBrowserTimezone();
+
+const Timezones: FC<TimezonesProps> = ({ timezoneState, defaultTimezone, onChange }) => {
   const { isMobile } = useDeviceDetect();
   const timezones = getTimezoneList();
 
@@ -29,6 +37,25 @@ const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
     setFalse: handleCloseList,
   } = useBoolean(false);
 
+  const pinnedTimezones = useMemo(() => [
+    {
+      title: `Default time (${defaultTimezone})`,
+      region: defaultTimezone,
+      utc: defaultTimezone ? getUTCByTimezone(defaultTimezone) : "UTC"
+    },
+    {
+      title: browserTimezone.title,
+      region: browserTimezone.region,
+      utc: getUTCByTimezone(browserTimezone.region),
+      isInvalid: !browserTimezone.isValid
+    },
+    {
+      title: "UTC (Coordinated Universal Time)",
+      region: "UTC",
+      utc: "UTC"
+    },
+  ].filter(t => t.region) as PinnedTimezone[], [defaultTimezone]);
+
   const searchTimezones = useMemo(() => {
     if (!search) return timezones;
     try {
@@ -40,11 +67,6 @@ const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
 
   const timezonesGroups = useMemo(() => Object.keys(searchTimezones), [searchTimezones]);
 
-  const localTimezone = useMemo(() => ({
-    region: dayjs.tz.guess(),
-    utc: getUTCByTimezone(dayjs.tz.guess())
-  }), []);
-
   const activeTimezone = useMemo(() => ({
     region: timezoneState,
     utc: getUTCByTimezone(timezoneState)
@@ -108,13 +130,16 @@ const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
                 onChange={handleChangeSearch}
               />
             </div>
-            <div
-              className="vm-timezones-item vm-timezones-list-group-options__item"
-              onClick={createHandlerSetTimezone(localTimezone)}
-            >
-              <div className="vm-timezones-item__title">Browser Time ({localTimezone.region})</div>
-              <div className="vm-timezones-item__utc">{localTimezone.utc}</div>
-            </div>
+            {pinnedTimezones.map((t, i) => t && (
+              <div
+                key={`${i}_${t.region}`}
+                className="vm-timezones-item vm-timezones-list-group-options__item"
+                onClick={createHandlerSetTimezone(t)}
+              >
+                <div className="vm-timezones-item__title">{t.title}{t.isInvalid && <WarningTimezone/>}</div>
+                <div className="vm-timezones-item__utc">{t.utc}</div>
+              </div>
+            ))}
           </div>
           {timezonesGroups.map(t => (
             <div

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/WarningTimezone.tsx

@@ -0,0 +1,16 @@
+import React, { FC } from "preact/compat";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import { WarningIcon } from "../../../Main/Icons";
+
+const waringText = "Browser timezone is not recognized, supported, or could not be determined.";
+
+const WarningTimezone: FC = () => {
+
+  return (
+    <Tooltip title={waringText}>
+      <WarningIcon/>
+    </Tooltip>
+  );
+};
+
+export default WarningTimezone;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/style.scss

@@ -16,7 +16,15 @@
     }
 
     &__title {
+      display: flex;
+      align-items: center;
+      gap: $padding-small;
       text-transform: capitalize;
+
+      svg {
+        width: 14px;
+        color: $color-warning;
+      }
     }
 
     &__utc {

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryAutocompleteCache.ts

@@ -24,7 +24,7 @@ export class QueryAutocompleteCache {
       const equalRange = cacheItem.start === key.start && cacheItem.end === key.end;
       const equalType = cacheItem.type === key.type;
       const isIncluded = key.value && cacheItem.value && key.value.includes(cacheItem.value);
-      const isSimilar = cacheItem.match === key.match || isIncluded;
+      const isSimilar = (cacheItem.match === key.match) || isIncluded;
       const isUnderLimit = cacheValue.length < AUTOCOMPLETE_LIMITS.queryLimit;
       if (isSimilar && equalRange && equalType && isUnderLimit) {
         return cacheValue;

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx

@@ -1,5 +1,5 @@
-import React, { FC, useRef, useState } from "preact/compat";
-import { KeyboardEvent, useEffect } from "react";
+import React, { FC, useEffect, useRef, useState } from "preact/compat";
+import { KeyboardEvent } from "react";
 import { ErrorTypes } from "../../../types";
 import TextField from "../../Main/TextField/TextField";
 import QueryEditorAutocomplete from "./QueryEditorAutocomplete";
@@ -7,7 +7,8 @@ import "./style.scss";
 import { QueryStats } from "../../../api/types";
 import { partialWarning, seriesFetchedWarning } from "./warningText";
 import { AutocompleteOptions } from "../../Main/Autocomplete/Autocomplete";
-import { useQueryDispatch } from "../../../state/query/QueryStateContext";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
+import { useQueryState } from "../../../state/query/QueryStateContext";
 
 export interface QueryEditorProps {
   onChange: (query: string) => void;
@@ -35,11 +36,12 @@ const QueryEditor: FC<QueryEditorProps> = ({
   label,
   disabled = false
 }) => {
+  const { autocompleteQuick } = useQueryState();
+  const { isMobile } = useDeviceDetect();
 
   const [openAutocomplete, setOpenAutocomplete] = useState(false);
   const [caretPosition, setCaretPosition] = useState([0, 0]);
   const autocompleteAnchorEl = useRef<HTMLInputElement>(null);
-  const queryDispatch = useQueryDispatch();
 
   const warning = [
     {
@@ -103,8 +105,8 @@ const QueryEditor: FC<QueryEditorProps> = ({
   };
 
   useEffect(() => {
-    queryDispatch({ type: "SET_AUTOCOMPLETE_QUICK", payload: false });
-  }, [value]);
+    setOpenAutocomplete(autocomplete);
+  }, [autocompleteQuick]);
 
   return (
     <div
@@ -115,7 +117,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
         value={value}
         label={label}
         type={"textarea"}
-        autofocus={!!value}
+        autofocus={!isMobile}
         error={error}
         warning={warning}
         onKeyDown={handleKeyDown}

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -4,9 +4,8 @@ import { useFetchQueryOptions } from "../../../hooks/useFetchQueryOptions";
 import { getTextWidth } from "../../../utils/uplot";
 import { escapeRegexp } from "../../../utils/regexp";
 import useGetMetricsQL from "../../../hooks/useGetMetricsQL";
-import { RefreshIcon } from "../../Main/Icons";
 import { QueryContextType } from "../../../types";
-import { AUTOCOMPLETE_LIMITS, AUTOCOMPLETE_MIN_SYMBOLS } from "../../../constants/queryAutocomplete";
+import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
 
 interface QueryEditorAutocompleteProps {
   value: string;
@@ -26,22 +25,27 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   const [leftOffset, setLeftOffset] = useState(0);
   const metricsqlFunctions = useGetMetricsQL();
 
+  const exprLastPart = useMemo(() => {
+    const parts = value.split("}");
+    return parts[parts.length - 1];
+  }, [value]);
+
   const metric = useMemo(() => {
     const regexp = /\b[^{}(),\s]+(?={|$)/g;
-    const match = value.match(regexp);
+    const match = exprLastPart.match(regexp);
     return match ? match[0] : "";
-  }, [value]);
+  }, [exprLastPart]);
 
   const label = useMemo(() => {
     const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
-    const match = value.match(regexp);
+    const match = exprLastPart.match(regexp);
     return match ? match[match.length - 1] : "";
-  }, [value]);
+  }, [exprLastPart]);
 
   const context = useMemo(() => {
-    if (!value) return QueryContextType.empty;
+    if (!value || value.endsWith("}")) return QueryContextType.empty;
 
-    const labelRegexp = /\{[^}]*?(\w+)$/gm;
+    const labelRegexp = /\{[^}]*?(\w+)*$/gm;
     const labelValueRegexp = new RegExp(`(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`, "g");
 
     switch (true) {
@@ -91,8 +95,8 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     // Add quotes around the value if the context is labelValue
     if (context === QueryContextType.labelValue) {
       const quote = "\"";
-      const needsQuote = !beforeValueByContext.endsWith(quote);
-      insert = `${needsQuote ? quote : ""}${insert}${quote}`;
+      const needsQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
+      insert = `${needsQuote ? quote : ""}${insert}`;
     }
 
     // Assemble the new value with the inserted text
@@ -116,11 +120,12 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   return (
     <>
       <Autocomplete
+        loading={loading}
         disabledFullScreen
         value={valueByContext}
-        options={options?.length < AUTOCOMPLETE_LIMITS.queryLimit ? options : []}
+        options={options}
         anchor={anchorEl}
-        minLength={AUTOCOMPLETE_MIN_SYMBOLS[context]}
+        minLength={0}
         offset={{ top: 0, left: leftOffset }}
         onSelect={handleSelect}
         onFoundOptions={onFoundOptions}
@@ -129,7 +134,6 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
           message: "Please, specify the query more precisely."
         }}
       />
-      {loading && <div className="vm-query-editor-autocomplete"><RefreshIcon/></div>}
     </>
   );
 };

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss

@@ -2,20 +2,4 @@
 
 .vm-query-editor {
   position: relative;
-
-  &-autocomplete {
-    position: absolute;
-    display: flex;
-    align-items: center;
-    justify-items: center;
-    top: 0;
-    bottom: 0;
-    right: $padding-global;
-    width: 12px;
-    height: 100%;
-    color: $color-text-secondary;
-    z-index: 2;
-    animation: half-circle-spinner-animation 1s infinite linear, vm-fade 0.5s ease-in;
-    pointer-events: none;
-  }
 }

app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx

@@ -71,7 +71,7 @@ const ExploreMetricsHeader: FC<ExploreMetricsHeaderProps> = ({
             label="Job"
             placeholder="Please select job"
             onChange={onChangeJob}
-            autofocus={!job}
+            autofocus={!job && !!jobs.length && !isMobile}
             searchable
           />
         </div>

app/vmui/packages/vmui/src/components/Main/Autocomplete/Autocomplete.tsx

@@ -2,7 +2,7 @@ import React, { FC, Ref, useCallback, useEffect, useMemo, useRef, useState, JSX
 import classNames from "classnames";
 import Popper from "../Popper/Popper";
 import "./style.scss";
-import { DoneIcon } from "../Icons";
+import { DoneIcon, RefreshIcon } from "../Icons";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
 import useEventListener from "../../../hooks/useEventListener";
@@ -27,9 +27,11 @@ interface AutocompleteProps {
   disabledFullScreen?: boolean
   offset?: {top: number, left: number}
   maxDisplayResults?: {limit: number, message?: string}
+  loading?: boolean;
   onSelect: (val: string) => void
   onOpenAutocomplete?: (val: boolean) => void
   onFoundOptions?: (val: AutocompleteOptions[]) => void
+  onChangeWrapperRef?: (elementRef: Ref<HTMLDivElement>) => void
 }
 
 enum FocusType {
@@ -50,9 +52,11 @@ const Autocomplete: FC<AutocompleteProps> = ({
   disabledFullScreen,
   offset,
   maxDisplayResults,
+  loading,
   onSelect,
   onOpenAutocomplete,
-  onFoundOptions
+  onFoundOptions,
+  onChangeWrapperRef
 }) => {
   const { isMobile } = useDeviceDetect();
   const wrapperEl = useRef<HTMLDivElement>(null);
@@ -85,6 +89,10 @@ const Autocomplete: FC<AutocompleteProps> = ({
     }
   }, [openAutocomplete, options, value]);
 
+  const hideFoundedOptions = useMemo(() => {
+    return foundOptions.length === 1 && foundOptions[0]?.value === value;
+  }, [foundOptions]);
+
   const displayNoOptionsText = useMemo(() => {
     return noOptionsText && !foundOptions.length;
   }, [noOptionsText,foundOptions]);
@@ -159,8 +167,12 @@ const Autocomplete: FC<AutocompleteProps> = ({
   }, [openAutocomplete]);
 
   useEffect(() => {
-    onFoundOptions && onFoundOptions(foundOptions);
-  }, [foundOptions]);
+    onFoundOptions && onFoundOptions(hideFoundedOptions ? [] : foundOptions);
+  }, [foundOptions, hideFoundedOptions]);
+
+  useEffect(() => {
+    onChangeWrapperRef && onChangeWrapperRef(wrapperEl);
+  }, [wrapperEl]);
 
   return (
     <Popper
@@ -180,8 +192,9 @@ const Autocomplete: FC<AutocompleteProps> = ({
         })}
         ref={wrapperEl}
       >
+        {loading && <div className="vm-autocomplete__loader"><RefreshIcon/><span>Loading...</span></div>}
         {displayNoOptionsText && <div className="vm-autocomplete__no-options">{noOptionsText}</div>}
-        {!(foundOptions.length === 1 && foundOptions[0]?.value === value) && foundOptions.map((option, i) =>
+        {!hideFoundedOptions && foundOptions.map((option, i) =>
           <div
             className={classNames({
               "vm-list-item": true,

app/vmui/packages/vmui/src/components/Main/Autocomplete/style.scss

@@ -16,6 +16,22 @@
     color: $color-text-disabled;
   }
 
+  &__loader {
+    display: grid;
+    grid-template-columns: 14px auto;
+    align-items: center;
+    justify-content: center;
+    gap: $padding-small;
+    padding: $padding-global;
+    color: $color-text-secondary;
+    z-index: 2;
+    pointer-events: none;
+
+    svg {
+      animation: half-circle-spinner-animation 1s infinite linear, vm-fade 0.5s ease-in;
+    }
+  }
+
   &-info,
   &-message {
     padding: $padding-global;

app/vmui/packages/vmui/src/components/Main/Button/Button.tsx

@@ -5,7 +5,7 @@ import "./style.scss";
 
 interface ButtonProps {
   variant?: "contained" | "outlined" | "text"
-  color?: "primary" | "secondary" | "success" | "error" | "gray"  | "warning"
+  color?: "primary" | "secondary" | "success" | "error" | "gray"  | "warning" | "white"
   size?: "small" | "medium" | "large"
   ariaLabel?: string // https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Attributes/aria-label
   endIcon?: ReactNode

app/vmui/packages/vmui/src/components/Main/Button/style.scss

@@ -9,7 +9,7 @@ $button-radius: 6px;
   justify-content: center;
   padding: 6px 14px;
   font-size: $font-size-small;
-  line-height: 1.3;
+  line-height: calc($font-size + 1px);
   font-weight: normal;
   min-height: 31px;
   border-radius: $button-radius;
@@ -56,7 +56,7 @@ $button-radius: 6px;
     transform: translateZ(1px);
 
     svg {
-      width: 15px;
+      width: calc($font-size + 1px);
     }
   }
 
@@ -180,6 +180,10 @@ $button-radius: 6px;
     color: $color-text-secondary;
   }
 
+  &_text_white {
+    color: $color-white;
+  }
+
   &_text_warning {
     color: $color-warning;
   }
@@ -211,6 +215,11 @@ $button-radius: 6px;
     color: $color-text-secondary;
   }
 
+  &_outlined_white {
+    border: 1px solid $color-white;
+    color: $color-white;
+  }
+
   &_outlined_warning {
     border: 1px solid $color-warning;
     color: $color-warning;

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -511,3 +511,12 @@ export const ValueIcon = () => (
     />
   </svg>
 );
+
+export const DownloadIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z"></path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Popper/Popper.tsx

@@ -23,6 +23,7 @@ interface PopperProps {
   fullWidth?: boolean
   title?: string
   disabledFullScreen?: boolean
+  variant?: "default" | "dark"
 }
 
 const Popper: FC<PopperProps> = ({
@@ -35,7 +36,8 @@ const Popper: FC<PopperProps> = ({
   clickOutside = true,
   fullWidth,
   title,
-  disabledFullScreen
+  disabledFullScreen,
+  variant
 }) => {
   const { isMobile } = useDeviceDetect();
   const navigate = useNavigate();
@@ -52,18 +54,16 @@ const Popper: FC<PopperProps> = ({
 
   useEffect(() => {
     setIsOpen(open);
-  }, [open]);
 
-  useEffect(() => {
-    if (!isOpen && onClose) onClose();
-    if (isOpen && isMobile && !disabledFullScreen) {
+    if (!open && onClose) onClose();
+    if (open && isMobile && !disabledFullScreen) {
       document.body.style.overflow = "hidden";
     }
 
     return () => {
       document.body.style.overflow = "auto";
     };
-  }, [isOpen]);
+  }, [open]);
 
   useEffect(() => {
     setPopperSize({
@@ -149,6 +149,7 @@ const Popper: FC<PopperProps> = ({
         <div
           className={classNames({
             "vm-popper": true,
+            [`vm-popper_${variant}`]: variant,
             "vm-popper_mobile": isMobile && !disabledFullScreen,
             "vm-popper_open": (isMobile || Object.keys(popperStyle).length) && isOpen,
           })}
@@ -160,6 +161,7 @@ const Popper: FC<PopperProps> = ({
               <p className="vm-popper-header__title">{title}</p>
               <Button
                 variant="text"
+                color={variant === "dark" ? "white" : "primary"}
                 size="small"
                 onClick={handleClickClose}
                 ariaLabel="close"

app/vmui/packages/vmui/src/components/Main/Popper/style.scss

@@ -49,6 +49,16 @@
       user-select: none;
     }
   }
+
+  &_dark {
+    background-color: $color-background-tooltip;
+    color: $color-white;
+  }
+
+  &_dark &-header {
+    background-color: transparent;
+    color: $color-white;
+  }
 }
 
 @keyframes vm-slider {

app/vmui/packages/vmui/src/components/Main/Select/Select.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useEffect, useMemo, useRef, useState,  } from "preact/compat";
+import React, { FC, Ref, useEffect, useMemo, useRef, useState, } from "preact/compat";
 import classNames from "classnames";
 import { ArrowDropDownIcon, CloseIcon } from "../Icons";
 import { FormEvent, MouseEvent } from "react";
@@ -8,6 +8,7 @@ import "./style.scss";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import MultipleSelectedValue from "./MultipleSelectedValue/MultipleSelectedValue";
 import useEventListener from "../../../hooks/useEventListener";
+import useClickOutside from "../../../hooks/useClickOutside";
 
 interface SelectProps {
   value: string | string[]
@@ -39,6 +40,7 @@ const Select: FC<SelectProps> = ({
 
   const [search, setSearch] = useState("");
   const autocompleteAnchorEl = useRef<HTMLDivElement>(null);
+  const [wrapperRef, setWrapperRef] = useState<Ref<HTMLDivElement> | null>(null);
   const [openList, setOpenList] = useState(false);
 
   const inputRef = useRef<HTMLInputElement>(null);
@@ -76,6 +78,7 @@ const Select: FC<SelectProps> = ({
   };
 
   const handleSelected = (val: string) => {
+    setSearch("");
     onChange(val);
     if (!isMultiple) handleCloseList();
     if (isMultiple && inputRef.current) inputRef.current.focus();
@@ -110,6 +113,7 @@ const Select: FC<SelectProps> = ({
   }, [autofocus, inputRef]);
 
   useEventListener("keyup", handleKeyUp);
+  useClickOutside(autocompleteAnchorEl, handleCloseList, wrapperRef);
 
   return (
     <div
@@ -172,6 +176,7 @@ const Select: FC<SelectProps> = ({
         noOptionsText={noOptionsText}
         onSelect={handleSelected}
         onOpenAutocomplete={setOpenList}
+        onChangeWrapperRef={setWrapperRef}
       />
     </div>
   );

app/vmui/packages/vmui/src/components/TraceQuery/NestedNav/NestedNav.tsx

@@ -14,13 +14,14 @@ interface RecursiveProps {
   isRoot?: boolean;
   trace: Trace;
   totalMsec: number;
+  isExpandedAll? : boolean;
 }
 
 interface OpenLevels {
   [x: number]: boolean
 }
 
-const NestedNav: FC<RecursiveProps> = ({ isRoot, trace, totalMsec })  => {
+const NestedNav: FC<RecursiveProps> = ({ isRoot, trace, totalMsec, isExpandedAll })  => {
   const { isDarkTheme } = useAppState();
   const { isMobile } = useDeviceDetect();
   const [openLevels, setOpenLevels] = useState({} as OpenLevels);
@@ -53,6 +54,26 @@ const NestedNav: FC<RecursiveProps> = ({ isRoot, trace, totalMsec })  => {
     });
   };
 
+  const getIdsFromChildren = (tracingData: Trace) => {
+    const ids = [tracingData.idValue];
+    tracingData?.children?.forEach((child) => {
+      ids.push(...getIdsFromChildren(child));
+    });
+    return ids;
+  };
+
+  useEffect(() => {
+    if (!isExpandedAll) {
+      setOpenLevels([]);
+      return;
+    }
+
+    const allIds = getIdsFromChildren(trace);
+    const openLevels = {} as OpenLevels;
+    allIds.forEach(id => { openLevels[id] = true; });
+    setOpenLevels(openLevels);
+  }, [isExpandedAll]);
+
   return (
     <div
       className={classNames({
@@ -106,13 +127,14 @@ const NestedNav: FC<RecursiveProps> = ({ isRoot, trace, totalMsec })  => {
           )}
         </div>
       </div>
-      {openLevels[trace.idValue] && (
+      {(openLevels[trace.idValue]) && (
         <div className="vm-nested-nav__childrens">
           {hasChildren && trace.children.map((trace) => (
             <NestedNav
               key={trace.duration}
               trace={trace}
               totalMsec={totalMsec}
+              isExpandedAll={isExpandedAll}
             />
           ))}
         </div>

app/vmui/packages/vmui/src/components/TraceQuery/NestedNav/style.scss

@@ -2,6 +2,8 @@
 
 $color-base-nested-nav: $color-tropical-blue;
 $color-base-nested-nav-dark: $color-background-body;
+$width-line: 2px;
+$left-position: calc(-1 * $padding-small);
 
 .vm-nested-nav {
   position: relative;
@@ -48,19 +50,19 @@ $color-base-nested-nav-dark: $color-background-body;
       content: "";
       position: absolute;
       top: calc(50% - 1px);
-      height: 2px;
+      height: $width-line;
       width: $padding-small;
       background-color: $color-base-nested-nav;
-      left: calc(-1 * $padding-small);
+      left: $left-position;
     }
 
     &:before {
       content: "";
       position: absolute;
       bottom: 50%;
-      left: calc(-1 * $padding-small);
+      left: $left-position;
       height: calc(50% + $padding-small);
-      width: 2px;
+      width: $width-line;
       background-color: $color-base-nested-nav;
     }
 
@@ -122,9 +124,9 @@ $color-base-nested-nav-dark: $color-background-body;
       content: "";
       position: absolute;
       top: 0;
-      left: calc(-1 * $padding-small);
+      left: $left-position;
       height: 100%;
-      width: 2px;
+      width: $width-line;
       background-color: $color-base-nested-nav;
     }
   }

app/vmui/packages/vmui/src/components/TraceQuery/TracingsView.tsx

@@ -1,7 +1,7 @@
 import React, { FC, useState } from "preact/compat";
 import Trace from "./Trace";
 import Button from "../Main/Button/Button";
-import { CodeIcon, DeleteIcon } from "../Main/Icons";
+import { ArrowDownIcon, CodeIcon, DeleteIcon, DownloadIcon } from "../Main/Icons";
 import "./style.scss";
 import NestedNav from "./NestedNav/NestedNav";
 import Alert from "../Main/Alert/Alert";
@@ -20,6 +20,7 @@ interface TraceViewProps {
 const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDeleteClick }) => {
   const { isMobile } = useDeviceDetect();
   const [openTrace, setOpenTrace] = useState<Trace | null>(null);
+  const [expandedTraces, setExpandedTraces] = useState<number[]>([]);
 
   const handleCloseJson = () => {
     setOpenTrace(null);
@@ -52,6 +53,27 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
     setOpenTrace(tracingData);
   };
 
+  const handleSaveToFile = (tracingData: Trace) => () => {
+    const blob = new Blob([tracingData.originalJSON], { type: "application/json" });
+    const href = URL.createObjectURL(blob);
+
+    const link = document.createElement("a");
+    link.href = href;
+    link.download = `vmui_trace_${tracingData.queryValue}.json`;
+    document.body.appendChild(link);
+    link.click();
+
+    document.body.removeChild(link);
+    URL.revokeObjectURL(href);
+  };
+
+  const handleExpandAll = (tracingData: Trace) => () => {
+    setExpandedTraces(prev => prev.includes(tracingData.idValue)
+      ? prev.filter(n => n !== tracingData.idValue)
+      : [...prev, tracingData.idValue]
+    );
+  };
+
   return (
     <>
       <div className="vm-tracings-view">
@@ -64,6 +86,28 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
               <h3 className="vm-tracings-view-trace-header-title">
               Trace for <b className="vm-tracings-view-trace-header-title__query">{trace.queryValue}</b>
               </h3>
+              <Tooltip title={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}>
+                <Button
+                  variant="text"
+                  startIcon={(
+                    <div
+                      className={classNames({
+                        "vm-tracings-view-trace-header__expand-icon": true,
+                        "vm-tracings-view-trace-header__expand-icon_open": expandedTraces.includes(trace.idValue) })}
+                    ><ArrowDownIcon/></div>
+                  )}
+                  onClick={handleExpandAll(trace)}
+                  ariaLabel={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}
+                />
+              </Tooltip>
+              <Tooltip title={"Save Trace to JSON"}>
+                <Button
+                  variant="text"
+                  startIcon={<DownloadIcon/>}
+                  onClick={handleSaveToFile(trace)}
+                  ariaLabel="Save trace to JSON"
+                />
+              </Tooltip>
               <Tooltip title={"Open JSON"}>
                 <Button
                   variant="text"
@@ -92,6 +136,7 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
                 isRoot
                 trace={trace}
                 totalMsec={trace.duration}
+                isExpandedAll={expandedTraces.includes(trace.idValue)}
               />
             </nav>
           </div>