docs: consistently use https://docs.victoriametrics.com/lts-releases/ link for LTS releases

See https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.13

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/workflows/check-licenses.yml

@@ -25,7 +25,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
         if: ${{ matrix.language == 'go' }}
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -75,7 +75,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +86,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +100,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -102,7 +102,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -115,6 +115,6 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.txt

.gitignore

@@ -22,3 +22,4 @@ Gemfile.lock
 /_site
 _site
 *.tmp
+/docs/.jekyll-metadata

Makefile

@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
+MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
@@ -178,7 +178,8 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64
+	victoria-metrics-openbsd-amd64 \
+	victoria-metrics-windows-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
@@ -465,7 +466,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.20
+	go mod tidy -compat=1.21
 	go mod vendor
 
 app-local:

SECURITY.md

@@ -2,13 +2,17 @@
 
 ## Supported Versions
 
+The following versions of VictoriaMetrics receive regular security fixes:
+
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.93.x LTS release | :white_check_mark: |
-| v1.87.x LTS release | :white_check_mark: |
+| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
+See [this page](https://victoriametrics.com/security/) for more details.
+
 ## Reporting a Vulnerability
 
 Please report any security issues to security@victoriametrics.com

app/victoria-logs/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -22,11 +21,10 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -34,18 +32,21 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
 
-	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":9428"}
+	}
+	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -53,9 +54,9 @@ func main() {
 	logger.Infof("received signal %s", sig)
 	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-metrics/main.go

@@ -26,8 +26,8 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
@@ -66,7 +66,11 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8428"}
+	}
+	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
@@ -76,7 +80,7 @@ func main() {
 
 	startSelfScraper()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -86,9 +90,9 @@ func main() {
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -124,6 +128,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -38,11 +39,13 @@ const (
 )
 
 const (
-	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
-	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
-	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
+
+	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -55,14 +58,15 @@ var (
 )
 
 type test struct {
-	Name          string   `json:"name"`
-	Data          []string `json:"data"`
-	InsertQuery   string   `json:"insert_query"`
-	Query         []string `json:"query"`
-	ResultMetrics []Metric `json:"result_metrics"`
-	ResultSeries  Series   `json:"result_series"`
-	ResultQuery   Query    `json:"result_query"`
-	Issue         string   `json:"issue"`
+	Name                     string   `json:"name"`
+	Data                     []string `json:"data"`
+	InsertQuery              string   `json:"insert_query"`
+	Query                    []string `json:"query"`
+	ResultMetrics            []Metric `json:"result_metrics"`
+	ResultSeries             Series   `json:"result_series"`
+	ResultQuery              Query    `json:"result_query"`
+	Issue                    string   `json:"issue"`
+	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
 }
 
 type Metric struct {
@@ -180,7 +184,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddr, false, requestHandler)
+	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -226,7 +230,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(*httpListenAddrs); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -260,6 +264,14 @@ func testWrite(t *testing.T) {
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
+	t.Run("csv", func(t *testing.T) {
+		for _, test := range readIn("csv", t, insertionTime) {
+			if test.Data == nil {
+				continue
+			}
+			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
+		}
+	})
 
 	t.Run("influxdb", func(t *testing.T) {
 		for _, x := range readIn("influxdb", t, insertionTime) {
@@ -301,7 +313,7 @@ func testWrite(t *testing.T) {
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
 			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
@@ -312,7 +324,12 @@ func testRead(t *testing.T) {
 						if test.Issue != "" {
 							test.Issue = "\nRegression in " + test.Issue
 						}
-						switch true {
+						switch {
+						case strings.HasPrefix(q, "/api/v1/export/csv"):
+							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
+							if len(data) == test.ExpectedResultLinesCount {
+								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
+							}
 						case strings.HasPrefix(q, "/api/v1/export"):
 							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
 								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
@@ -413,6 +430,7 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
+
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -425,6 +443,20 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
+func httpReadData(t *testing.T, address, query string) []byte {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	s.equalInt(resp.StatusCode, 200)
+	data, err := io.ReadAll(resp.Body)
+	s.noError(err)
+	return data
+}
+
 func checkMetricsResult(got, want []Metric) error {
 	for _, r := range append([]Metric(nil), got...) {
 		want = removeIfFoundMetrics(r, want)
@@ -497,3 +529,73 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
+
+func TestImportJSONLines(t *testing.T) {
+	f := func(labelsCount, labelLen int) {
+		t.Helper()
+
+		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
+		line := generateJSONLine(labelsCount, labelLen)
+		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
+		}
+		if resp.StatusCode != 204 {
+			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
+		}
+	}
+
+	// labels with various lengths
+	for i := 0; i < 500; i++ {
+		f(10, i*5)
+	}
+
+	// Too many labels
+	f(1000, 100)
+
+	// Too long labels
+	f(1, 100_000)
+	f(10, 100_000)
+	f(10, 10_000)
+}
+
+func generateJSONLine(labelsCount, labelLen int) string {
+	m := make(map[string]string, labelsCount)
+	m["__name__"] = generateSizedRandomString(labelLen)
+	for j := 1; j < labelsCount; j++ {
+		labelName := generateSizedRandomString(labelLen)
+		labelValue := generateSizedRandomString(labelLen)
+		m[labelName] = labelValue
+	}
+
+	type jsonLine struct {
+		Metric     map[string]string `json:"metric"`
+		Values     []float64         `json:"values"`
+		Timestamps []int64           `json:"timestamps"`
+	}
+	line := &jsonLine{
+		Metric:     m,
+		Values:     []float64{1.34},
+		Timestamps: []int64{time.Now().UnixNano() / 1e6},
+	}
+	data, err := json.Marshal(&line)
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal JSON: %w", err))
+	}
+	data = append(data, '\n')
+	return string(data)
+}
+
+const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
+
+func generateSizedRandomString(size int) string {
+	dst := make([]byte, size)
+	for i := range dst {
+		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
+	}
+	return string(dst)
+}

app/victoria-metrics/self_scraper.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -49,16 +50,8 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	var currentTimestamp int64
-	for {
-		select {
-		case <-selfScraperStopCh:
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			currentTimestamp = currentTime.UnixNano() / 1e6
-		}
+	f := func(currentTime time.Time, sendStaleMarkers bool) {
+		currentTimestamp := currentTime.UnixNano() / 1e6
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -83,12 +76,27 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			mr.Value = r.Value
+			if sendStaleMarkers {
+				mr.Value = decimal.StaleNaN
+			} else {
+				mr.Value = r.Value
+			}
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
+	for {
+		select {
+		case <-selfScraperStopCh:
+			f(time.Now(), true)
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			f(currentTime, false)
+		}
+	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {

app/victoria-metrics/testdata/csv/basic.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export",
+  "data": [
+    "rfc3339,4,{TIME_MS}",
+    "rfc3339milli,6,{TIME_MS}",
+    "ts,8,{TIME_MS}",
+    "tsms,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
+  "query": [
+    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export with extra_labels",
+  "data": [
+    "location-1,4,{TIME_MS}",
+    "location-2,6,{TIME_MS}",
+    "location-3,8,{TIME_MS}",
+    "location-4,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
+  "query": [
+    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/vlselect/logsql/logsql.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
@@ -17,13 +18,18 @@ var (
 )
 
 // ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}, cancel func()) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
 
 	qStr := r.FormValue("query")
 	q, err := logstorage.ParseQuery(qStr)
@@ -34,7 +40,7 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
 
 	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN())
+	sw.Init(w, maxSortBufferSize.IntN(), limit)
 	tenantIDs := []logstorage.TenantID{tenantID}
 	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
 		if len(columns) == 0 {
@@ -46,7 +52,11 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
 			WriteJSONRow(bb, columns, rowIdx)
 		}
-		sw.MustWrite(bb.B)
+
+		if !sw.TryWrite(bb.B) {
+			cancel()
+		}
+
 		blockResultPool.Put(bb)
 	})
 	sw.FinalFlush()

app/vlselect/logsql/sort_writer.go

@@ -36,8 +36,12 @@ var sortWriterPool sync.Pool
 // If the buf isn't empty at FinalFlush() call, then the buffered data
 // is sorted by _time field.
 type sortWriter struct {
-	mu         sync.Mutex
-	w          io.Writer
+	mu sync.Mutex
+	w  io.Writer
+
+	maxLines     int
+	linesWritten int
+
 	maxBufLen  int
 	buf        []byte
 	bufFlushed bool
@@ -47,58 +51,121 @@ type sortWriter struct {
 
 func (sw *sortWriter) reset() {
 	sw.w = nil
+
+	sw.maxLines = 0
+	sw.linesWritten = 0
+
 	sw.maxBufLen = 0
 	sw.buf = sw.buf[:0]
 	sw.bufFlushed = false
 	sw.hasErr = false
 }
 
-func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
+// Init initializes sw.
+//
+// If maxLines is set to positive value, then sw accepts up to maxLines
+// and then rejects all the other lines by returning false from TryWrite.
+func (sw *sortWriter) Init(w io.Writer, maxBufLen, maxLines int) {
 	sw.reset()
 
 	sw.w = w
 	sw.maxBufLen = maxBufLen
+	sw.maxLines = maxLines
 }
 
-func (sw *sortWriter) MustWrite(p []byte) {
+// TryWrite writes p to sw.
+//
+// True is returned on successful write, false otherwise.
+//
+// Unsuccessful write may occur on underlying write error or when maxLines lines are already written to sw.
+func (sw *sortWriter) TryWrite(p []byte) bool {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
 
 	if sw.hasErr {
-		return
+		return false
 	}
 
 	if sw.bufFlushed {
-		if _, err := sw.w.Write(p); err != nil {
+		if !sw.writeToUnderlyingWriterLocked(p) {
 			sw.hasErr = true
+			return false
 		}
-		return
+		return true
 	}
+
 	if len(sw.buf)+len(p) < sw.maxBufLen {
 		sw.buf = append(sw.buf, p...)
-		return
+		return true
 	}
+
 	sw.bufFlushed = true
-	if len(sw.buf) > 0 {
-		if _, err := sw.w.Write(sw.buf); err != nil {
-			sw.hasErr = true
-			return
+	if !sw.writeToUnderlyingWriterLocked(sw.buf) {
+		sw.hasErr = true
+		return false
+	}
+	sw.buf = sw.buf[:0]
+
+	if !sw.writeToUnderlyingWriterLocked(p) {
+		sw.hasErr = true
+		return false
+	}
+	return true
+}
+
+func (sw *sortWriter) writeToUnderlyingWriterLocked(p []byte) bool {
+	if len(p) == 0 {
+		return true
+	}
+	if sw.maxLines > 0 {
+		if sw.linesWritten >= sw.maxLines {
+			return false
 		}
-		sw.buf = sw.buf[:0]
+		var linesLeft int
+		p, linesLeft = trimLines(p, sw.maxLines-sw.linesWritten)
+		println("DEBUG: end trimLines", string(p), linesLeft)
+		sw.linesWritten += linesLeft
 	}
 	if _, err := sw.w.Write(p); err != nil {
-		sw.hasErr = true
+		return false
 	}
+	return true
 }
 
+func trimLines(p []byte, maxLines int) ([]byte, int) {
+	println("DEBUG: start trimLines", string(p), maxLines)
+	if maxLines <= 0 {
+		return nil, 0
+	}
+	n := bytes.Count(p, newline)
+	if n < maxLines {
+		return p, n
+	}
+	for n >= maxLines {
+		idx := bytes.LastIndexByte(p, '\n')
+		p = p[:idx]
+		n--
+	}
+	return p[:len(p)+1], maxLines
+}
+
+var newline = []byte("\n")
+
 func (sw *sortWriter) FinalFlush() {
 	if sw.hasErr || sw.bufFlushed {
 		return
 	}
+
 	rs := getRowsSorter()
 	rs.parseRows(sw.buf)
 	rs.sort()
-	WriteJSONRows(sw.w, rs.rows)
+
+	rows := rs.rows
+	if sw.maxLines > 0 && len(rows) > sw.maxLines {
+		rows = rows[:sw.maxLines]
+	}
+	WriteJSONRows(sw.w, rows)
+
 	putRowsSorter(rs)
 }
 

app/vlselect/logsql/sort_writer_test.go

@@ -7,15 +7,16 @@ import (
 )
 
 func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen int, data string, expectedResult string) {
+	f := func(maxBufLen, maxLines int, data string, expectedResult string) {
 		t.Helper()
 
 		var bb bytes.Buffer
 		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen)
-
+		sw.Init(&bb, maxBufLen, maxLines)
 		for _, s := range strings.Split(data, "\n") {
-			sw.MustWrite([]byte(s + "\n"))
+			if !sw.TryWrite([]byte(s + "\n")) {
+				break
+			}
 		}
 		sw.FinalFlush()
 		putSortWriter(sw)
@@ -26,14 +27,20 @@ func TestSortWriter(t *testing.T) {
 		}
 	}
 
-	f(100, "", "")
-	f(100, "{}", "{}\n")
+	f(100, 0, "", "")
+	f(100, 0, "{}", "{}\n")
 
 	data := `{"_time":"def","_msg":"xxx"}
 {"_time":"abc","_msg":"foo"}`
 	resultExpected := `{"_time":"abc","_msg":"foo"}
 {"_time":"def","_msg":"xxx"}
 `
-	f(100, data, resultExpected)
-	f(10, data, data+"\n")
+	f(100, 0, data, resultExpected)
+	f(10, 0, data, data+"\n")
+
+	// Test with the maxLines
+	f(100, 1, data, `{"_time":"abc","_msg":"foo"}`+"\n")
+	f(10, 1, data, `{"_time":"def","_msg":"xxx"}`+"\n")
+	f(10, 2, data, data+"\n")
+	f(100, 2, data, resultExpected)
 }

app/vlselect/main.go

@@ -1,6 +1,7 @@
 package vlselect
 
 import (
+	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -101,7 +102,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
-	stopCh := r.Context().Done()
+	ctx := r.Context()
+	stopCh := ctx.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
@@ -139,11 +141,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 	}
 
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+	stopCh = ctxWithCancel.Done()
+
 	switch {
 	case path == "/logsql/query":
 		logsqlQueryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh)
+		logsql.ProcessQueryRequest(w, r, stopCh, cancel)
 		return true
 	default:
 		return false

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.d1313636.css",
-    "main.js": "./static/js/main.1919fefe.js",
-    "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
+    "main.css": "./static/css/main.bc07cc78.css",
+    "main.js": "./static/js/main.53048302.js",
+    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.61a686c0661a23e4f2eb.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d1313636.css",
-    "static/js/main.1919fefe.js"
+    "static/css/main.bc07cc78.css",
+    "static/js/main.53048302.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.53048302.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.53048302.js.LICENSE.txt

@@ -4,10 +4,8 @@
 	http://jedwatson.github.io/classnames
 */
 
-/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
-
 /**
- * @remix-run/router v1.10.0
+ * @remix-run/router v1.15.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -18,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.17.0
+ * React Router DOM v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -29,7 +27,7 @@
  */
 
 /**
- * React Router v6.17.0
+ * React Router v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.61a686c0661a23e4f2eb.md

@@ -26,12 +26,18 @@ and introduction into [basic querying via MetricsQL](https://docs.victoriametric
 
 The following functionality is implemented differently in MetricsQL compared to PromQL. This improves user experience:
 
-* MetricsQL takes into account the previous point before the window in square brackets for range functions such as [rate](#rate) and [increase](#increase).
-  This allows returning the exact results users expect for `increase(metric[$__interval])` queries instead of incomplete results Prometheus returns for such queries.
-* MetricsQL doesn't extrapolate range function results. This addresses [this issue from Prometheus](https://github.com/prometheus/prometheus/issues/3746).
+* MetricsQL takes into account the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples) before the lookbehind window
+  in square brackets for [increase](#increase) and [rate](#rate) functions. This allows returning the exact results users expect for `increase(metric[$__interval])` queries
+  instead of incomplete results Prometheus returns for such queries. Prometheus misses the increase between the last sample before the lookbehind window
+  and the first sample inside the lookbehind window.
+* MetricsQL doesn't extrapolate [rate](#rate) and [increase](#increase) function results, so it always returns the expected results. For example, it returns
+  integer results from `increase()` over slow-changing integer counter. Prometheus in this case returns unexpected fractional results,
+  which may significantly differ from the expected results. This addresses [this issue from Prometheus](https://github.com/prometheus/prometheus/issues/3746).
   See technical details about VictoriaMetrics and Prometheus calculations for [rate](#rate)
   and [increase](#increase) [in this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1215#issuecomment-850305711).
-* MetricsQL returns the expected non-empty responses for [rate](#rate) with `step` values smaller than scrape interval.
+* MetricsQL returns the expected non-empty responses for [rate](#rate) function when Grafana or [vmui](https://docs.victoriametrics.com/#vmui)
+  passes `step` values smaller than the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+  to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query).
   This addresses [this issue from Grafana](https://github.com/grafana/grafana/issues/11451).
   See also [this blog post](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/).
 * MetricsQL treats `scalar` type the same as `instant vector` without labels, since subtle differences between these types usually confuse users.
@@ -61,16 +67,17 @@ The list of MetricsQL features on top of PromQL:
 
 * Graphite-compatible filters can be passed via `{__graphite__="foo.*.bar"}` syntax.
   See [these docs](https://docs.victoriametrics.com/#selecting-graphite-metrics).
-  VictoriaMetrics also can be used as Graphite datasource in Grafana.
-  See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details.
+  VictoriaMetrics can be used as Graphite datasource in Grafana. See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details.
   See also [label_graphite_group](#label_graphite_group) function, which can be used for extracting the given groups from Graphite metric name.
-* Lookbehind window in square brackets may be omitted. VictoriaMetrics automatically selects the lookbehind window
-  depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)).
+* Lookbehind window in square brackets for [rollup functions](#rollup-functions) may be omitted. VictoriaMetrics automatically selects the lookbehind window
+  depending on the `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)
+  and the real interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) (aka `scrape_interval`).
   For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
-  It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
+  It is roughly equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
+  The difference is documented in [rate() docs](#rate).
 * Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
-  selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
+  selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
 * Support for `group_left(*)` and `group_right(*)` for copying all the labels from time series on the `one` side
   of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches).
@@ -117,7 +124,8 @@ The list of MetricsQL features on top of PromQL:
   Go to [WITH templates playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/expand-with-exprs) and try it.
 * String literals may be concatenated. This is useful with `WITH` templates:
   `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`.
-* `keep_metric_names` modifier can be applied to all the [rollup functions](#rollup-functions), [transform functions](#transform-functions) and [binary operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operators).
+* `keep_metric_names` modifier can be applied to all the [rollup functions](#rollup-functions), [transform functions](#transform-functions)
+  and [binary operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operators).
   This modifier prevents from dropping metric names in function results. See [these docs](#keep_metric_names).
 
 ## keep_metric_names
@@ -155,14 +163,15 @@ Additional details:
   The interval between points is set as `step` query arg passed by Grafana to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
 * If the given [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) returns multiple time series,
   then rollups are calculated individually per each returned series.
-* If lookbehind window in square brackets is missing, then MetricsQL automatically sets the lookbehind window
-  to the interval between points on the graph (aka `step` query arg at [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query),
-  `$__interval` value from Grafana or `1i` duration in MetricsQL).
-  For example, `rate(http_requests_total)` is equivalent to `rate(http_requests_total[$__interval])` in Grafana.
-  It is also equivalent to `rate(http_requests_total[1i])`.
+* If lookbehind window in square brackets is missing, then it is automatically set to the following value:
+  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query)
+    for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
+    For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
+  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+    for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * Every [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) in MetricsQL must be wrapped into a rollup function.
   Otherwise, it is automatically wrapped into [default_rollup](#default_rollup). For example, `foo{bar="baz"}`
-  is automatically converted to `default_rollup(foo{bar="baz"}[1i])` before performing the calculations.
+  is automatically converted to `default_rollup(foo{bar="baz"})` before performing the calculations.
 * If something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) is passed to rollup function,
   then the inner arg is automatically converted to a [subquery](#subqueries).
 * All the rollup functions accept optional `keep_metric_names` modifier. If it is set, then the function keeps metric names in results.
@@ -177,7 +186,9 @@ The list of supported rollup functions:
 `absent_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns 1
 if the given lookbehind window `d` doesn't contain raw samples. Otherwise, it returns an empty result.
 
-This function is supported by PromQL. See also [present_over_time](#present_over_time).
+This function is supported by PromQL.
+
+See also [present_over_time](#present_over_time).
 
 #### aggr_over_time
 
@@ -207,7 +218,9 @@ See also [descent_over_time](#descent_over_time).
 over raw samples on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [median_over_time](#median_over_time).
+This function is supported by PromQL.
+
+See also [median_over_time](#median_over_time).
 
 #### changes
 
@@ -220,7 +233,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [changes_prometheus](#changes_prometheus).
+This function is supported by PromQL.
+
+See also [changes_prometheus](#changes_prometheus).
 
 #### changes_prometheus
 
@@ -233,7 +248,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [changes](#changes).
+This function is supported by PromQL.
+
+See also [changes](#changes).
 
 #### count_eq_over_time
 
@@ -243,7 +260,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
 
@@ -253,7 +270,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
 
@@ -263,7 +280,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
 
@@ -282,8 +299,19 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time),
-[count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time).
+This function is supported by PromQL.
+
+See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time), [count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time).
+
+#### count_values_over_time
+
+`count_values_over_time("label", series_selector[d])` is a [rollup function](#rollup-functions), which counts the number of raw samples
+with the same value over the given lookbehind window and stores the counts in a time series with an additional `label`, which contains each initial value.
+The results are calculated independently per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
 
@@ -299,6 +327,11 @@ See also [increases_over_time](#increases_over_time).
 `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last raw sample value on the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
+If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
+passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
+while `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) for the selected time series.
+This allows avoiding unexpected gaps on the graph when `step` is smaller than the `scrape_interval`.
+
 #### delta
 
 `delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the difference between
@@ -310,7 +343,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [increase](#increase) and [delta_prometheus](#delta_prometheus).
+This function is supported by PromQL.
+
+See also [increase](#increase) and [delta_prometheus](#delta_prometheus).
 
 #### delta_prometheus
 
@@ -333,7 +368,9 @@ The derivative is calculated using linear regression.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv).
+This function is supported by PromQL.
+
+See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv).
 
 #### deriv_fast
 
@@ -364,6 +401,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_values_over_time](#count_values_over_time).
+
 #### duration_over_time
 
 `duration_over_time(series_selector[d], max_interval)` is a [rollup function](#rollup-functions), which returns the duration in seconds
@@ -423,7 +462,9 @@ over the given lookbehind window `d` using the given smoothing factor `sf` and t
 Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering)
 returns time series of [gauge type](https://docs.victoriametrics.com/keyConcepts.html#gauge).
 
-This function is supported by PromQL. See also [range_linear_regression](#range_linear_regression).
+This function is supported by PromQL.
+
+See also [range_linear_regression](#range_linear_regression).
 
 #### idelta
 
@@ -432,7 +473,9 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [delta](#delta).
+This function is supported by PromQL.
+
+See also [delta](#delta).
 
 #### ideriv
 
@@ -455,7 +498,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
+This function is supported by PromQL.
+
+See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
 
 #### increase_prometheus
 
@@ -499,7 +544,9 @@ It is expected that the `series_selector` returns time series of [counter type](
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [rate](#rate) and [rollup_rate](#rollup_rate).
+This function is supported by PromQL.
+
+See also [rate](#rate) and [rollup_rate](#rollup_rate).
 
 #### lag
 
@@ -516,7 +563,9 @@ See also [lifetime](#lifetime) and [duration_over_time](#duration_over_time).
 `last_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last raw sample value on the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [first_over_time](#first_over_time) and [tlast_over_time](#tlast_over_time).
+This function is supported by PromQL.
+
+See also [first_over_time](#first_over_time) and [tlast_over_time](#tlast_over_time).
 
 #### lifetime
 
@@ -539,7 +588,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over raw samples
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [tmax_over_time](#tmax_over_time).
+This function is supported by PromQL.
+
+See also [tmax_over_time](#tmax_over_time).
 
 #### median_over_time
 
@@ -554,7 +605,9 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over raw samples
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [tmin_over_time](#tmin_over_time).
+This function is supported by PromQL.
+
+See also [tmin_over_time](#tmin_over_time).
 
 #### mode_over_time
 
@@ -580,7 +633,9 @@ See also [outliers_iqr](#outliers_iqr).
 linear interpolation over raw samples on the given lookbehind window `d`. The predicted value is calculated individually per each time series
 returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [range_linear_regression](#range_linear_regression).
+This function is supported by PromQL.
+
+See also [range_linear_regression](#range_linear_regression).
 
 #### present_over_time
 
@@ -597,7 +652,9 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 The `phi` value must be in the range `[0...1]`.
 
-This function is supported by PromQL. See also [quantiles_over_time](#quantiles_over_time).
+This function is supported by PromQL.
+
+See also [quantiles_over_time](#quantiles_over_time).
 
 #### quantiles_over_time
 
@@ -622,9 +679,16 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyConcepts.html#counter).
 
+If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
+passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
+while `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) for the selected time series.
+This allows avoiding unexpected gaps on the graph when `step` is smaller than the `scrape_interval`.
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [irate](#irate) and [rollup_rate](#rollup_rate).
+This function is supported by PromQL.
+
+See also [irate](#irate) and [rollup_rate](#rollup_rate).
 
 #### rate_over_sum
 
@@ -652,6 +716,7 @@ on the given lookbehind window `d` and returns them in time series with `rollup=
 These values are calculated individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 #### rollup_candlestick
 
@@ -660,7 +725,8 @@ over raw samples on the given lookbehind window `d` and returns them in time ser
 The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). This function is useful for financial applications.
 
-Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 #### rollup_delta
 
@@ -670,6 +736,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -683,6 +750,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -694,6 +762,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
@@ -707,10 +776,10 @@ See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-c
 when to use `rollup_rate()`.
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### rollup_scrape_interval
@@ -721,6 +790,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [scrape_interval](#scrape_interval).
 
@@ -743,7 +813,7 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_le_over_time](#share_le_over_time).
+See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
 
@@ -756,7 +826,7 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_gt_over_time](#share_gt_over_time).
+See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
 
@@ -766,6 +836,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_eq_over_time](#count_eq_over_time).
+
 #### stale_samples_over_time
 
 `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
@@ -781,7 +853,9 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [stdvar_over_time](#stdvar_over_time).
+This function is supported by PromQL.
+
+See also [stdvar_over_time](#stdvar_over_time).
 
 #### stdvar_over_time
 
@@ -790,7 +864,36 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
+This function is supported by PromQL.
+
+See also [stddev_over_time](#stddev_over_time).
+
+#### sum_eq_over_time
+
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values equal to `eq`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
+
+#### sum_gt_over_time
+
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values bigger than `gt`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
+
+#### sum_le_over_time
+
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values smaller or equal to `le`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
 
@@ -810,25 +913,27 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [timestamp_with_name](#timestamp_with_name).
+This function is supported by PromQL.
+
+See also [time](#time) and [now](#now).
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
 
-See also [timestamp](#timestamp).
+See also [timestamp](#timestamp) and [keep_metric_names](#keep_metric_names) modifier.
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +942,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +957,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +967,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -891,7 +996,7 @@ Additional details:
 
 * If transform function is applied directly to a [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before calculating the transformations.
-  For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature[1i]))`.
+  For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature))`.
 * All the transform functions accept optional `keep_metric_names` modifier. If it is set,
   then the function doesn't drop metric names from the resulting time series. See [these docs](#keep_metric_names).
 
@@ -909,7 +1014,9 @@ This function is supported by PromQL.
 
 `absent(q)` is a [transform function](#transform-functions), which returns 1 if `q` has no points. Otherwise, returns an empty result.
 
-This function is supported by PromQL. See also [absent_over_time](#absent_over_time).
+This function is supported by PromQL.
+
+See also [absent_over_time](#absent_over_time).
 
 #### acos
 
@@ -918,7 +1025,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [asin](#asin) and [cos](#cos).
+This function is supported by PromQL.
+
+See also [asin](#asin) and [cos](#cos).
 
 #### acosh
 
@@ -927,7 +1036,9 @@ This function is supported by PromQL. See also [asin](#asin) and [cos](#cos).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sinh](#cosh).
+This function is supported by PromQL.
+
+See also [sinh](#cosh).
 
 #### asin
 
@@ -936,7 +1047,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [acos](#acos) and [sin](#sin).
+This function is supported by PromQL.
+
+See also [acos](#acos) and [sin](#sin).
 
 #### asinh
 
@@ -945,7 +1058,9 @@ This function is supported by PromQL. See also [acos](#acos) and [sin](#sin).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sinh](#sinh).
+This function is supported by PromQL.
+
+See also [sinh](#sinh).
 
 #### atan
 
@@ -954,7 +1069,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [tan](#tan).
+This function is supported by PromQL.
+
+See also [tan](#tan).
 
 #### atanh
 
@@ -963,7 +1080,9 @@ This function is supported by PromQL. See also [tan](#tan).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [tanh](#tanh).
+This function is supported by PromQL.
+
+See also [tanh](#tanh).
 
 #### bitmap_and
 
@@ -994,25 +1113,33 @@ See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#his
 
 `ceil(q)` is a [transform function](#transform-functions), which rounds every point for every time series returned by `q` to the upper nearest integer.
 
-This function is supported by PromQL. See also [floor](#floor) and [round](#round).
+This function is supported by PromQL.
+
+See also [floor](#floor) and [round](#round).
 
 #### clamp
 
 `clamp(q, min, max)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `min` and `max` values.
 
-This function is supported by PromQL. See also [clamp_min](#clamp_min) and [clamp_max](#clamp_max).
+This function is supported by PromQL.
+
+See also [clamp_min](#clamp_min) and [clamp_max](#clamp_max).
 
 #### clamp_max
 
 `clamp_max(q, max)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `max` value.
 
-This function is supported by PromQL. See also [clamp](#clamp) and [clamp_min](#clamp_min).
+This function is supported by PromQL.
+
+See also [clamp](#clamp) and [clamp_min](#clamp_min).
 
 #### clamp_min
 
 `clamp_min(q, min)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `min` value.
 
-This function is supported by PromQL. See also [clamp](#clamp) and [clamp_max](#clamp_max).
+This function is supported by PromQL.
+
+See also [clamp](#clamp) and [clamp_max](#clamp_max).
 
 #### cos
 
@@ -1020,7 +1147,9 @@ This function is supported by PromQL. See also [clamp](#clamp) and [clamp_max](#
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sin](#sin).
+This function is supported by PromQL.
+
+See also [sin](#sin).
 
 #### cosh
 
@@ -1029,7 +1158,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [acosh](#acosh).
+This function is supported by PromQL.
+
+See also [acosh](#acosh).
 
 #### day_of_month
 
@@ -1040,6 +1171,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_week](#day_of_week) and [day_of_year](#day_of_year).
+
 #### day_of_week
 
 `day_of_week(q)` is a [transform function](#transform-functions), which returns the day of week for every point of every time series returned by `q`.
@@ -1049,6 +1182,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_month](#day_of_month) and [day_of_year](#day_of_year).
+
 #### day_of_year
 
 `day_of_year(q)` is a [transform function](#transform-functions), which returns the day of year for every point of every time series returned by `q`.
@@ -1058,6 +1193,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_week](#day_of_week) and [day_of_month](#day_of_month).
+
 #### days_in_month
 
 `days_in_month(q)` is a [transform function](#transform-functions), which returns the number of days in the month identified
@@ -1075,7 +1212,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [rad](#rad).
+This function is supported by PromQL.
+
+See also [rad](#rad).
 
 #### drop_empty_series
 
@@ -1101,13 +1240,17 @@ See also [start](#start), [time](#time) and [now](#now).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [ln](#ln).
+This function is supported by PromQL.
+
+See also [ln](#ln).
 
 #### floor
 
 `floor(q)` is a [transform function](#transform-functions), which rounds every point for every time series returned by `q` to the lower nearest integer.
 
-This function is supported by PromQL. See also [ceil](#ceil) and [round](#round).
+This function is supported by PromQL.
+
+See also [ceil](#ceil) and [round](#round).
 
 #### histogram_avg
 
@@ -1130,8 +1273,9 @@ When the [percentile](https://en.wikipedia.org/wiki/Percentile) is calculated ov
 then all the input histograms **must** have buckets with identical boundaries, e.g. they must have the same set of `le` or `vmrange` labels.
 Otherwise, the returned result may be invalid. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3231) for details.
 
-This function is supported by PromQL (except of the `boundLabel` arg). See also [histogram_quantiles](#histogram_quantiles), [histogram_share](#histogram_share)
-and [quantile](#quantile).
+This function is supported by PromQL (except of the `boundLabel` arg).
+
+See also [histogram_quantiles](#histogram_quantiles), [histogram_share](#histogram_share) and [quantile](#quantile).
 
 #### histogram_quantiles
 
@@ -1203,7 +1347,9 @@ This allows implementing simple paging for `q` time series. See also [limitk](#l
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [exp](#exp) and [log2](#log2).
+This function is supported by PromQL.
+
+See also [exp](#exp) and [log2](#log2).
 
 #### log2
 
@@ -1211,7 +1357,9 @@ This function is supported by PromQL. See also [exp](#exp) and [log2](#log2).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [log10](#log10) and [ln](#ln).
+This function is supported by PromQL.
+
+See also [log10](#log10) and [ln](#ln).
 
 #### log10
 
@@ -1219,7 +1367,9 @@ This function is supported by PromQL. See also [log10](#log10) and [ln](#ln).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [log2](#log2) and [ln](#ln).
+This function is supported by PromQL.
+
+See also [log2](#log2) and [ln](#ln).
 
 #### minute
 
@@ -1258,7 +1408,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [deg](#deg).
+This function is supported by PromQL.
+
+See also [deg](#deg).
 
 #### prometheus_buckets
 
@@ -1386,7 +1538,9 @@ for points returned by `q`, e.g. it is equivalent to the following query: `(q -
 `round(q, nearest)` is a [transform function](#transform-functions), which rounds every point of every time series returned by `q` to the `nearest` multiple.
 If `nearest` is missing then the rounding is performed to the nearest integer.
 
-This function is supported by PromQL. See also [floor](#floor) and [ceil](#ceil).
+This function is supported by PromQL.
+
+See also [floor](#floor) and [ceil](#ceil).
 
 #### ru
 
@@ -1430,7 +1584,9 @@ This function is supported by PromQL.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [cos](#cos).
+This function is supported by MetricsQL.
+
+See also [cos](#cos).
 
 #### sinh
 
@@ -1439,7 +1595,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [cosh](#cosh).
+This function is supported by MetricsQL.
+
+See also [cosh](#cosh).
 
 #### tan
 
@@ -1447,7 +1605,9 @@ This function is supported by MetricsQL. See also [cosh](#cosh).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [atan](#atan).
+This function is supported by MetricsQL.
+
+See also [atan](#atan).
 
 #### tanh
 
@@ -1456,7 +1616,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [atanh](#atanh).
+This function is supported by MetricsQL.
+
+See also [atanh](#atanh).
 
 #### smooth_exponential
 
@@ -1467,13 +1629,17 @@ by `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_a
 
 `sort(q)` is a [transform function](#transform-functions), which sorts series in ascending order by the last point in every time series returned by `q`.
 
-This function is supported by PromQL. See also [sort_desc](#sort_desc) and [sort_by_label](#sort_by_label).
+This function is supported by PromQL.
+
+See also [sort_desc](#sort_desc) and [sort_by_label](#sort_by_label).
 
 #### sort_desc
 
 `sort_desc(q)` is a [transform function](#transform-functions), which sorts series in descending order by the last point in every time series returned by `q`.
 
-This function is supported by PromQL. See also [sort](#sort) and [sort_by_label](#sort_by_label_desc).
+This function is supported by PromQL.
+
+See also [sort](#sort) and [sort_by_label](#sort_by_label_desc).
 
 #### sqrt
 
@@ -1502,7 +1668,9 @@ See also [start](#start) and [end](#end).
 
 `time()` is a [transform function](#transform-functions), which returns unix timestamp for every returned point.
 
-This function is supported by PromQL. See also [now](#now), [start](#start) and [end](#end).
+This function is supported by PromQL.
+
+See also [timestamp](#timestamp), [now](#now), [start](#start) and [end](#end).
 
 #### timezone_offset
 
@@ -1551,7 +1719,7 @@ Additional details:
 
 * If label manipulation function is applied directly to a [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before performing the label transformation.
-  For example, `alias(temperature, "foo")` is implicitly transformed to `alias(default_rollup(temperature[1i]), "foo")`.
+  For example, `alias(temperature, "foo")` is implicitly transformed to `alias(default_rollup(temperature), "foo")`.
 
 See also [implicit query conversions](#implicit-query-conversions).
 
@@ -1728,7 +1896,7 @@ Additional details:
   Multiple labels can be put in `by` and `without` modifiers.
 * If the aggregate function is applied directly to a [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before calculating the aggregate.
-  For example, `count(up)` is implicitly transformed to `count(default_rollup(up[1i]))`.
+  For example, `count(up)` is implicitly transformed to `count(default_rollup(up))`.
 * Aggregate functions accept arbitrary number of args. For example, `avg(q1, q2, q3)` would return the average values for every point
   across time series returned by `q1`, `q2` and `q3`.
 * Aggregate functions support optional `limit N` suffix, which can be used for limiting the number of output groups.
@@ -1756,7 +1924,9 @@ This function is supported by PromQL.
 `bottomk(k, q)` is [aggregate function](#aggregate-functions), which returns up to `k` points with the smallest values across all the time series returned by `q`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [topk](#topk).
+This function is supported by PromQL.
+
+See also [topk](#topk), [bottomk_min](#bottomk_min) and [#bottomk_last](#bottomk_last).
 
 #### bottomk_avg
 
@@ -1818,10 +1988,14 @@ The aggregate is calculated individually per each group of points with the same
 
 This function is supported by PromQL.
 
+See also [count_values_over_time](#count_values_over_time) and [label_match](#label_match).
+
 #### distinct
 
 `distinct(q)` is [aggregate function](#aggregate-functions), which calculates the number of unique values per each group of points with the same timestamp.
 
+See also [distinct_over_time](#distinct_over_time).
+
 #### geomean
 
 `geomean(q)` is [aggregate function](#aggregate-functions), which calculates geometric mean per each group of points with the same timestamp.
@@ -1913,7 +2087,9 @@ See also [outliers_iqr](#outliers_iqr) and [outliers_mad](#outliers_mad).
 for all the time series returned by `q`. `phi` must be in the range `[0...1]`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [quantiles](#quantiles) and [histogram_quantile](#histogram_quantile).
+This function is supported by PromQL.
+
+See also [quantiles](#quantiles) and [histogram_quantile](#histogram_quantile).
 
 #### quantiles
 
@@ -1972,7 +2148,9 @@ for all the time series returned by `q`. The aggregate is calculated individuall
 `topk(k, q)` is [aggregate function](#aggregate-functions), which returns up to `k` points with the biggest values across all the time series returned by `q`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [bottomk](#bottomk).
+This function is supported by PromQL.
+
+See also [bottomk](#bottomk), [topk_max](#topk_max) and [topk_last](#topk_last).
 
 #### topk_avg
 
@@ -2032,7 +2210,7 @@ See also [zscore_over_time](#zscore_over_time), [range_trim_zscore](#range_trim_
 MetricsQL supports and extends PromQL subqueries. See [this article](https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3) for details.
 Any [rollup function](#rollup-functions) for something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) form a subquery.
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
-For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m[1i]))[1i:1i])`, so it becomes a subquery,
+For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
 
 VictoriaMetrics performs subqueries in the following way:
@@ -2047,21 +2225,23 @@ VictoriaMetrics performs subqueries in the following way:
 
 VictoriaMetrics performs the following implicit conversions for incoming queries before starting the calculations:
 
-* If lookbehind window in square brackets is missing inside [rollup function](#rollup-functions),
-  then `[1i]` is automatically added there. The `[1i]` means one `step` value, which is passed
-  to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
-  It is also known as `$__interval` in Grafana. For example, `rate(http_requests_count)` is automatically transformed to `rate(http_requests_count[1i])`.
+* If lookbehind window in square brackets is missing inside [rollup function](#rollup-functions), then it is automatically set to the following value:
+  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query)
+    for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
+    For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
+  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+    for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * All the [series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   which aren't wrapped into [rollup functions](#rollup-functions), are automatically wrapped into [default_rollup](#default_rollup) function.
   Examples:
-  * `foo` is transformed to `default_rollup(foo[1i])`
-  * `foo + bar` is transformed to `default_rollup(foo[1i]) + default_rollup(bar[1i])`
-  * `count(up)` is transformed to `count(default_rollup(up[1i]))`, because [count](#count) isn't a [rollup function](#rollup-functions) -
+  * `foo` is transformed to `default_rollup(foo)`
+  * `foo + bar` is transformed to `default_rollup(foo) + default_rollup(bar)`
+  * `count(up)` is transformed to `count(default_rollup(up))`, because [count](#count) isn't a [rollup function](#rollup-functions) -
     it is [aggregate function](#aggregate-functions)
-  * `abs(temperature)` is transformed to `abs(default_rollup(temperature[1i]))`, because [abs](#abs) isn't a [rollup function](#rollup-functions) -
+  * `abs(temperature)` is transformed to `abs(default_rollup(temperature))`, because [abs](#abs) isn't a [rollup function](#rollup-functions) -
     it is [transform function](#transform-functions)
 * If `step` in square brackets is missing inside [subquery](#subqueries), then `1i` step is automatically added there.
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up[1i])))[1i:1i])`.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.

app/vmagent/datadogsketches/request_handler.go

@@ -0,0 +1,95 @@
+package datadogsketches
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
+		return insertRows(at, sketches, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for _, sketch := range sketches {
+		ms := sketch.ToSummary()
+		for _, m := range ms {
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: m.Name,
+			})
+			for _, label := range m.Labels {
+				labels = append(labels, prompbmarshal.Label{
+					Name:  label.Name,
+					Value: label.Value,
+				})
+			}
+			for _, tag := range sketch.Tags {
+				name, value := datadogutils.SplitTag(tag)
+				if name == "host" {
+					name = "exported_host"
+				}
+				labels = append(labels, prompbmarshal.Label{
+					Name:  name,
+					Value: value,
+				})
+			}
+			labels = append(labels, extraLabels...)
+			samplesLen := len(samples)
+			for _, p := range m.Points {
+				samples = append(samples, prompbmarshal.Sample{
+					Timestamp: p.Timestamp,
+					Value:     p.Value,
+				})
+			}
+			rowsTotal += len(m.Points)
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[samplesLen:],
+			})
+		}
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/main.go

@@ -8,10 +8,10 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
@@ -45,10 +45,10 @@ import (
 )
 
 var (
-	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -69,7 +69,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -118,7 +119,11 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8429"}
+	}
+	logger.Infof("starting vmagent at %q...", listenAddrs)
 	startTime := time.Now()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
@@ -141,9 +146,7 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	if len(*httpListenAddr) > 0 {
-		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
-	}
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -152,13 +155,11 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	if len(*httpListenAddr) > 0 {
-		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-		if err := httpserver.Stop(*httpListenAddr); err != nil {
-			logger.Fatalf("cannot stop the webservice: %s", err)
-		}
-		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -312,7 +313,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push":
+	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
 			opentelemetryPushErrors.Inc()
@@ -367,6 +368,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "/datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -421,7 +431,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -430,7 +440,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -440,12 +450,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -546,7 +559,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push":
+	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
 			opentelemetryPushErrors.Inc()
@@ -599,6 +612,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -655,13 +677,16 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
+	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)

app/vmagent/opentelemetry/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -27,10 +28,15 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
-		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		if req.Header.Get("X-Amz-Firehouse-Protocol-Version") != "" {
+			processBody = firehose.ProcessRequestBody
+		} else {
+			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		}
 	}
-	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/remotewrite/client.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -34,6 +35,7 @@ var (
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
+	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -121,7 +123,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	tr := &http.Transport{
 		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 10 * time.Second,
+		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -395,7 +397,8 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
-	retryDuration := time.Second
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:
@@ -405,8 +408,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > time.Minute {
-			retryDuration = time.Minute
+		if retryDuration > maxRetryDuration {
+			retryDuration = maxRetryDuration
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -452,8 +455,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > time.Minute {
-		retryDuration = time.Minute
+	if retryDuration > maxRetryDuration {
+		retryDuration = maxRetryDuration
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()

app/vmagent/remotewrite/pendingseries.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -15,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -69,7 +71,8 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	ticker := time.NewTicker(*flushInterval)
+	d := timeutil.AddJitterToDuration(*flushInterval)
+	ticker := time.NewTicker(d)
 	defer ticker.Stop()
 	for {
 		select {
@@ -79,7 +82,7 @@ func (ps *pendingSeries) periodicFlusher() {
 			ps.mu.Unlock()
 			return
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
 				continue
 			}
 		}
@@ -90,8 +93,7 @@ func (ps *pendingSeries) periodicFlusher() {
 }
 
 type writeRequest struct {
-	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
-	lastFlushTime uint64
+	lastFlushTime atomic.Uint64
 
 	// The queue to send blocks to.
 	fq *persistentqueue.FastQueue
@@ -107,11 +109,12 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss []prompbmarshal.TimeSeries
-
+	tss     []prompbmarshal.TimeSeries
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-	buf     []byte
+
+	// buf holds labels data
+	buf []byte
 }
 
 func (wr *writeRequest) reset() {
@@ -151,7 +154,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
-	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
 	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
 		return false
 	}
@@ -222,33 +225,45 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
+// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
+var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
+
 func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
+
+	marshalConcurrencyCh <- struct{}{}
+
 	bb := writeRequestBufPool.Get()
 	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := snappyBufPool.Get()
+		zb := compressBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
+
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			if !tryPushBlock(zb.B) {
-				return false
+			zbLen := len(zb.B)
+			ok := tryPushBlock(zb.B)
+			compressBufPool.Put(zb)
+			if ok {
+				blockSizeRows.Update(float64(len(wr.Timeseries)))
+				blockSizeBytes.Update(float64(zbLen))
 			}
-			blockSizeRows.Update(float64(len(wr.Timeseries)))
-			blockSizeBytes.Update(float64(len(zb.B)))
-			snappyBufPool.Put(zb)
-			return true
+			return ok
 		}
-		snappyBufPool.Put(zb)
+		compressBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -294,5 +309,7 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var writeRequestBufPool bytesutil.ByteBufferPool
-var snappyBufPool bytesutil.ByteBufferPool
+var (
+	writeRequestBufPool bytesutil.ByteBufferPool
+	compressBufPool     bytesutil.ByteBufferPool
+)

app/vmagent/remotewrite/remotewrite.go

@@ -536,22 +536,22 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 		// Push sharded data to remote storages in parallel in order to reduce
 		// the time needed for sending the data to multiple remote storage systems.
 		var wg sync.WaitGroup
-		wg.Add(len(rwctxs))
-		var anyPushFailed uint64
+		var anyPushFailed atomic.Bool
 		for i, rwctx := range rwctxs {
 			tssShard := tssByURL[i]
 			if len(tssShard) == 0 {
 				continue
 			}
+			wg.Add(1)
 			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
 				defer wg.Done()
 				if !rwctx.TryPush(tss) {
-					atomic.StoreUint64(&anyPushFailed, 1)
+					anyPushFailed.Store(true)
 				}
 			}(rwctx, tssShard)
 		}
 		wg.Wait()
-		return atomic.LoadUint64(&anyPushFailed) == 0
+		return !anyPushFailed.Load()
 	}
 
 	// Replicate data among rwctxs.
@@ -559,17 +559,17 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
-	var anyPushFailed uint64
+	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
 			if !rwctx.TryPush(tssBlock) {
-				atomic.StoreUint64(&anyPushFailed, 1)
+				anyPushFailed.Store(true)
 			}
 		}(rwctx)
 	}
 	wg.Wait()
-	return atomic.LoadUint64(&anyPushFailed) == 0
+	return !anyPushFailed.Load()
 }
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
@@ -670,7 +670,7 @@ type remoteWriteCtx struct {
 	streamAggrDropInput bool
 
 	pss        []*pendingSeries
-	pssNextIdx uint64
+	pssNextIdx atomic.Uint64
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
@@ -872,7 +872,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	}
 
 	pss := rwctx.pss
-	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
 	ok := pss[idx].TryPush(tss)
 

app/vmagent/remotewrite/statconn.go

@@ -50,7 +50,7 @@ var (
 )
 
 type statConn struct {
-	closed uint64
+	closed atomic.Int32
 	net.Conn
 }
 
@@ -76,7 +76,7 @@ func (sc *statConn) Write(p []byte) (int, error) {
 
 func (sc *statConn) Close() error {
 	err := sc.Conn.Close()
-	if atomic.AddUint64(&sc.closed, 1) == 1 {
+	if sc.closed.Add(1) == 1 {
 		conns.Dec()
 	}
 	return err

app/vmalert-tool/unittest/unittest.go

@@ -25,6 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -184,7 +185,8 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
+	var ab flagutil.ArrayBool
+	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
 		switch r.URL.Path {
 		case "/prometheus/api/v1/query":
 			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
@@ -225,7 +227,7 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop(httpListenAddr); err != nil {
+	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
 		logger.Errorf("cannot stop the webservice: %s", err)
 	}
 	vmstorage.Stop()

app/vmalert/Makefile

@@ -68,6 +68,7 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -22,6 +22,7 @@ groups:
               {{ . | first | value }}
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
+          link: http://localhost:3000/d/wNf0q_kZk?viewPanel=51&from={{($activeAt.Add (parseDurationTime "1h")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime "-1h")).UnixMilli}}
       - alert: ExampleAlertAlwaysFiring
         update_entries_limit: -1
         expr: sum by(job)

app/vmalert/datasource/init.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -93,7 +94,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
 	}
 
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/datasource/vm.go

@@ -137,11 +137,11 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := s.newQueryRequest(query, ts)
+	req, err := s.newQueryRequest(ctx, query, ts)
 	if err != nil {
 		return Result{}, nil, err
 	}
-	resp, err := s.do(ctx, req)
+	resp, err := s.do(req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -149,11 +149,11 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRequest(query, ts)
+		req, err = s.newQueryRequest(ctx, query, ts)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(ctx, req)
+		resp, err = s.do(req)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
@@ -182,11 +182,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	req, err := s.newQueryRangeRequest(query, start, end)
+	req, err := s.newQueryRangeRequest(ctx, query, start, end)
 	if err != nil {
 		return res, err
 	}
-	resp, err := s.do(ctx, req)
+	resp, err := s.do(req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -194,11 +194,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRangeRequest(query, start, end)
+		req, err = s.newQueryRangeRequest(ctx, query, start, end)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(ctx, req)
+		resp, err = s.do(req)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
@@ -210,7 +210,7 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	return res, err
 }
 
-func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
 	ru := req.URL.Redacted()
 	if *showDatasourceURL {
 		ru = req.URL.String()
@@ -218,7 +218,7 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	if s.debug {
 		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru)
 	}
-	resp, err := s.c.Do(req.WithContext(ctx))
+	resp, err := s.c.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", ru, err)
 	}
@@ -230,8 +230,8 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	return resp, nil
 }
 
-func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*http.Request, error) {
-	req, err := s.newRequest()
+func (s *VMStorage) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) {
+	req, err := s.newRequest(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -239,8 +239,8 @@ func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*h
 	return req, nil
 }
 
-func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request, error) {
-	req, err := s.newRequest()
+func (s *VMStorage) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) {
+	req, err := s.newRequest(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -255,8 +255,8 @@ func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request,
 	return req, nil
 }
 
-func (s *VMStorage) newRequest() (*http.Request, error) {
-	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
+func (s *VMStorage) newRequest(ctx context.Context) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}

app/vmalert/datasource/vm_test.go

@@ -637,7 +637,7 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequest()
+			req, err := tc.vm.newRequest(ctx)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -735,7 +735,7 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req, err := vm.newQueryRequest("foo", time.Now())
+			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
 			if err != nil {
 				t.Fatal(err)
 			}

app/vmalert/main.go

@@ -59,8 +59,8 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -178,15 +178,19 @@ func main() {
 
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8880"}
+	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
 	pushmetrics.Stop()
 
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
@@ -248,7 +252,13 @@ func newManager(ctx context.Context) (*manager, error) {
 func getExternalURL(customURL string) (*url.URL, error) {
 	if customURL == "" {
 		// use local hostname as external URL
-		return getHostnameAsExternalURL(*httpListenAddr, httpserver.IsTLS())
+		listenAddr := ":8880"
+		if len(*httpListenAddrs) > 0 {
+			listenAddr = (*httpListenAddrs)[0]
+		}
+		isTLS := httpserver.IsTLS(0)
+
+		return getHostnameAsExternalURL(listenAddr, isTLS)
 	}
 	u, err := url.Parse(customURL)
 	if err != nil {
@@ -260,13 +270,13 @@ func getExternalURL(customURL string) (*url.URL, error) {
 	return u, nil
 }
 
-func getHostnameAsExternalURL(httpListenAddr string, isSecure bool) (*url.URL, error) {
+func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
 	hname, err := os.Hostname()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get hostname: %w", err)
 	}
 	port := ""
-	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+	if ipport := strings.Split(addr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]
 	}
 	schema := "http://"

app/vmalert/manager.go

@@ -156,11 +156,14 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
+			// cancel evaluation so the Update will be applied as fast as possible.
+			// it is important to call InterruptEval before the update, because cancel fn
+			// can be re-assigned during the update.
+			item.old.InterruptEval()
 			go func(old *rule.Group, new *rule.Group) {
 				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
-			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alertmanager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -127,7 +128,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remoteread/init.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -60,7 +61,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/client.go

@@ -279,7 +279,7 @@ L:
 
 func (c *Client) send(ctx context.Context, data []byte) error {
 	r := bytes.NewReader(data)
-	req, err := http.NewRequest(http.MethodPost, c.addr, r)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
@@ -302,7 +302,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
-	resp, err := c.c.Do(req.WithContext(ctx))
+	resp, err := c.c.Do(req)
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
 			req.URL.Redacted(), err, len(data), r.Size())

app/vmalert/remotewrite/client_test.go

@@ -91,14 +91,12 @@ func newRWServer() *rwServer {
 }
 
 type rwServer struct {
-	// WARN: ordering of fields is important for alignment!
-	// see https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	acceptedRows uint64
+	acceptedRows atomic.Uint64
 	*httptest.Server
 }
 
 func (rw *rwServer) accepted() int {
-	return int(atomic.LoadUint64(&rw.acceptedRows))
+	return int(rw.acceptedRows.Load())
 }
 
 func (rw *rwServer) err(w http.ResponseWriter, err error) {
@@ -144,7 +142,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))
 		return
 	}
-	atomic.AddUint64(&rw.acceptedRows, uint64(len(wr.Timeseries)))
+	rw.acceptedRows.Add(uint64(len(wr.Timeseries)))
 	w.WriteHeader(http.StatusNoContent)
 }
 

app/vmalert/remotewrite/debug_client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -30,7 +30,7 @@ func NewDebugClient() (*DebugClient, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/init.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -64,7 +65,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/rule/alerting.go

@@ -324,16 +324,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 
-		// if alert is instant, For: 0
-		if ar.For == 0 {
-			a.State = notifier.StateFiring
-			for i := range s.Values {
-				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
-			}
-			continue
-		}
-
-		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
@@ -354,6 +344,10 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.Start = at
 			}
 			prevT = at
+			if ar.For == 0 {
+				// rules with `for: 0` are always firing when they have Value
+				a.State = notifier.StateFiring
+			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
 			// save alert's state on last iteration, so it can be used on the next execRange call
@@ -446,14 +440,13 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, start, qFn)
+		a, err := ar.newAlert(m, ls, ts, qFn)
 		if err != nil {
 			curState.Err = fmt.Errorf("failed to create alert: %w", err)
 			return nil, curState.Err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
-		a.ActiveAt = ts
 		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
@@ -479,7 +472,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -559,9 +552,9 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 }
 
 const (
-	// alertMetricName is the metric name for synthetic alert timeseries.
+	// alertMetricName is the metric name for time series reflecting the alert state.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for 'for' state of alert.
+	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -576,12 +569,10 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(a, timestamp))
-	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(a, timestamp))
+	return []prompbmarshal.TimeSeries{
+		alertToTimeSeries(a, timestamp),
+		alertForToTimeSeries(a, timestamp),
 	}
-	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {

app/vmalert/rule/alerting_test.go

@@ -28,20 +28,26 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				"job":      "foo",
-				"instance": "bar",
-			}},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					"job":      "foo",
+					"instance": "bar",
+				}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -49,19 +55,33 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+						"job":      "foo",
+						"instance": "bar",
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				alertStateLabel: "foo",
-				"__name__":      "bar",
-			}},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					alertStateLabel: "foo",
+					"__name__":      "bar",
+				}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__":      alertForStateMetricName,
+						alertStateLabel: "foo",
+					}),
 			},
 		},
 		{
@@ -308,14 +328,17 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &datasource.FakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
+			ts := time.Now()
 			for i, step := range tc.steps {
 				fq.Reset()
 				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
+				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-				// artificial delay between applying steps
-				time.Sleep(defaultStep)
+
+				// shift the execution timestamp before the next iteration
+				ts = ts.Add(defaultStep)
+
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -367,7 +390,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 			nil,
 		},
@@ -378,8 +401,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels: map[string]string{"name": "foo"},
-					State:  notifier.StateFiring,
+					Labels:   map[string]string{"name": "foo"},
+					State:    notifier.StateFiring,
+					ActiveAt: time.Unix(1, 0),
 				},
 			},
 			nil,
@@ -390,9 +414,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
 			},
 			nil,
 		},
@@ -460,6 +484,20 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				For:         time.Second,
 			}},
 		},
+		{
+			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+			},
+			nil,
+		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
 			[]datasource.Metric{
@@ -534,21 +572,25 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					}},
 				//
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					}},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					}},
 			},
 			nil,
 		},
@@ -1095,6 +1137,12 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 	return &rule
 }
 
+func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
+	rule := newTestAlertingRule(name, waitFor)
+	rule.EvalInterval = evalInterval
+	return rule
+}
+
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor

app/vmalert/static/js/custom.js

@@ -1,5 +1,12 @@
 function expandAll() {
-    $('.collapse').addClass('show');
+    $('.group-heading').each(function () {
+        let style = $(this).attr("style")
+        // display only elements that are currently visible
+        if (style === "display: none;") {
+            return
+        }
+        $(this).next().addClass('show')
+    });
 }
 
 function collapseAll() {
@@ -15,6 +22,100 @@ function toggleByID(id) {
     }
 }
 
+function debounce(func, delay) {
+    let timer;
+    return function (...args) {
+        clearTimeout(timer);
+        timer = setTimeout(() => {
+            func.apply(this, args);
+        }, delay);
+    };
+}
+
+$('#search').on("keyup", debounce(search, 500));
+
+// search shows or hides groups&rules that satisfy the search phrase.
+// case-insensitive, respects GET param `search`.
+function search() {
+    $(".rule").show();
+
+    let groupHeader = $(".group-heading")
+    let searchPhrase = $("#search").val().toLowerCase()
+    if (searchPhrase.length === 0) {
+        groupHeader.show()
+        setParamURL('search', '')
+        return
+    }
+
+    $(".rule-table").removeClass('show');
+    groupHeader.hide()
+
+    searchPhrase = searchPhrase.toLowerCase()
+    filterRuleByName(searchPhrase);
+    filterRuleByLabels(searchPhrase);
+    filterGroupsByName(searchPhrase);
+
+    setParamURL('search', searchPhrase)
+}
+
+function setParamURL(key, value) {
+    let url = new URL(location.href)
+    url.searchParams.set(key, value);
+    window.history.replaceState(null, null, `?${url.searchParams.toString()}`);
+}
+
+function getParamURL(key) {
+    let url = new URL(location.href)
+    return url.searchParams.get(key)
+}
+
+function filterGroupsByName(searchPhrase) {
+    $(".group-heading").each(function () {
+        const groupName = $(this).attr('data-group-name').toLowerCase();
+        const hasValue = groupName.indexOf(searchPhrase) >= 0
+
+        if (!hasValue) {
+            return
+        }
+
+        const target = $(this).attr("data-bs-target");
+        $(`div[id="${target}"] .rule`).show();
+        $(this).show();
+    });
+}
+
+function filterRuleByName(searchPhrase) {
+    $(".rule").each(function () {
+        const ruleName = $(this).attr("data-rule-name").toLowerCase();
+        const hasValue = ruleName.indexOf(searchPhrase) >= 0
+        if (!hasValue) {
+            $(this).hide();
+            return
+        }
+
+        const target = $(this).attr('data-bs-target')
+        $(`#rules-${target}`).addClass('show');
+        $(`div[data-bs-target='rules-${target}']`).show();
+        $(this).show();
+    });
+}
+
+function filterRuleByLabels(searchPhrase) {
+    $(".rule").each(function () {
+        const matches = $(".label", this).filter(function () {
+            const label = $(this).text().toLowerCase();
+            return label.indexOf(searchPhrase) >= 0;
+        }).length;
+
+        if (matches > 0) {
+            const target = $(this).attr('data-bs-target')
+            $(`#rules-${target}`).addClass('show');
+            $(`div[data-bs-target='rules-${target}']`).show();
+            $(this).show();
+        }
+    });
+}
+
 $(document).ready(function () {
     $(".group-heading a").click(function (e) {
         e.stopPropagation(); // prevent collapse logic on link click
@@ -32,6 +133,13 @@ $(document).ready(function () {
         });
     });
 
+    // update search element with value from URL, if any
+    let searchPhrase = getParamURL('search')
+    $("#search").val(searchPhrase)
+
+    // apply filtering by search phrase
+    search()
+
     let hash = window.location.hash.substr(1);
     toggleByID(hash);
 });

app/vmalert/web.go

@@ -12,11 +12,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+
 var (
 	apiLinks = [][2]string{
 		// api links are relative since they can be used by external clients,
@@ -84,7 +88,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -95,12 +102,20 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		data, err := rh.listGroups()
+		var data []byte
+		var err error
+
+		rf := extractRulesFilter(r)
+		data, err = rh.listGroups(rf)
+
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -108,6 +123,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
+
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
@@ -151,6 +167,9 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -201,26 +220,94 @@ type listGroupsResponse struct {
 	} `json:"data"`
 }
 
-func (rh *requestHandler) groups() []apiGroup {
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
+type rulesFilter struct {
+	files         []string
+	groupNames    []string
+	ruleNames     []string
+	ruleType      string
+	excludeAlerts bool
+}
+
+func extractRulesFilter(r *http.Request) rulesFilter {
+	rf := rulesFilter{}
+
+	var ruleType string
+	ruleTypeParam := r.URL.Query().Get("type")
+	// for some reason, `type` in filter doesn't match `type` in response,
+	// so we use this matching here
+	if ruleTypeParam == "alert" {
+		ruleType = ruleTypeAlerting
+	} else if ruleTypeParam == "record" {
+		ruleType = ruleTypeRecording
+	}
+	rf.ruleType = ruleType
+
+	rf.excludeAlerts = httputils.GetBool(r, "exclude_alerts")
+	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
+	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
+	rf.files = append([]string{}, r.Form["file[]"]...)
+	return rf
+}
+
+func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]apiGroup, 0)
-	for _, g := range rh.m.groups {
-		groups = append(groups, groupToAPI(g))
+	isInList := func(list []string, needle string) bool {
+		if len(list) < 1 {
+			return true
+		}
+		for _, i := range list {
+			if i == needle {
+				return true
+			}
+		}
+		return false
 	}
 
-	// sort list of alerts for deterministic output
-	sort.Slice(groups, func(i, j int) bool {
-		return groups[i].Name < groups[j].Name
-	})
+	groups := make([]apiGroup, 0)
+	for _, group := range rh.m.groups {
+		if !isInList(rf.groupNames, group.Name) {
+			continue
+		}
+		if !isInList(rf.files, group.File) {
+			continue
+		}
 
+		g := groupToAPI(group)
+		// the returned list should always be non-nil
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
+		filteredRules := make([]apiRule, 0)
+		for _, r := range g.Rules {
+			if rf.ruleType != "" && rf.ruleType != r.Type {
+				continue
+			}
+			if !isInList(rf.ruleNames, r.Name) {
+				continue
+			}
+			if rf.excludeAlerts {
+				r.Alerts = nil
+			}
+			filteredRules = append(filteredRules, r)
+		}
+		g.Rules = filteredRules
+		groups = append(groups, g)
+	}
+	// sort list of groups for deterministic output
+	sort.Slice(groups, func(i, j int) bool {
+		a, b := groups[i], groups[j]
+		if a.Name != b.Name {
+			return a.Name < b.Name
+		}
+		return a.File < b.File
+	})
 	return groups
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups()
+	lr.Data.Groups = rh.groups(rf)
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{

app/vmalert/web.qtpl

@@ -70,15 +70,29 @@ btn-primary
                 }
             }
         %}
-         <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
-         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-         <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
-         <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
+        <div class="btn-toolbar mb-3" role="toolbar">
+          <div>
+            <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
+            <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+            <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+            <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
+            <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
+          </div>
+          <div class="col-md-4 col-lg-5">
+            <div class="px-3 input-group">
+              <div class="input-group-prepend">
+                <span class="input-group-text">
+                  <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
+                </span>
+              </div>
+              <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
+            </div>
+          </div>
+        </div>
         {%  if len(groups) > 0 %}
             {% for _, g := range groups  %}
                   <div
-                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}">
+                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
                     <span class="anchor" id="group-{%s g.ID %}"></span>
                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                      {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
@@ -100,7 +114,7 @@ btn-primary
                         </div>
                     {% endif %}
                 </div>
-                <div class="collapse" id="rules-{%s g.ID %}">
+                <div class="collapse rule-table" id="rules-{%s g.ID %}">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
                             <tr>
@@ -111,7 +125,7 @@ btn-primary
                         </thead>
                         <tbody>
                         {% for _, r := range g.Rules %}
-                            <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
+                            <tr class="rule{% if r.LastError != "" %} alert-danger{% endif %}" data-rule-name="{%s r.Name %}" data-bs-target="{%s g.ID %}">
                                 <td>
                                     <div class="row">
                                         <div class="col-12 mb-2">
@@ -134,7 +148,7 @@ btn-primary
                                         <div class="col-12 mb-2">
                                             {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
                                             {% for k, v := range r.Labels %}
-                                                    <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                                    <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
                                             {% endfor %}
                                         </div>
                                         {% if r.LastError != "" %}
@@ -170,11 +184,25 @@ btn-primary
     {%code prefix := utils.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {% if len(groupAlerts) > 0 %}
-         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         <div class="btn-toolbar mb-3" role="toolbar">
+              <div>
+                <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+                 <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+              </div>
+              <div class="col-md-4 col-lg-5">
+                <div class="px-3 input-group">
+                  <div class="input-group-prepend">
+                    <span class="input-group-text">
+                      <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
+                    </span>
+                  </div>
+                  <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
+                </div>
+              </div>
+          </div>
          {% for _, ga := range groupAlerts %}
             {%code g := ga.Group %}
-            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
                 <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
@@ -192,7 +220,7 @@ btn-primary
                 }
                 sort.Strings(keys)
             %}
-            <div class="collapse" id="rules-{%s g.ID %}">
+            <div class="collapse rule-table" id="rules-{%s g.ID %}">
                 {% for _, ruleID := range keys %}
                     {%code
                         defaultAR := alertsByRule[ruleID][0]
@@ -203,45 +231,46 @@ btn-primary
                         sort.Strings(labelKeys)
                     %}
                     <br>
-                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
-                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
-                    <br>
-                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
-                    <table class="table table-striped table-hover table-sm">
-                        <thead>
-                            <tr>
-                                <th scope="col">Labels</th>
-                                <th scope="col">State</th>
-                                <th scope="col">Active at</th>
-                                <th scope="col">Value</th>
-                                <th scope="col">Link</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                        {% for _, ar := range alertsByRule[ruleID] %}
-                            <tr>
-                                <td>
-                                    {% for _, k := range labelKeys %}
-                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
-                                    {% endfor %}
-                                </td>
-                                <td>{%= badgeState(ar.State) %}</td>
-                                <td>
-                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
-                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
-                                    {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
-                                </td>
-                                <td>{%s ar.Value %}</td>
-                                <td>
-                                    <a href="{%s prefix+ar.WebLink() %}">Details</a>
-                                </td>
-                            </tr>
-                        {% endfor %}
-                     </tbody>
-                    </table>
+                    <div class="rule" data-rule-name="{%s defaultAR.Name %}" data-bs-target="{%s g.ID %}">
+                      <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                       | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                      <br>
+                      <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                      <table class="table table-striped table-hover table-sm">
+                          <thead>
+                              <tr>
+                                  <th scope="col">Labels</th>
+                                  <th scope="col">State</th>
+                                  <th scope="col">Active at</th>
+                                  <th scope="col">Value</th>
+                                  <th scope="col">Link</th>
+                              </tr>
+                          </thead>
+                          <tbody>
+                          {% for _, ar := range alertsByRule[ruleID] %}
+                              <tr>
+                                  <td>
+                                      {% for _, k := range labelKeys %}
+                                          <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
+                                      {% endfor %}
+                                  </td>
+                                  <td>{%= badgeState(ar.State) %}</td>
+                                  <td>
+                                      {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                      {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                      {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
+                                  </td>
+                                  <td>{%s ar.Value %}</td>
+                                  <td>
+                                      <a href="{%s prefix+ar.WebLink() %}">Details</a>
+                                  </td>
+                              </tr>
+                          {% endfor %}
+                       </tbody>
+                      </table>
+                    </div>
                 {% endfor %}
             </div>
-            <br>
         {% endfor %}
 
     {% else %}

app/vmalert/web_test.go

@@ -23,6 +23,7 @@ func TestHandler(t *testing.T) {
 	})
 	g := &rule.Group{
 		Name:        "group",
+		File:        "rules.yaml",
 		Concurrency: 1,
 	}
 	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
@@ -165,6 +166,74 @@ func TestHandler(t *testing.T) {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
+
+	t.Run("/api/v1/rules&filters", func(t *testing.T) {
+		check := func(url string, expGroups, expRules int) {
+			t.Helper()
+			lr := listGroupsResponse{}
+			getResp(ts.URL+url, &lr, 200)
+			if length := len(lr.Data.Groups); length != expGroups {
+				t.Errorf("expected %d groups got %d", expGroups, length)
+			}
+			if len(lr.Data.Groups) < 1 {
+				return
+			}
+			var rulesN int
+			for _, gr := range lr.Data.Groups {
+				rulesN += len(gr.Rules)
+			}
+			if rulesN != expRules {
+				t.Errorf("expected %d rules got %d", expRules, rulesN)
+			}
+		}
+
+		check("/api/v1/rules?type=alert", 1, 1)
+		check("/api/v1/rules?type=record", 1, 1)
+
+		check("/vmalert/api/v1/rules?type=alert", 1, 1)
+		check("/vmalert/api/v1/rules?type=record", 1, 1)
+
+		// no filtering expected due to bad params
+		check("/api/v1/rules?type=badParam", 1, 2)
+		check("/api/v1/rules?foo=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
+	})
+	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
+		// check if response returns active alerts by default
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml", &lr, 200)
+		activeAlerts := 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts == 0 {
+			t.Fatalf("expected at least 1 active alert in response; got 0")
+		}
+
+		// disable returning alerts via param
+		lr = listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml&exclude_alerts=true", &lr, 200)
+		activeAlerts = 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts != 0 {
+			t.Fatalf("expected to get 0 active alert in response; got %d", activeAlerts)
+		}
+	})
 }
 
 func TestEmptyResponse(t *testing.T) {

app/vmalert/web_types.go

@@ -193,10 +193,15 @@ func ruleToAPI(r interface{}) apiRule {
 	return apiRule{}
 }
 
+const (
+	ruleTypeRecording = "recording"
+	ruleTypeAlerting  = "alerting"
+)
+
 func recordingToAPI(rr *rule.RecordingRule) apiRule {
 	lastState := rule.GetLastEntry(rr)
 	r := apiRule{
-		Type:              "recording",
+		Type:              ruleTypeRecording,
 		DatasourceType:    rr.Type.String(),
 		Name:              rr.Name,
 		Query:             rr.Expr,
@@ -224,7 +229,7 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 func alertingToAPI(ar *rule.AlertingRule) apiRule {
 	lastState := rule.GetLastEntry(ar)
 	r := apiRule{
-		Type:              "alerting",
+		Type:              ruleTypeAlerting,
 		DatasourceType:    ar.Type.String(),
 		Name:              ar.Name,
 		Query:             ar.Expr,

app/vmauth/auth_config.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,12 +17,13 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
@@ -41,6 +43,9 @@ var (
 type AuthConfig struct {
 	Users            []UserInfo `yaml:"users,omitempty"`
 	UnauthorizedUser *UserInfo  `yaml:"unauthorized_user,omitempty"`
+
+	// ms holds all the metrics for the given AuthConfig
+	ms *metrics.Set
 }
 
 // UserInfo is user information read from authConfigPath
@@ -60,12 +65,15 @@ type UserInfo struct {
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
+	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
+	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -156,7 +164,7 @@ type Regex struct {
 
 // URLPrefix represents passed `url_prefix`
 type URLPrefix struct {
-	n uint32
+	n atomic.Uint32
 
 	// the list of backend urls
 	bus []*backendURL
@@ -184,27 +192,28 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 }
 
 type backendURL struct {
-	brokenDeadline     uint64
-	concurrentRequests int32
-	url                *url.URL
+	brokenDeadline     atomic.Uint64
+	concurrentRequests atomic.Int32
+
+	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
 	ct := fasttime.UnixTimestamp()
-	return ct < atomic.LoadUint64(&bu.brokenDeadline)
+	return ct < bu.brokenDeadline.Load()
 }
 
 func (bu *backendURL) setBroken() {
 	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	atomic.StoreUint64(&bu.brokenDeadline, deadline)
+	bu.brokenDeadline.Store(deadline)
 }
 
 func (bu *backendURL) get() {
-	atomic.AddInt32(&bu.concurrentRequests, 1)
+	bu.concurrentRequests.Add(1)
 }
 
 func (bu *backendURL) put() {
-	atomic.AddInt32(&bu.concurrentRequests, -1)
+	bu.concurrentRequests.Add(-1)
 }
 
 func (up *URLPrefix) getBackendsCount() int {
@@ -258,7 +267,7 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 	}
 
 	// Slow path - select other backend urls.
-	n := atomic.AddUint32(&up.n, 1)
+	n := up.n.Add(1)
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -266,20 +275,22 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 		if bu.isBroken() {
 			continue
 		}
-		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
+		if bu.concurrentRequests.Load() == 0 {
 			// Fast path - return the backend with zero concurrently executed requests.
+			// Do not use CompareAndSwap() instead of Load(), since it is much slower on systems with many CPU cores.
+			bu.concurrentRequests.Add(1)
 			return bu
 		}
 	}
 
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMin := bus[n%uint32(len(bus))]
-	minRequests := atomic.LoadInt32(&buMin.concurrentRequests)
+	minRequests := buMin.concurrentRequests.Load()
 	for _, bu := range bus {
 		if bu.isBroken() {
 			continue
 		}
-		if n := atomic.LoadInt32(&bu.concurrentRequests); n < minRequests {
+		if n := bu.concurrentRequests.Load(); n < minRequests {
 			buMin = bu
 			minRequests = n
 		}
@@ -462,17 +473,19 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var authConfig atomic.Pointer[AuthConfig]
-var authUsers atomic.Pointer[map[string]*UserInfo]
-var authConfigWG sync.WaitGroup
-var stopCh chan struct{}
+var (
+	authConfig   atomic.Pointer[AuthConfig]
+	authUsers    atomic.Pointer[map[string]*UserInfo]
+	authConfigWG sync.WaitGroup
+	stopCh       chan struct{}
+)
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
 func loadAuthConfig() (bool, error) {
-	data, err := fs.ReadFileOrHTTP(*authConfigPath)
+	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
@@ -494,6 +507,11 @@ func loadAuthConfig() (bool, error) {
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
+	prevAc := authConfig.Load()
+	if prevAc != nil {
+		metrics.UnregisterSet(prevAc.ms)
+	}
+	metrics.RegisterSet(ac.ms)
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
@@ -506,10 +524,13 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-	var ac AuthConfig
-	if err = yaml.UnmarshalStrict(data, &ac); err != nil {
+	ac := &AuthConfig{
+		ms: metrics.NewSet(),
+	}
+	if err = yaml.UnmarshalStrict(data, ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
+
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		if ui.Username != "" {
@@ -527,23 +548,30 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
-		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
+		}
+		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
+		ui.concurrencyLimitReached = ac.ms.NewCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
 		ui.httpTransport = tr
 	}
-	return &ac, nil
+	return ac, nil
 }
 
 func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
@@ -554,43 +582,47 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		if ui.BearerToken == "" && ui.Username == "" {
-			return nil, fmt.Errorf("either bearer_token or username must be set")
+		if ui.Username != "" && ui.Password == "" {
+			// Do not allow setting username without password if there are other auth configs exist.
+			// This should prevent from typical mis-configuration when access by username without password
+			// remains open if other authorization schemes are defined.
+			if ui.BearerToken != "" {
+				return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+			}
 		}
-		if ui.BearerToken != "" && ui.Username != "" {
-			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+		ats := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if len(ats) == 0 {
+			return nil, fmt.Errorf("one of bearer_token, username or mtls must be set")
 		}
-		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[at1] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
-		}
-		if byAuthToken[at2] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
+		for _, at := range ats {
+			if uiOld := byAuthToken[at]; uiOld != nil {
+				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
+					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
+			}
 		}
 
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
-		name := ui.name()
-		if ui.BearerToken != "" {
-			if ui.Password != "" {
-				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			}
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		if ui.BearerToken != "" && ui.Password != "" {
+			return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 		}
-		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
+		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -600,12 +632,36 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		}
 		ui.httpTransport = tr
 
-		byAuthToken[at1] = ui
-		byAuthToken[at2] = ui
+		for _, at := range ats {
+			byAuthToken[at] = ui
+		}
 	}
 	return byAuthToken, nil
 }
 
+var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
+
+func (ui *UserInfo) getMetricLabels() (string, error) {
+	name := ui.name()
+	if len(name) == 0 && len(ui.MetricLabels) == 0 {
+		// fast path
+		return "", nil
+	}
+	labels := make([]string, 0, len(ui.MetricLabels)+1)
+	if len(name) > 0 {
+		labels = append(labels, fmt.Sprintf(`username=%q`, name))
+	}
+	for k, v := range ui.MetricLabels {
+		if !labelNameRegexp.MatchString(k) {
+			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
+		}
+		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
+	}
+	sort.Strings(labels)
+	labelsStr := "{" + strings.Join(labels, ",") + "}"
+	return labelsStr, nil
+}
+
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
@@ -676,29 +732,51 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		return "bearer_token"
+		h := xxhash.Sum64([]byte(ui.BearerToken))
+		return fmt.Sprintf("bearer_token:hash:%016X", h)
 	}
 	return ""
 }
 
-func getAuthTokens(bearerToken, username, password string) (string, string) {
+func getAuthTokens(bearerToken, username, password string) []string {
+	var ats []string
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getAuthToken(bearerToken, "", "")
-		at2 := getAuthToken("", bearerToken, "")
-		return at1, at2
+		at1 := getHTTPAuthBearerToken(bearerToken)
+		at2 := getHTTPAuthBasicToken(bearerToken, "")
+		ats = append(ats, at1, at2)
+	} else if username != "" {
+		at := getHTTPAuthBasicToken(username, password)
+		ats = append(ats, at)
 	}
-	at := getAuthToken("", username, password)
-	return at, at
+	return ats
 }
 
-func getAuthToken(bearerToken, username, password string) string {
-	if bearerToken != "" {
-		return "Bearer " + bearerToken
-	}
+func getHTTPAuthBearerToken(bearerToken string) string {
+	return "http_auth:Bearer " + bearerToken
+}
+
+func getHTTPAuthBasicToken(username, password string) string {
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "Basic " + token64
+	return "http_auth:Basic " + token64
+}
+
+func getAuthTokensFromRequest(r *http.Request) []string {
+	var ats []string
+
+	ah := r.Header.Get("Authorization")
+	if ah == "" {
+		return ats
+	}
+	if strings.HasPrefix(ah, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		ah = strings.Replace(ah, "Token", "Bearer", 1)
+	}
+	at := "http_auth:" + ah
+	ats = append(ats, at)
+	return ats
 }
 
 func (up *URLPrefix) sanitize() error {

app/vmauth/auth_config_test.go

@@ -229,6 +229,14 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
+`)
+	// Invalid metric label name
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  metric_labels:
+    not-prometheus-compatible: value
 `)
 }
 
@@ -259,7 +267,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -282,7 +290,7 @@ users:
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
@@ -304,11 +312,11 @@ users:
 - username: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "bar", ""): {
+		getHTTPAuthBasicToken("bar", ""): {
 			Username:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
@@ -328,7 +336,7 @@ users:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -357,7 +365,7 @@ users:
 				},
 			},
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -387,7 +395,7 @@ users:
 			},
 		},
 	})
-	// Multiple users with the same name
+	// Multiple users with the same name - this should work, since these users have different passwords
 	f(`
 users:
 - username: foo-same
@@ -397,17 +405,18 @@ users:
   password: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getAuthToken("", "foo-same", "baz"): {
+		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "foo-same", "bar"): {
+		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
+
 	// with default url
 	f(`
 users:
@@ -424,7 +433,7 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -456,7 +465,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -489,7 +498,41 @@ users:
 			}),
 		},
 	})
-
+	// With metric_labels
+	f(`
+users:
+- username: foo-same
+  password: baz
+  url_prefix: http://foo
+  metric_labels:
+    dc: eu
+    team: dev
+- username: foo-same
+  password: bar
+  url_prefix: https://bar/x///
+  metric_labels:
+    backend_env: test
+    team: accounting
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo-same", "baz"): {
+			Username:  "foo-same",
+			Password:  "baz",
+			URLPrefix: mustParseURL("http://foo"),
+			MetricLabels: map[string]string{
+				"dc":   "eu",
+				"team": "dev",
+			},
+		},
+		getHTTPAuthBasicToken("foo-same", "bar"): {
+			Username:  "foo-same",
+			Password:  "bar",
+			URLPrefix: mustParseURL("https://bar/x"),
+			MetricLabels: map[string]string{
+				"backend_env": "test",
+				"team":        "accounting",
+			},
+		},
+	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -516,7 +559,7 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getAuthToken("", "foo", "bar")]
+	ui := m[getHTTPAuthBasicToken("foo", "bar")]
 	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
@@ -526,6 +569,86 @@ unauthorized_user:
 	}
 }
 
+func TestUserInfoGetMetricLabels(t *testing.T) {
+	t.Run("empty-labels", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-username", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-name", func(t *testing.T) {
+		ui := &UserInfo{
+			Name:        "user1",
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-bearer-token", func(t *testing.T) {
+		ui := &UserInfo{
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("invalid-label", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "foo",
+			MetricLabels: map[string]string{
+				",{": "aaaa",
+			},
+		}
+		_, err := ui.getMetricLabels()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	})
+}
+
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false

app/vmauth/example_config.yml

@@ -10,6 +10,11 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Adds labels to the exported metrics for given user section
+  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
+  metric_labels:
+    backend_dc: eu
+    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.

app/vmauth/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -33,8 +33,8 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
@@ -45,7 +45,7 @@ var (
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
@@ -65,10 +65,14 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	logger.Infof("starting vmauth at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8427"}
+	}
+	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -77,8 +81,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -89,7 +93,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, *reloadAuthKey, "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -97,8 +101,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
-	authToken := r.Header.Get("Authorization")
-	if authToken == "" {
+
+	ats := getAuthTokensFromRequest(r)
+	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
@@ -110,18 +115,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
-	if strings.HasPrefix(authToken, "Token ") {
-		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-		// See https://docs.influxdata.com/influxdb/v2.0/api/
-		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
-	}
 
-	ac := *authUsers.Load()
-	ui := ac[authToken]
+	ui := getUserInfoByAuthTokens(ats)
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
+			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
@@ -137,6 +136,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+func getUserInfoByAuthTokens(ats []string) *UserInfo {
+	ac := *authUsers.Load()
+	for _, at := range ats {
+		ui := ac[at]
+		if ui != nil {
+			return ui
+		}
+	}
+	return nil
+}
+
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
@@ -150,12 +160,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if err := ui.beginConcurrencyLimit(); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
+
+			// Requests failed because of concurrency limit must be counted as errors,
+			// since this usually means the backend cannot keep up with the current load.
+			ui.backendErrors.Inc()
 			return
 		}
 	default:
 		concurrentRequestsLimitReached.Inc()
 		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
 		handleConcurrencyLimitError(w, r, err)
+
+		// Requests failed because of concurrency limit must be counted as errors,
+		// since this usually means the backend cannot keep up with the current load.
+		ui.backendErrors.Inc()
 		return
 	}
 	processRequest(w, r, ui)
@@ -201,7 +219,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -213,15 +231,16 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
+	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	req.Host = targetURL.Host
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := transport.RoundTrip(req)
+	res, err := ui.httpTransport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -229,15 +248,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				ui.backendErrors.Inc()
+			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
+				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -247,7 +271,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
 		return false
 	}
-	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
+	if hasInt(retryStatusCodes, res.StatusCode) {
+		_ = res.Body.Close()
+		if !rtbOK || !rtb.canRetry() {
+			// If we get an error from the retry_status_codes list, but cannot execute retry,
+			// we consider such a request an error as well.
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+					res.StatusCode, targetURL),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
+			return true
+		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
@@ -266,6 +303,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
 	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
 	copyBufPool.Put(copyBuf)
+	_ = res.Body.Close()
 	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
@@ -393,8 +431,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var transportMap = make(map[string]*http.Transport)
-var transportMapLock sync.Mutex
+var (
+	transportMap     = make(map[string]*http.Transport)
+	transportMapLock sync.Mutex
+)
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)
@@ -422,7 +462,7 @@ func newTransport(insecureSkipVerify bool, caFile string) (*http.Transport, erro
 		tlsCfg.ClientSessionCache = tls.NewLRUClientSessionCache(0)
 		tlsCfg.InsecureSkipVerify = insecureSkipVerify
 		if caFile != "" {
-			data, err := fs.ReadFileOrHTTP(caFile)
+			data, err := fscore.ReadFileOrHTTP(caFile)
 			if err != nil {
 				return nil, fmt.Errorf("cannot read tls_ca_file: %w", err)
 			}

app/vmbackup/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
 )
 
 var (
@@ -93,7 +94,8 @@ func main() {
 		}
 	}
 
-	go httpserver.Serve(*httpListenAddr, false, nil)
+	listenAddrs := []string{*httpListenAddr}
+	go httpserver.Serve(listenAddrs, nil, nil)
 
 	pushmetrics.Init()
 	err := makeBackup()
@@ -104,8 +106,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
@@ -168,7 +170,7 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
 }
 
 func newSrcFS() (*fslocal.FS, error) {
-	if err := snapshot.Validate(*snapshotName); err != nil {
+	if err := snapshotutil.Validate(*snapshotName); err != nil {
 		return nil, fmt.Errorf("invalid -snapshotName=%q: %w", *snapshotName, err)
 	}
 	snapshotPath := filepath.Join(*storageDataPath, "snapshots", *snapshotName)

app/vmctl/backoff/backoff_test.go

@@ -15,7 +15,6 @@ func TestRetry_Do(t *testing.T) {
 		backoffFactor      float64
 		backoffMinDuration time.Duration
 		retryableFunc      retryableFunc
-		ctx                context.Context
 		cancelTimeout      time.Duration
 		want               uint64
 		wantErr            bool
@@ -25,7 +24,6 @@ func TestRetry_Do(t *testing.T) {
 			retryableFunc: func() error {
 				return ErrBadRequest
 			},
-			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -35,7 +33,6 @@ func TestRetry_Do(t *testing.T) {
 				time.Sleep(time.Millisecond * 100)
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -58,7 +55,6 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    1,
 			wantErr: false,
 		},
@@ -75,7 +71,6 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
-			ctx:     context.Background(),
 			want:    5,
 			wantErr: true,
 		},
@@ -85,14 +80,8 @@ func TestRetry_Do(t *testing.T) {
 			backoffFactor:      1.7,
 			backoffMinDuration: time.Millisecond * 10,
 			retryableFunc: func() error {
-				t := time.NewTicker(time.Millisecond * 5)
-				defer t.Stop()
-				for range t.C {
-					return fmt.Errorf("got some error")
-				}
-				return nil
+				return fmt.Errorf("got some error")
 			},
-			ctx:           context.Background(),
 			cancelTimeout: time.Millisecond * 40,
 			want:          3,
 			wantErr:       true,
@@ -101,12 +90,13 @@ func TestRetry_Do(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			r := &Backoff{retries: tt.backoffRetries, factor: tt.backoffFactor, minDuration: tt.backoffMinDuration}
+			ctx := context.Background()
 			if tt.cancelTimeout != 0 {
-				newCtx, cancelFn := context.WithTimeout(tt.ctx, tt.cancelTimeout)
-				tt.ctx = newCtx
+				newCtx, cancelFn := context.WithTimeout(context.Background(), tt.cancelTimeout)
+				ctx = newCtx
 				defer cancelFn()
 			}
-			got, err := r.Retry(tt.ctx, tt.retryableFunc)
+			got, err := r.Retry(ctx, tt.retryableFunc)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
 				return

app/vmctl/flags.go

@@ -123,15 +123,20 @@ var (
 )
 
 const (
-	otsdbAddr        = "otsdb-addr"
-	otsdbConcurrency = "otsdb-concurrency"
-	otsdbQueryLimit  = "otsdb-query-limit"
-	otsdbOffsetDays  = "otsdb-offset-days"
-	otsdbHardTSStart = "otsdb-hard-ts-start"
-	otsdbRetentions  = "otsdb-retentions"
-	otsdbFilters     = "otsdb-filters"
-	otsdbNormalize   = "otsdb-normalize"
-	otsdbMsecsTime   = "otsdb-msecstime"
+	otsdbAddr               = "otsdb-addr"
+	otsdbConcurrency        = "otsdb-concurrency"
+	otsdbQueryLimit         = "otsdb-query-limit"
+	otsdbOffsetDays         = "otsdb-offset-days"
+	otsdbHardTSStart        = "otsdb-hard-ts-start"
+	otsdbRetentions         = "otsdb-retentions"
+	otsdbFilters            = "otsdb-filters"
+	otsdbNormalize          = "otsdb-normalize"
+	otsdbMsecsTime          = "otsdb-msecstime"
+	otsdbCertFile           = "otsdb-cert-file"
+	otsdbKeyFile            = "otsdb-key-file"
+	otsdbCAFile             = "otsdb-CA-file"
+	otsdbServerName         = "otsdb-server-name"
+	otsdbInsecureSkipVerify = "otsdb-insecure-skip-verify"
 )
 
 var (
@@ -191,6 +196,27 @@ var (
 			Value: false,
 			Usage: "Whether to normalize all data received to lower case before forwarding to VictoriaMetrics",
 		},
+		&cli.StringFlag{
+			Name:  otsdbCertFile,
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr",
+		},
+		&cli.StringFlag{
+			Name:  otsdbKeyFile,
+			Usage: "Optional path to client-side TLS key to use when connecting to -otsdb-addr",
+		},
+		&cli.StringFlag{
+			Name:  otsdbCAFile,
+			Usage: "Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used",
+		},
+		&cli.StringFlag{
+			Name:  otsdbServerName,
+			Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from otsdbAddr is used",
+		},
+		&cli.BoolFlag{
+			Name:  otsdbInsecureSkipVerify,
+			Usage: "Whether to skip tls verification when connecting to -otsdb-addr",
+			Value: false,
+		},
 	}
 )
 
@@ -208,6 +234,11 @@ const (
 	influxMeasurementFieldSeparator = "influx-measurement-field-separator"
 	influxSkipDatabaseLabel         = "influx-skip-database-label"
 	influxPrometheusMode            = "influx-prometheus-mode"
+	influxCertFile                  = "influx-cert-file"
+	influxKeyFile                   = "influx-key-file"
+	influxCAFile                    = "influx-CA-file"
+	influxServerName                = "influx-server-name"
+	influxInsecureSkipVerify        = "influx-insecure-skip-verify"
 )
 
 var (
@@ -272,7 +303,28 @@ var (
 		},
 		&cli.BoolFlag{
 			Name:  influxPrometheusMode,
-			Usage: "Wether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write.",
+			Usage: "Whether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write.",
+			Value: false,
+		},
+		&cli.StringFlag{
+			Name:  influxCertFile,
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to -influx-addr",
+		},
+		&cli.StringFlag{
+			Name:  influxKeyFile,
+			Usage: "Optional path to client-side TLS key to use when connecting to -influx-addr",
+		},
+		&cli.StringFlag{
+			Name:  influxCAFile,
+			Usage: "Optional path to TLS CA file to use for verifying connections to -influx-addr. By default, system CA is used",
+		},
+		&cli.StringFlag{
+			Name:  influxServerName,
+			Usage: "Optional TLS server name to use for connections to -influx-addr. By default, the server name from -influx-addr is used",
+		},
+		&cli.BoolFlag{
+			Name:  influxInsecureSkipVerify,
+			Usage: "Whether to skip tls verification when connecting to -influx-addr",
 			Value: false,
 		},
 	}
@@ -496,6 +548,10 @@ const (
 	remoteReadPassword           = "remote-read-password"
 	remoteReadHTTPTimeout        = "remote-read-http-timeout"
 	remoteReadHeaders            = "remote-read-headers"
+	remoteReadCertFile           = "remote-read-cert-file"
+	remoteReadKeyFile            = "remote-read-key-file"
+	remoteReadCAFile             = "remote-read-CA-file"
+	remoteReadServerName         = "remote-read-server-name"
 	remoteReadInsecureSkipVerify = "remote-read-insecure-skip-verify"
 	remoteReadDisablePathAppend  = "remote-read-disable-path-append"
 )
@@ -574,6 +630,22 @@ var (
 				"For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. \n" +
 				"Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'",
 		},
+		&cli.StringFlag{
+			Name:  remoteReadCertFile,
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to -remote-read-src-addr",
+		},
+		&cli.StringFlag{
+			Name:  remoteReadKeyFile,
+			Usage: "Optional path to client-side TLS key to use when connecting to -remote-read-src-addr",
+		},
+		&cli.StringFlag{
+			Name:  remoteReadCAFile,
+			Usage: "Optional path to TLS CA file to use for verifying connections to -remote-read-src-addr. By default, system CA is used",
+		},
+		&cli.StringFlag{
+			Name:  remoteReadServerName,
+			Usage: "Optional TLS server name to use for connections to remoteReadSrcAddr. By default, the server name from -remote-read-src-addr is used",
+		},
 		&cli.BoolFlag{
 			Name:  remoteReadInsecureSkipVerify,
 			Usage: "Whether to skip TLS certificate verification when connecting to the remote read address",

app/vmctl/influx/influx.go

@@ -1,6 +1,7 @@
 package influx
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io"
 	"log"
@@ -33,7 +34,8 @@ type Config struct {
 	Retention string
 	ChunkSize int
 
-	Filter Filter
+	Filter    Filter
+	TLSConfig *tls.Config
 }
 
 // Filter contains configuration for filtering
@@ -86,10 +88,10 @@ type LabelPair struct {
 // configured with passed Config
 func NewClient(cfg Config) (*Client, error) {
 	c := influx.HTTPConfig{
-		Addr:               cfg.Addr,
-		Username:           cfg.Username,
-		Password:           cfg.Password,
-		InsecureSkipVerify: true,
+		Addr:      cfg.Addr,
+		Username:  cfg.Username,
+		Password:  cfg.Password,
+		TLSConfig: cfg.TLSConfig,
 	}
 	hc, err := influx.NewHTTPClient(c)
 	if err != nil {

app/vmctl/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 )
@@ -49,8 +50,20 @@ func main() {
 				Action: func(c *cli.Context) error {
 					fmt.Println("OpenTSDB import mode")
 
+					// create Transport with given TLS config
+					certFile := c.String(otsdbCertFile)
+					keyFile := c.String(otsdbKeyFile)
+					caFile := c.String(otsdbCAFile)
+					serverName := c.String(otsdbServerName)
+					insecureSkipVerify := c.Bool(otsdbInsecureSkipVerify)
+					addr := c.String(otsdbAddr)
+
+					tr, err := httputils.Transport(addr, certFile, caFile, keyFile, serverName, insecureSkipVerify)
+					if err != nil {
+						return fmt.Errorf("failed to create Transport: %s", err)
+					}
 					oCfg := opentsdb.Config{
-						Addr:       c.String(otsdbAddr),
+						Addr:       addr,
 						Limit:      c.Int(otsdbQueryLimit),
 						Offset:     c.Int64(otsdbOffsetDays),
 						HardTS:     c.Int64(otsdbHardTSStart),
@@ -58,6 +71,7 @@ func main() {
 						Filters:    c.StringSlice(otsdbFilters),
 						Normalize:  c.Bool(otsdbNormalize),
 						MsecsTime:  c.Bool(otsdbMsecsTime),
+						Transport:  tr,
 					}
 					otsdbClient, err := opentsdb.NewClient(oCfg)
 					if err != nil {
@@ -84,6 +98,18 @@ func main() {
 				Action: func(c *cli.Context) error {
 					fmt.Println("InfluxDB import mode")
 
+					// create TLS config
+					certFile := c.String(influxCertFile)
+					keyFile := c.String(influxKeyFile)
+					caFile := c.String(influxCAFile)
+					serverName := c.String(influxServerName)
+					insecureSkipVerify := c.Bool(influxInsecureSkipVerify)
+
+					tc, err := httputils.TLSConfig(certFile, caFile, keyFile, serverName, insecureSkipVerify)
+					if err != nil {
+						return fmt.Errorf("failed to create TLS Config: %s", err)
+					}
+
 					iCfg := influx.Config{
 						Addr:      c.String(influxAddr),
 						Username:  c.String(influxUser),
@@ -96,7 +122,9 @@ func main() {
 							TimeEnd:   c.String(influxFilterTimeEnd),
 						},
 						ChunkSize: c.Int(influxChunkSize),
+						TLSConfig: tc,
 					}
+
 					influxClient, err := influx.NewClient(iCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create influx client: %s", err)
@@ -134,6 +162,10 @@ func main() {
 						Headers:            c.String(remoteReadHeaders),
 						LabelName:          c.String(remoteReadFilterLabel),
 						LabelValue:         c.String(remoteReadFilterLabelValue),
+						CertFile:           c.String(remoteReadCertFile),
+						KeyFile:            c.String(remoteReadKeyFile),
+						CAFile:             c.String(remoteReadCAFile),
+						ServerName:         c.String(remoteReadServerName),
 						InsecureSkipVerify: c.Bool(remoteReadInsecureSkipVerify),
 						DisablePathAppend:  c.Bool(remoteReadDisablePathAppend),
 					})
@@ -298,14 +330,14 @@ func main() {
 					if err != nil {
 						return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1)
 					}
-					var blocksCount uint64
+					var blocksCount atomic.Uint64
 					if err := stream.Parse(f, isBlockGzipped, func(block *stream.Block) error {
-						atomic.AddUint64(&blocksCount, 1)
+						blocksCount.Add(1)
 						return nil
 					}); err != nil {
-						return cli.Exit(fmt.Errorf("cannot parse block at path=%q, blocksCount=%d, err=%w", blockPath, blocksCount, err), 1)
+						return cli.Exit(fmt.Errorf("cannot parse block at path=%q, blocksCount=%d, err=%w", blockPath, blocksCount.Load(), err), 1)
 					}
-					log.Printf("successfully verified block at path=%q, blockCount=%d", blockPath, blocksCount)
+					log.Printf("successfully verified block at path=%q, blockCount=%d", blockPath, blocksCount.Load())
 					return nil
 				},
 			},

app/vmctl/opentsdb/opentsdb.go

@@ -47,6 +47,8 @@ type Client struct {
 	Normalize  bool
 	HardTS     int64
 	MsecsTime  bool
+
+	c *http.Client
 }
 
 // Config contains fields required
@@ -60,6 +62,7 @@ type Config struct {
 	Filters    []string
 	Normalize  bool
 	MsecsTime  bool
+	Transport  *http.Transport
 }
 
 // TimeRange contains data about time ranges to query
@@ -107,7 +110,8 @@ type Metric struct {
 // FindMetrics discovers all metrics that OpenTSDB knows about (given a filter)
 // e.g. /api/suggest?type=metrics&q=system&max=100000
 func (c Client) FindMetrics(q string) ([]string, error) {
-	resp, err := http.Get(q)
+
+	resp, err := c.c.Get(q)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
@@ -131,7 +135,7 @@ func (c Client) FindMetrics(q string) ([]string, error) {
 // e.g. /api/search/lookup?m=system.load5&limit=1000000
 func (c Client) FindSeries(metric string) ([]Meta, error) {
 	q := fmt.Sprintf("%s/api/search/lookup?m=%s&limit=%d", c.Addr, metric, c.Limit)
-	resp, err := http.Get(q)
+	resp, err := c.c.Get(q)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err)
 	}
@@ -184,7 +188,7 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64, m
 		series.Metric, tagStr)
 
 	q := fmt.Sprintf("%s/api/query?%s", c.Addr, queryStr)
-	resp, err := http.Get(q)
+	resp, err := c.c.Get(q)
 	if err != nil {
 		return Metric{}, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
@@ -325,6 +329,7 @@ func NewClient(cfg Config) (*Client, error) {
 		Normalize:  cfg.Normalize,
 		HardTS:     cfg.HardTS,
 		MsecsTime:  cfg.MsecsTime,
+		c:          &http.Client{Transport: cfg.Transport},
 	}
 	return client, nil
 }

app/vmctl/remoteread/remoteread.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
@@ -64,6 +64,13 @@ type Config struct {
 	// LabelName, LabelValue stands for label=~value pair used for read requests.
 	// Is optional.
 	LabelName, LabelValue string
+
+	// Optional cert file, key file, CA file and server name for client side TLS configuration
+	CertFile   string
+	KeyFile    string
+	CAFile     string
+	ServerName string
+
 	// TLSSkipVerify defines whether to skip TLS certificate verification when connecting to the remote read address.
 	InsecureSkipVerify bool
 }
@@ -103,10 +110,15 @@ func NewClient(cfg Config) (*Client, error) {
 		}
 	}
 
+	tr, err := httputils.Transport(cfg.Addr, cfg.CertFile, cfg.KeyFile, cfg.CAFile, cfg.ServerName, cfg.InsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %s", err)
+	}
+
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.Timeout,
-			Transport: utils.Transport(cfg.Addr, cfg.InsecureSkipVerify),
+			Transport: tr,
 		},
 		addr:              strings.TrimSuffix(cfg.Addr, "/"),
 		disablePathAppend: cfg.DisablePathAppend,
@@ -170,7 +182,7 @@ func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback
 	if c.disablePathAppend {
 		u = c.addr
 	}
-	req, err := http.NewRequest(http.MethodPost, u, r)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
@@ -183,7 +195,7 @@ func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback
 	}
 	req.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0")
 
-	resp, err := c.do(req.WithContext(ctx))
+	resp, err := c.do(req)
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
 			req.URL.Redacted(), err, len(data), r.Size())

app/vmctl/time.go

@@ -1,4 +1,4 @@
-package utils
+package main
 
 import (
 	"fmt"
@@ -13,13 +13,11 @@ const (
 	maxTimeMsecs = int64(1<<63-1) / 1e6
 )
 
-// GetTime  returns time from the given string.
-func GetTime(s string) (time.Time, error) {
-	secs, err := promutils.ParseTime(s)
+func parseTime(s string) (time.Time, error) {
+	msecs, err := promutils.ParseTimeMsec(s)
 	if err != nil {
 		return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)
 	}
-	msecs := int64(secs * 1e3)
 	if msecs < minTimeMsecs {
 		msecs = 0
 	}

app/vmctl/time_test.go

@@ -1,4 +1,4 @@
-package utils
+package main
 
 import (
 	"testing"
@@ -165,7 +165,7 @@ func TestGetTime(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetTime(tt.s)
+			got, err := parseTime(tt.s)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr)
 				return

app/vmctl/utils/tls.go

@@ -1,25 +0,0 @@
-package utils
-
-import (
-	"crypto/tls"
-	"net/http"
-	"strings"
-)
-
-// Transport creates http.Transport object based on provided URL.
-// Returns Transport with TLS configuration if URL contains `https` prefix
-func Transport(URL string, insecureSkipVerify bool) *http.Transport {
-	t := http.DefaultTransport.(*http.Transport).Clone()
-	if !strings.HasPrefix(URL, "https") {
-		return t
-	}
-	t.TLSClientConfig = TLSConfig(insecureSkipVerify)
-	return t
-}
-
-// TLSConfig creates tls.Config object from provided arguments
-func TLSConfig(insecureSkipVerify bool) *tls.Config {
-	return &tls.Config{
-		InsecureSkipVerify: insecureSkipVerify,
-	}
-}

app/vmctl/vm_native.go

@@ -16,7 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -54,14 +53,14 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 		startTime: time.Now(),
 	}
 
-	start, err := utils.GetTime(p.filter.TimeStart)
+	start, err := parseTime(p.filter.TimeStart)
 	if err != nil {
 		return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeStart, p.filter.TimeStart, err)
 	}
 
 	end := time.Now().In(start.Location())
 	if p.filter.TimeEnd != "" {
-		end, err = utils.GetTime(p.filter.TimeEnd)
+		end, err = parseTime(p.filter.TimeEnd)
 		if err != nil {
 			return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeEnd, p.filter.TimeEnd, err)
 		}

app/vminsert/datadogsketches/request_handler.go

@@ -0,0 +1,85 @@
+package datadogsketches
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="datadogsketches"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="datadogsketches"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
+		return insertRows(sketches, extraLabels)
+	})
+}
+
+func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsLen := 0
+	for _, sketch := range sketches {
+		rowsLen += sketch.RowsCount()
+	}
+	ctx.Reset(rowsLen)
+	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
+	for _, sketch := range sketches {
+		ms := sketch.ToSummary()
+		for _, m := range ms {
+			ctx.Labels = ctx.Labels[:0]
+			ctx.AddLabel("", m.Name)
+			for _, label := range m.Labels {
+				ctx.AddLabel(label.Name, label.Value)
+			}
+			for _, tag := range sketch.Tags {
+				name, value := datadogutils.SplitTag(tag)
+				if name == "host" {
+					name = "exported_host"
+				}
+				ctx.AddLabel(name, value)
+			}
+			for j := range extraLabels {
+				label := &extraLabels[j]
+				ctx.AddLabel(label.Name, label.Value)
+			}
+			if hasRelabeling {
+				ctx.ApplyRelabeling()
+			}
+			if len(ctx.Labels) == 0 {
+				// Skip metric without labels.
+				continue
+			}
+			ctx.SortLabelsIfNeeded()
+			var metricNameRaw []byte
+			var err error
+			for _, p := range m.Points {
+				metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, p.Timestamp, p.Value)
+				if err != nil {
+					return err
+				}
+			}
+			rowsTotal += len(m.Points)
+		}
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/main.go

@@ -6,13 +6,13 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	vminsertCommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -29,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
@@ -62,7 +63,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )
@@ -214,7 +216,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		addInfluxResponseHeaders(w)
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push":
+	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(r); err != nil {
 			opentelemetryPushErrors.Inc()
@@ -269,6 +271,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "/datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -315,7 +326,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -324,7 +335,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -334,12 +345,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
-		w.WriteHeader(http.StatusNoContent)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrape config to init targets, configs left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -389,13 +403,16 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
+	datadogsketchesWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+
 	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
@@ -417,12 +434,12 @@ var (
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
 
 	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
-		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+		return float64(storage.MetricsWithDroppedLabels.Load())
 	})
 	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
-		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+		return float64(storage.TooLongLabelNames.Load())
 	})
 	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
-		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+		return float64(storage.TooLongLabelValues.Load())
 	})
 )

app/vminsert/opentelemetry/request_handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -24,10 +25,15 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
-		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
+			processBody = firehose.ProcessRequestBody
+		} else {
+			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		}
 	}
-	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(tss, extraLabels)
 	})
 }

app/vmrestore/main.go

@@ -37,7 +37,8 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	go httpserver.Serve(*httpListenAddr, false, nil)
+	listenAddrs := []string{*httpListenAddr}
+	go httpserver.Serve(listenAddrs, nil, nil)
 
 	srcFS, err := newSrcFS()
 	if err != nil {
@@ -62,8 +63,8 @@ func main() {
 	dstFS.MustStop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())

app/vmselect/graphite/transform.go

@@ -3842,18 +3842,18 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 		errCh <- err
 		close(errCh)
 	}()
-	var skipProcessing uint32
+	var skipProcessing atomic.Bool
 	for i := 0; i < goroutines; i++ {
 		go func() {
 			defer wg.Done()
 			for s := range seriesCh {
-				if atomic.LoadUint32(&skipProcessing) != 0 {
+				if skipProcessing.Load() {
 					continue
 				}
 				sNew, err := f(s)
 				if err != nil {
 					// Drain the rest of series and do not call f for them in order to conserve CPU time.
-					atomic.StoreUint32(&skipProcessing, 1)
+					skipProcessing.Store(true)
 					resultCh <- &result{
 						err: err,
 					}
@@ -5609,9 +5609,9 @@ func (nsf *nextSeriesFunc) peekStep(step int64) (int64, error) {
 	if s != nil {
 		step = s.step
 	}
-	calls := uint64(0)
+	var calls atomic.Uint64
 	*nsf = func() (*series, error) {
-		if atomic.AddUint64(&calls, 1) == 1 {
+		if calls.Add(1) == 1 {
 			return s, nil
 		}
 		return nextSeries()

app/vmselect/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
@@ -29,13 +30,13 @@ import (
 )
 
 var (
-	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
+	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	resetCacheAuthKey    = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
+	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
@@ -148,8 +149,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
 					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
+				StatusCode: http.StatusTooManyRequests,
 			}
+			w.Header().Add("Retry-After", "10")
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -170,7 +172,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	if path == "/internal/resetRollupResultCache" {
-		if !httpserver.CheckAuthFlag(w, r, *resetCacheAuthKey, "resetCacheAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, resetCacheAuthKey.Get(), "resetCacheAuthKey") {
 			return true
 		}
 		promql.ResetRollupResultCache()
@@ -367,7 +369,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
-		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
 			return true
 		}
 		graphiteTagsDelSeriesRequests.Inc()
@@ -386,7 +388,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
-		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
 			return true
 		}
 		deleteRequests.Inc()
@@ -425,6 +427,14 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		}
 		return true
 	}
+	if path == "/vmui/timezone" {
+		httpserver.EnableCORS(w, r)
+		if err := handleVMUITimezone(w); err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	}
 	if strings.HasPrefix(path, "/vmui/") {
 		if strings.HasPrefix(path, "/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.

app/vmselect/netstorage/netstorage.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -22,11 +23,14 @@ import (
 )
 
 var (
-	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned from /api/v1/labels")
-	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned from /api/v1/label/<label_name>/values")
-	maxSamplesPerSeries   = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
-	maxSamplesPerQuery    = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
-	maxWorkersPerQuery    = flag.Int("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "The maximum number of CPU cores a single query can use. "+
+	maxTagKeysPerSearch = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned from /api/v1/labels . "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned from /api/v1/label/<label_name>/values . "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxSamplesPerSeries = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
+	maxSamplesPerQuery  = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. "+
+		"This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
+	maxWorkersPerQuery = flag.Int("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "The maximum number of CPU cores a single query can use. "+
 		"The default value should work good for most cases. "+
 		"The flag can be set to lower values for improving performance of big number of concurrently executed queries. "+
 		"The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). "+
@@ -79,7 +83,7 @@ func (rss *Results) mustClose() {
 }
 
 type timeseriesWork struct {
-	mustStop *uint32
+	mustStop *atomic.Bool
 	rss      *Results
 	pts      *packedTimeseries
 	f        func(rs *Result, workerID uint) error
@@ -88,47 +92,23 @@ type timeseriesWork struct {
 	rowsProcessed int
 }
 
-func (tsw *timeseriesWork) reset() {
-	tsw.mustStop = nil
-	tsw.rss = nil
-	tsw.pts = nil
-	tsw.f = nil
-	tsw.err = nil
-	tsw.rowsProcessed = 0
-}
-
-func getTimeseriesWork() *timeseriesWork {
-	v := tswPool.Get()
-	if v == nil {
-		v = &timeseriesWork{}
-	}
-	return v.(*timeseriesWork)
-}
-
-func putTimeseriesWork(tsw *timeseriesWork) {
-	tsw.reset()
-	tswPool.Put(tsw)
-}
-
-var tswPool sync.Pool
-
 func (tsw *timeseriesWork) do(r *Result, workerID uint) error {
-	if atomic.LoadUint32(tsw.mustStop) != 0 {
+	if tsw.mustStop.Load() {
 		return nil
 	}
 	rss := tsw.rss
 	if rss.deadline.Exceeded() {
-		atomic.StoreUint32(tsw.mustStop, 1)
+		tsw.mustStop.Store(true)
 		return fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 	}
 	if err := tsw.pts.Unpack(r, rss.tbf, rss.tr); err != nil {
-		atomic.StoreUint32(tsw.mustStop, 1)
+		tsw.mustStop.Store(true)
 		return fmt.Errorf("error during time series unpacking: %w", err)
 	}
 	tsw.rowsProcessed = len(r.Timestamps)
 	if len(r.Timestamps) > 0 {
 		if err := tsw.f(r, workerID); err != nil {
-			atomic.StoreUint32(tsw.mustStop, 1)
+			tsw.mustStop.Store(true)
 			return err
 		}
 	}
@@ -260,7 +240,7 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 		return 0, nil
 	}
 
-	var mustStop uint32
+	var mustStop atomic.Bool
 	initTimeseriesWork := func(tsw *timeseriesWork, pts *packedTimeseries) {
 		tsw.rss = rss
 		tsw.pts = pts
@@ -270,22 +250,20 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	maxWorkers := MaxWorkers()
 	if maxWorkers == 1 || tswsLen == 1 {
 		// It is faster to process time series in the current goroutine.
-		tsw := getTimeseriesWork()
+		var tsw timeseriesWork
 		tmpResult := getTmpResult()
 		rowsProcessedTotal := 0
 		var err error
 		for i := range rss.packedTimeseries {
-			initTimeseriesWork(tsw, &rss.packedTimeseries[i])
+			initTimeseriesWork(&tsw, &rss.packedTimeseries[i])
 			err = tsw.do(&tmpResult.rs, 0)
 			rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 			rowsProcessedTotal += tsw.rowsProcessed
 			if err != nil {
 				break
 			}
-			tsw.reset()
 		}
 		putTmpResult(tmpResult)
-		putTimeseriesWork(tsw)
 
 		return rowsProcessedTotal, err
 	}
@@ -295,11 +273,9 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// which reduces the scalability on systems with many CPU cores.
 
 	// Prepare the work for workers.
-	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
+	tsws := make([]timeseriesWork, len(rss.packedTimeseries))
 	for i := range rss.packedTimeseries {
-		tsw := getTimeseriesWork()
-		initTimeseriesWork(tsw, &rss.packedTimeseries[i])
-		tsws[i] = tsw
+		initTimeseriesWork(&tsws[i], &rss.packedTimeseries[i])
 	}
 
 	// Prepare worker channels.
@@ -314,9 +290,9 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	}
 
 	// Spread work among workers.
-	for i, tsw := range tsws {
+	for i := range tsws {
 		idx := i % len(workChs)
-		workChs[idx] <- tsw
+		workChs[idx] <- &tsws[i]
 	}
 	// Mark worker channels as closed.
 	for _, workCh := range workChs {
@@ -339,14 +315,14 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// Collect results.
 	var firstErr error
 	rowsProcessedTotal := 0
-	for _, tsw := range tsws {
+	for i := range tsws {
+		tsw := &tsws[i]
 		if tsw.err != nil && firstErr == nil {
 			// Return just the first error, since other errors are likely duplicate the first error.
 			firstErr = tsw.err
 		}
 		rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 		rowsProcessedTotal += tsw.rowsProcessed
-		putTimeseriesWork(tsw)
 	}
 	return rowsProcessedTotal, firstErr
 }
@@ -1034,7 +1010,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 	var (
 		errGlobal     error
 		errGlobalLock sync.Mutex
-		mustStop      uint32
+		mustStop      atomic.Bool
 	)
 	var wg sync.WaitGroup
 	wg.Add(gomaxprocs)
@@ -1044,9 +1020,9 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 			for xw := range workCh {
 				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
 					errGlobalLock.Lock()
-					if errGlobal != nil {
+					if errGlobal == nil {
 						errGlobal = err
-						atomic.StoreUint32(&mustStop, 1)
+						mustStop.Store(true)
 					}
 					errGlobalLock.Unlock()
 				}
@@ -1064,7 +1040,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		if deadline.Exceeded() {
 			return fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
-		if atomic.LoadUint32(&mustStop) != 0 {
+		if mustStop.Load() {
 			break
 		}
 		xw := exportWorkPool.Get().(*exportWork)
@@ -1169,15 +1145,44 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
-		brsPrealloc [4]blockRef
-		brs         []blockRef
+		brs []blockRef
 	}
-	m := make(map[string]*blockRefs, maxSeriesCount)
-	orderedMetricNames := make([]string, 0, maxSeriesCount)
+
 	blocksRead := 0
 	samples := 0
 	tbf := getTmpBlocksFile()
 	var buf []byte
+	var metricNamePrev []byte
+
+	// metricNamesBuf is used for holding all the loaded unique metric names at m and orderedMetricNames.
+	// It should reduce pressure on Go GC by reducing the number of string allocations
+	// when constructing metricName string from byte slice.
+	metricNamesBufCap := maxSeriesCount * 100
+	if metricNamesBufCap > maxFastAllocBlockSize {
+		metricNamesBufCap = maxFastAllocBlockSize
+	}
+	metricNamesBuf := make([]byte, 0, metricNamesBufCap)
+
+	// brssPool is used for holding all the blockRefs objects across all the loaded time series.
+	// It should reduce pressure on Go GC by reducing the number of blockRefs allocations.
+	brssPool := make([]blockRefs, 0, maxSeriesCount)
+
+	// brsPool is used for holding the most of blockRefs.brs slices across all the loaded time series.
+	// It should reduce pressure on Go GC by reducing the number of allocations for blockRefs.brs slices.
+	brsPoolCap := uintptr(maxSeriesCount)
+	if brsPoolCap > maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) {
+		brsPoolCap = maxFastAllocBlockSize / unsafe.Sizeof(blockRef{})
+	}
+	brsPool := make([]blockRef, 0, brsPoolCap)
+
+	// m maps from metricName to the index of blockRefs inside brssPool
+	m := make(map[string]int, maxSeriesCount)
+
+	// orderedMetricNames contains the list of loaded unique metric names
+	// in the load order. This order is important for triggering sequential data reading.
+	orderedMetricNames := make([]string, 0, maxSeriesCount)
+
+	var brsIdx int
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
@@ -1190,8 +1195,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		if *maxSamplesPerQuery > 0 && samples > *maxSamplesPerQuery {
 			putTmpBlocksFile(tbf)
 			putStorageSearch(sr)
-			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: to increase the -search.maxSamplesPerQuery; to reduce time range for the query; to use more specific label filters in order to select lower number of series", *maxSamplesPerQuery)
+			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: increase the -search.maxSamplesPerQuery; "+
+				"reduce time range for the query; use more specific label filters in order to select fewer series", *maxSamplesPerQuery)
 		}
+
 		buf = br.Marshal(buf[:0])
 		addr, err := tbf.WriteBlockRefData(buf)
 		if err != nil {
@@ -1199,24 +1206,59 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-		// Do not intern mb.MetricName, since it leads to increased memory usage.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
+
 		metricName := sr.MetricBlockRef.MetricName
-		brs := m[string(metricName)]
-		if brs == nil {
-			brs = &blockRefs{}
-			brs.brs = brs.brsPrealloc[:0]
+		if metricNamePrev == nil || string(metricName) != string(metricNamePrev) {
+			idx, ok := m[string(metricName)]
+			if !ok {
+				if cap(brssPool) > len(brssPool) {
+					brssPool = brssPool[:len(brssPool)+1]
+				} else {
+					brssPool = append(brssPool, blockRefs{})
+				}
+				idx = len(brssPool) - 1
+			}
+			brsIdx = idx
+			metricNamePrev = append(metricNamePrev[:0], metricName...)
+		}
+
+		brs := &brssPool[brsIdx]
+		partRef := br.PartRef()
+		if uintptr(cap(brsPool)) >= maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) && len(brsPool) == cap(brsPool) {
+			// Allocate a new brsPool in order to avoid slow allocation of an object
+			// bigger than maxFastAllocBlockSize bytes at append() below.
+			brsPool = make([]blockRef, 0, maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}))
+		}
+		if canAppendToBlockRefPool(brsPool, brs.brs) {
+			// It is safe appending blockRef to brsPool, since there are no other items added there yet.
+			brsPool = append(brsPool, blockRef{
+				partRef: partRef,
+				addr:    addr,
+			})
+			brs.brs = brsPool[len(brsPool)-len(brs.brs)-1 : len(brsPool) : len(brsPool)]
+		} else {
+			// It is unsafe appending blockRef to brsPool, since there are other items added there.
+			// So just append it to brs.brs.
+			brs.brs = append(brs.brs, blockRef{
+				partRef: partRef,
+				addr:    addr,
+			})
 		}
-		brs.brs = append(brs.brs, blockRef{
-			partRef: br.PartRef(),
-			addr:    addr,
-		})
 		if len(brs.brs) == 1 {
-			metricNameStr := string(metricName)
+			if cap(metricNamesBuf) >= maxFastAllocBlockSize && len(metricNamesBuf)+len(metricName) > cap(metricNamesBuf) {
+				// Allocate a new metricNamesBuf in order to avoid slow allocation of byte slice
+				// bigger than maxFastAllocBlockSize bytes at append() below.
+				metricNamesBuf = make([]byte, 0, maxFastAllocBlockSize)
+			}
+			metricNamesBufLen := len(metricNamesBuf)
+			metricNamesBuf = append(metricNamesBuf, metricName...)
+			metricNameStr := bytesutil.ToUnsafeString(metricNamesBuf[metricNamesBufLen:])
+
 			orderedMetricNames = append(orderedMetricNames, metricNameStr)
-			m[metricNameStr] = brs
+			m[metricNameStr] = brsIdx
 		}
 	}
+
 	if err := sr.Error(); err != nil {
 		putTmpBlocksFile(tbf)
 		putStorageSearch(sr)
@@ -1239,7 +1281,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	for i, metricName := range orderedMetricNames {
 		pts[i] = packedTimeseries{
 			metricName: metricName,
-			brs:        m[metricName].brs,
+			brs:        brssPool[m[metricName]].brs,
 		}
 	}
 	rss.packedTimeseries = pts
@@ -1255,6 +1297,27 @@ type blockRef struct {
 	addr    tmpBlockAddr
 }
 
+// canAppendToBlockRefPool returns true if a points to the pool and the last item in a is the last item in the pool.
+//
+// In this case it is safe appending an item to the pool and then updating the a, so it refers to the extended slice.
+//
+// True is also returned if a is nil, since in this case it is safe appending an item to the pool and pointing a
+// to the last item in the pool.
+func canAppendToBlockRefPool(pool, a []blockRef) bool {
+	if a == nil {
+		return true
+	}
+	if len(a) > len(pool) {
+		// a doesn't belong to pool
+		return false
+	}
+	return getBlockRefsEnd(pool) == getBlockRefsEnd(a)
+}
+
+func getBlockRefsEnd(a []blockRef) uintptr {
+	return uintptr(unsafe.Pointer(unsafe.SliceData(a))) + uintptr(len(a))*unsafe.Sizeof(blockRef{})
+}
+
 func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]storage.TagFilter, maxMetrics int, deadline searchutils.Deadline) ([]*storage.TagFilters, error) {
 	tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
 	for _, tagFilters := range tagFilterss {
@@ -1300,3 +1363,8 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 	}
 	return dst, nil
 }
+
+// Go uses fast allocations for block sizes up to 32Kb.
+//
+// See https://github.com/golang/go/blob/704401ffa06c60e059c9e6e4048045b4ff42530a/src/runtime/malloc.go#L11
+const maxFastAllocBlockSize = 32 * 1024

app/vmselect/netstorage/tmp_blocks_file.go

@@ -136,21 +136,25 @@ func (tbf *tmpBlocksFile) Finalize() error {
 		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
 	}
 	tbf.buf = tbf.buf[:0]
-	r := fs.MustOpenReaderAt(fname)
+	r := fs.NewReaderAt(tbf.f)
+
 	// Hint the OS that the file is read almost sequentiallly.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
+
 	// Collect local stats in order to improve performance on systems with big number of CPU cores.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966
 	r.SetUseLocalStats()
+
 	tbf.r = r
+	tbf.f = nil
 	return nil
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
 	var buf []byte
-	if tbf.f == nil {
+	if tbf.r == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
 	} else {
 		bb := tmpBufPool.Get()
@@ -169,21 +173,45 @@ func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBl
 var tmpBufPool bytesutil.ByteBufferPool
 
 func (tbf *tmpBlocksFile) MustClose() {
-	if tbf.f == nil {
+	if tbf.f != nil {
+		// tbf.f could be non-nil if Finalize wasn't called.
+		// In this case tbf.r must be nil.
+		if tbf.r != nil {
+			logger.Panicf("BUG: tbf.r must be nil when tbf.f!=nil")
+		}
+
+		// Try removing the file before closing it in order to prevent from flushing the in-memory data
+		// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
+		// Gracefully handle this case by attempting to remove the file after closing it.
+		fname := tbf.f.Name()
+		errRemove := os.Remove(fname)
+		if err := tbf.f.Close(); err != nil {
+			logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+		}
+		if errRemove != nil {
+			if err := os.Remove(fname); err != nil {
+				logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+			}
+		}
+		tbf.f = nil
 		return
 	}
-	if tbf.r != nil {
-		// tbf.r could be nil if Finalize wasn't called.
-		tbf.r.MustClose()
-	}
-	fname := tbf.f.Name()
 
-	if err := tbf.f.Close(); err != nil {
-		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+	if tbf.r == nil {
+		// Nothing to do
+		return
 	}
-	// We cannot remove unclosed at non-posix filesystems, like windows
-	if err := os.Remove(fname); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+
+	// Try removing the file before closing it in order to prevent from flushing the in-memory data
+	// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
+	// Gracefully handle this case by attempting to remove the file after closing it.
+	fname := tbf.r.Path()
+	errRemove := os.Remove(fname)
+	tbf.r.MustClose()
+	if errRemove != nil {
+		if err := os.Remove(fname); err != nil {
+			logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+		}
 	}
-	tbf.f = nil
+	tbf.r = nil
 }

app/vmselect/prometheus/expand-with-exprs.qtpl

@@ -82,7 +82,7 @@ textarea { margin: 1em }
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -146,7 +146,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/expand-with-exprs.qtpl.go

@@ -195,7 +195,7 @@ func streamwithExprsTutorial(qw422016 *qt422016.Writer) {
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -259,7 +259,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/export.qtpl

@@ -24,6 +24,7 @@
 	{% endfor %}
 {% endfunc %}
 
+{%code const rfc3339Milli = "2006-01-02T15:04:05.999Z07:00" %}
 {% func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) %}
 	{% if fieldName == "__value__" %}
 		{%f= value %}
@@ -45,7 +46,7 @@
 		{% case "rfc3339" %}
 			{% code
 				bb := quicktemplate.AcquireByteBuffer()
-				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
+				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], rfc3339Milli)
 			%}
 			{%z= bb.B %}
 			{% code

app/vmselect/prometheus/export.qtpl.go

@@ -87,586 +87,589 @@ func ExportCSVLine(xb *exportBlock, fieldNames []string) string {
 }
 
 //line app/vmselect/prometheus/export.qtpl:27
-func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
+const rfc3339Milli = "2006-01-02T15:04:05.999Z07:00"
+
 //line app/vmselect/prometheus/export.qtpl:28
-	if fieldName == "__value__" {
+func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
 //line app/vmselect/prometheus/export.qtpl:29
-		qw422016.N().F(value)
+	if fieldName == "__value__" {
 //line app/vmselect/prometheus/export.qtpl:30
-		return
+		qw422016.N().F(value)
 //line app/vmselect/prometheus/export.qtpl:31
-	}
-//line app/vmselect/prometheus/export.qtpl:32
-	if fieldName == "__timestamp__" {
-//line app/vmselect/prometheus/export.qtpl:33
-		qw422016.N().DL(timestamp)
-//line app/vmselect/prometheus/export.qtpl:34
 		return
-//line app/vmselect/prometheus/export.qtpl:35
+//line app/vmselect/prometheus/export.qtpl:32
 	}
+//line app/vmselect/prometheus/export.qtpl:33
+	if fieldName == "__timestamp__" {
+//line app/vmselect/prometheus/export.qtpl:34
+		qw422016.N().DL(timestamp)
+//line app/vmselect/prometheus/export.qtpl:35
+		return
 //line app/vmselect/prometheus/export.qtpl:36
-	if strings.HasPrefix(fieldName, "__timestamp__:") {
+	}
 //line app/vmselect/prometheus/export.qtpl:37
+	if strings.HasPrefix(fieldName, "__timestamp__:") {
+//line app/vmselect/prometheus/export.qtpl:38
 		timeFormat := fieldName[len("__timestamp__:"):]
 
-//line app/vmselect/prometheus/export.qtpl:38
-		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:39
-		case "unix_s":
+		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:40
-			qw422016.N().DL(timestamp / 1000)
+		case "unix_s":
 //line app/vmselect/prometheus/export.qtpl:41
-		case "unix_ms":
+			qw422016.N().DL(timestamp / 1000)
 //line app/vmselect/prometheus/export.qtpl:42
-			qw422016.N().DL(timestamp)
+		case "unix_ms":
 //line app/vmselect/prometheus/export.qtpl:43
-		case "unix_ns":
+			qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:44
-			qw422016.N().DL(timestamp * 1e6)
+		case "unix_ns":
 //line app/vmselect/prometheus/export.qtpl:45
+			qw422016.N().DL(timestamp * 1e6)
+//line app/vmselect/prometheus/export.qtpl:46
 		case "rfc3339":
-//line app/vmselect/prometheus/export.qtpl:47
+//line app/vmselect/prometheus/export.qtpl:48
 			bb := quicktemplate.AcquireByteBuffer()
-			bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
+			bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], rfc3339Milli)
 
-//line app/vmselect/prometheus/export.qtpl:50
+//line app/vmselect/prometheus/export.qtpl:51
 			qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:52
+//line app/vmselect/prometheus/export.qtpl:53
 			quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:54
-		default:
 //line app/vmselect/prometheus/export.qtpl:55
+		default:
+//line app/vmselect/prometheus/export.qtpl:56
 			if strings.HasPrefix(timeFormat, "custom:") {
-//line app/vmselect/prometheus/export.qtpl:57
+//line app/vmselect/prometheus/export.qtpl:58
 				layout := timeFormat[len("custom:"):]
 				bb := quicktemplate.AcquireByteBuffer()
 				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
 
-//line app/vmselect/prometheus/export.qtpl:61
-				if bytes.ContainsAny(bb.B, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:62
-					qw422016.E().QZ(bb.B)
+				if bytes.ContainsAny(bb.B, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:63
-				} else {
+					qw422016.E().QZ(bb.B)
 //line app/vmselect/prometheus/export.qtpl:64
-					qw422016.N().Z(bb.B)
+				} else {
 //line app/vmselect/prometheus/export.qtpl:65
+					qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:66
 				}
-//line app/vmselect/prometheus/export.qtpl:67
+//line app/vmselect/prometheus/export.qtpl:68
 				quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:69
-			} else {
-//line app/vmselect/prometheus/export.qtpl:69
-				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:70
-				qw422016.N().S(timeFormat)
+			} else {
+//line app/vmselect/prometheus/export.qtpl:70
+				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:71
-			}
+				qw422016.N().S(timeFormat)
 //line app/vmselect/prometheus/export.qtpl:72
-		}
+			}
 //line app/vmselect/prometheus/export.qtpl:73
-		return
+		}
 //line app/vmselect/prometheus/export.qtpl:74
-	}
+		return
 //line app/vmselect/prometheus/export.qtpl:75
+	}
+//line app/vmselect/prometheus/export.qtpl:76
 	v := mn.GetTagValue(fieldName)
 
-//line app/vmselect/prometheus/export.qtpl:76
-	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:77
-		qw422016.N().QZ(v)
+	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:78
-	} else {
+		qw422016.N().QZ(v)
 //line app/vmselect/prometheus/export.qtpl:79
-		qw422016.N().Z(v)
+	} else {
 //line app/vmselect/prometheus/export.qtpl:80
-	}
+		qw422016.N().Z(v)
 //line app/vmselect/prometheus/export.qtpl:81
+	}
+//line app/vmselect/prometheus/export.qtpl:82
 }
 
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 func writeexportCSVField(qq422016 qtio422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	streamexportCSVField(qw422016, mn, fieldName, timestamp, value)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 }
 
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) string {
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	writeexportCSVField(qb422016, mn, fieldName, timestamp, value)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:81
+//line app/vmselect/prometheus/export.qtpl:82
 }
 
-//line app/vmselect/prometheus/export.qtpl:83
+//line app/vmselect/prometheus/export.qtpl:84
 func StreamExportPrometheusLine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:84
-	if len(xb.timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:84
-		return
-//line app/vmselect/prometheus/export.qtpl:84
-	}
 //line app/vmselect/prometheus/export.qtpl:85
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:85
+		return
+//line app/vmselect/prometheus/export.qtpl:85
+	}
+//line app/vmselect/prometheus/export.qtpl:86
 	bb := quicktemplate.AcquireByteBuffer()
 
-//line app/vmselect/prometheus/export.qtpl:86
+//line app/vmselect/prometheus/export.qtpl:87
 	writeprometheusMetricName(bb, xb.mn)
 
-//line app/vmselect/prometheus/export.qtpl:87
+//line app/vmselect/prometheus/export.qtpl:88
 	for i, ts := range xb.timestamps {
-//line app/vmselect/prometheus/export.qtpl:88
+//line app/vmselect/prometheus/export.qtpl:89
 		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:88
-		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:89
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/export.qtpl:90
 		qw422016.N().F(xb.values[i])
-//line app/vmselect/prometheus/export.qtpl:89
+//line app/vmselect/prometheus/export.qtpl:90
 		qw422016.N().S(` `)
-//line app/vmselect/prometheus/export.qtpl:90
+//line app/vmselect/prometheus/export.qtpl:91
 		qw422016.N().DL(ts)
-//line app/vmselect/prometheus/export.qtpl:90
+//line app/vmselect/prometheus/export.qtpl:91
 		qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:91
-	}
 //line app/vmselect/prometheus/export.qtpl:92
+	}
+//line app/vmselect/prometheus/export.qtpl:93
 	quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 }
 
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 func WriteExportPrometheusLine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	StreamExportPrometheusLine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 }
 
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 func ExportPrometheusLine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	WriteExportPrometheusLine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:93
+//line app/vmselect/prometheus/export.qtpl:94
 }
 
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:96
 func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:96
+//line app/vmselect/prometheus/export.qtpl:97
 	if len(xb.timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:96
+//line app/vmselect/prometheus/export.qtpl:97
 		return
-//line app/vmselect/prometheus/export.qtpl:96
+//line app/vmselect/prometheus/export.qtpl:97
 	}
-//line app/vmselect/prometheus/export.qtpl:96
+//line app/vmselect/prometheus/export.qtpl:97
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:98
+//line app/vmselect/prometheus/export.qtpl:99
 	streammetricNameObject(qw422016, xb.mn)
-//line app/vmselect/prometheus/export.qtpl:98
+//line app/vmselect/prometheus/export.qtpl:99
 	qw422016.N().S(`,"values":[`)
-//line app/vmselect/prometheus/export.qtpl:100
-	if len(xb.values) > 0 {
 //line app/vmselect/prometheus/export.qtpl:101
+	if len(xb.values) > 0 {
+//line app/vmselect/prometheus/export.qtpl:102
 		values := xb.values
 
-//line app/vmselect/prometheus/export.qtpl:102
-		streamconvertValueToSpecialJSON(qw422016, values[0])
 //line app/vmselect/prometheus/export.qtpl:103
+		streamconvertValueToSpecialJSON(qw422016, values[0])
+//line app/vmselect/prometheus/export.qtpl:104
 		values = values[1:]
 
-//line app/vmselect/prometheus/export.qtpl:104
-		for _, v := range values {
-//line app/vmselect/prometheus/export.qtpl:104
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:105
-			streamconvertValueToSpecialJSON(qw422016, v)
+		for _, v := range values {
+//line app/vmselect/prometheus/export.qtpl:105
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:106
+			streamconvertValueToSpecialJSON(qw422016, v)
+//line app/vmselect/prometheus/export.qtpl:107
 		}
-//line app/vmselect/prometheus/export.qtpl:107
+//line app/vmselect/prometheus/export.qtpl:108
 	}
-//line app/vmselect/prometheus/export.qtpl:107
+//line app/vmselect/prometheus/export.qtpl:108
 	qw422016.N().S(`],"timestamps":[`)
-//line app/vmselect/prometheus/export.qtpl:110
-	if len(xb.timestamps) > 0 {
 //line app/vmselect/prometheus/export.qtpl:111
+	if len(xb.timestamps) > 0 {
+//line app/vmselect/prometheus/export.qtpl:112
 		timestamps := xb.timestamps
 
-//line app/vmselect/prometheus/export.qtpl:112
-		qw422016.N().DL(timestamps[0])
 //line app/vmselect/prometheus/export.qtpl:113
+		qw422016.N().DL(timestamps[0])
+//line app/vmselect/prometheus/export.qtpl:114
 		timestamps = timestamps[1:]
 
-//line app/vmselect/prometheus/export.qtpl:114
-		for _, ts := range timestamps {
-//line app/vmselect/prometheus/export.qtpl:114
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:115
-			qw422016.N().DL(ts)
+		for _, ts := range timestamps {
+//line app/vmselect/prometheus/export.qtpl:115
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:116
+			qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:117
 		}
-//line app/vmselect/prometheus/export.qtpl:117
+//line app/vmselect/prometheus/export.qtpl:118
 	}
-//line app/vmselect/prometheus/export.qtpl:117
+//line app/vmselect/prometheus/export.qtpl:118
 	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 }
 
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 func WriteExportJSONLine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	StreamExportJSONLine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 }
 
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 func ExportJSONLine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	WriteExportJSONLine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:120
+//line app/vmselect/prometheus/export.qtpl:121
 }
 
-//line app/vmselect/prometheus/export.qtpl:122
+//line app/vmselect/prometheus/export.qtpl:123
 func StreamExportPromAPILine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:122
+//line app/vmselect/prometheus/export.qtpl:123
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:124
+//line app/vmselect/prometheus/export.qtpl:125
 	streammetricNameObject(qw422016, xb.mn)
-//line app/vmselect/prometheus/export.qtpl:124
+//line app/vmselect/prometheus/export.qtpl:125
 	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/export.qtpl:125
+//line app/vmselect/prometheus/export.qtpl:126
 	streamvaluesWithTimestamps(qw422016, xb.values, xb.timestamps)
-//line app/vmselect/prometheus/export.qtpl:125
+//line app/vmselect/prometheus/export.qtpl:126
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 }
 
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 func WriteExportPromAPILine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	StreamExportPromAPILine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 }
 
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 func ExportPromAPILine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	WriteExportPromAPILine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:127
+//line app/vmselect/prometheus/export.qtpl:128
 }
 
-//line app/vmselect/prometheus/export.qtpl:129
+//line app/vmselect/prometheus/export.qtpl:130
 func StreamExportPromAPIHeader(qw422016 *qt422016.Writer) {
-//line app/vmselect/prometheus/export.qtpl:129
+//line app/vmselect/prometheus/export.qtpl:130
 	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 }
 
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 func WriteExportPromAPIHeader(qq422016 qtio422016.Writer) {
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	StreamExportPromAPIHeader(qw422016)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 }
 
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 func ExportPromAPIHeader() string {
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	WriteExportPromAPIHeader(qb422016)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:136
 }
 
-//line app/vmselect/prometheus/export.qtpl:137
+//line app/vmselect/prometheus/export.qtpl:138
 func StreamExportPromAPIFooter(qw422016 *qt422016.Writer, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/export.qtpl:137
+//line app/vmselect/prometheus/export.qtpl:138
 	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/export.qtpl:141
+//line app/vmselect/prometheus/export.qtpl:142
 	qt.Donef("export format=promapi")
 
-//line app/vmselect/prometheus/export.qtpl:143
+//line app/vmselect/prometheus/export.qtpl:144
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/export.qtpl:143
+//line app/vmselect/prometheus/export.qtpl:144
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 }
 
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 func WriteExportPromAPIFooter(qq422016 qtio422016.Writer, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	StreamExportPromAPIFooter(qw422016, qt)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 }
 
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 func ExportPromAPIFooter(qt *querytracer.Tracer) string {
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	WriteExportPromAPIFooter(qb422016, qt)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:145
+//line app/vmselect/prometheus/export.qtpl:146
 }
 
-//line app/vmselect/prometheus/export.qtpl:147
-func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
 //line app/vmselect/prometheus/export.qtpl:148
+func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:149
 	qw422016.N().Z(mn.MetricGroup)
-//line app/vmselect/prometheus/export.qtpl:149
+//line app/vmselect/prometheus/export.qtpl:150
 	if len(mn.Tags) > 0 {
-//line app/vmselect/prometheus/export.qtpl:149
+//line app/vmselect/prometheus/export.qtpl:150
 		qw422016.N().S(`{`)
-//line app/vmselect/prometheus/export.qtpl:151
+//line app/vmselect/prometheus/export.qtpl:152
 		tags := mn.Tags
 
-//line app/vmselect/prometheus/export.qtpl:152
-		qw422016.N().Z(tags[0].Key)
-//line app/vmselect/prometheus/export.qtpl:152
-		qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:152
-		streamescapePrometheusLabel(qw422016, tags[0].Value)
 //line app/vmselect/prometheus/export.qtpl:153
+		qw422016.N().Z(tags[0].Key)
+//line app/vmselect/prometheus/export.qtpl:153
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:153
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/export.qtpl:154
 		tags = tags[1:]
 
-//line app/vmselect/prometheus/export.qtpl:154
-		for i := range tags {
 //line app/vmselect/prometheus/export.qtpl:155
+		for i := range tags {
+//line app/vmselect/prometheus/export.qtpl:156
 			tag := &tags[i]
 
-//line app/vmselect/prometheus/export.qtpl:155
+//line app/vmselect/prometheus/export.qtpl:156
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:156
+//line app/vmselect/prometheus/export.qtpl:157
 			qw422016.N().Z(tag.Key)
-//line app/vmselect/prometheus/export.qtpl:156
+//line app/vmselect/prometheus/export.qtpl:157
 			qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:156
+//line app/vmselect/prometheus/export.qtpl:157
 			streamescapePrometheusLabel(qw422016, tag.Value)
-//line app/vmselect/prometheus/export.qtpl:157
+//line app/vmselect/prometheus/export.qtpl:158
 		}
-//line app/vmselect/prometheus/export.qtpl:157
+//line app/vmselect/prometheus/export.qtpl:158
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:159
-	}
 //line app/vmselect/prometheus/export.qtpl:160
+	}
+//line app/vmselect/prometheus/export.qtpl:161
 }
 
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	streamprometheusMetricName(qw422016, mn)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 }
 
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 func prometheusMetricName(mn *storage.MetricName) string {
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	writeprometheusMetricName(qb422016, mn)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:161
 }
 
-//line app/vmselect/prometheus/export.qtpl:162
+//line app/vmselect/prometheus/export.qtpl:163
 func streamconvertValueToSpecialJSON(qw422016 *qt422016.Writer, v float64) {
-//line app/vmselect/prometheus/export.qtpl:163
+//line app/vmselect/prometheus/export.qtpl:164
 	if math.IsNaN(v) {
-//line app/vmselect/prometheus/export.qtpl:163
+//line app/vmselect/prometheus/export.qtpl:164
 		qw422016.N().S(`null`)
-//line app/vmselect/prometheus/export.qtpl:165
+//line app/vmselect/prometheus/export.qtpl:166
 	} else if math.IsInf(v, 0) {
-//line app/vmselect/prometheus/export.qtpl:166
+//line app/vmselect/prometheus/export.qtpl:167
 		if v > 0 {
-//line app/vmselect/prometheus/export.qtpl:166
+//line app/vmselect/prometheus/export.qtpl:167
 			qw422016.N().S(`"Infinity"`)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:169
 		} else {
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:169
 			qw422016.N().S(`"-Infinity"`)
-//line app/vmselect/prometheus/export.qtpl:170
-		}
 //line app/vmselect/prometheus/export.qtpl:171
-	} else {
+		}
 //line app/vmselect/prometheus/export.qtpl:172
-		qw422016.N().F(v)
+	} else {
 //line app/vmselect/prometheus/export.qtpl:173
+		qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:174
 	}
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 }
 
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 func writeconvertValueToSpecialJSON(qq422016 qtio422016.Writer, v float64) {
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	streamconvertValueToSpecialJSON(qw422016, v)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 }
 
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 func convertValueToSpecialJSON(v float64) string {
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	writeconvertValueToSpecialJSON(qb422016, v)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:174
+//line app/vmselect/prometheus/export.qtpl:175
 }
 
-//line app/vmselect/prometheus/export.qtpl:176
+//line app/vmselect/prometheus/export.qtpl:177
 func streamescapePrometheusLabel(qw422016 *qt422016.Writer, b []byte) {
-//line app/vmselect/prometheus/export.qtpl:176
+//line app/vmselect/prometheus/export.qtpl:177
 	qw422016.N().S(`"`)
-//line app/vmselect/prometheus/export.qtpl:178
-	for len(b) > 0 {
 //line app/vmselect/prometheus/export.qtpl:179
+	for len(b) > 0 {
+//line app/vmselect/prometheus/export.qtpl:180
 		n := bytes.IndexAny(b, "\\\n\"")
 
-//line app/vmselect/prometheus/export.qtpl:180
-		if n < 0 {
 //line app/vmselect/prometheus/export.qtpl:181
-			qw422016.N().Z(b)
+		if n < 0 {
 //line app/vmselect/prometheus/export.qtpl:182
-			break
+			qw422016.N().Z(b)
 //line app/vmselect/prometheus/export.qtpl:183
-		}
+			break
 //line app/vmselect/prometheus/export.qtpl:184
-		qw422016.N().Z(b[:n])
-//line app/vmselect/prometheus/export.qtpl:185
-		switch b[n] {
-//line app/vmselect/prometheus/export.qtpl:186
-		case '\\':
-//line app/vmselect/prometheus/export.qtpl:186
-			qw422016.N().S(`\\`)
-//line app/vmselect/prometheus/export.qtpl:188
-		case '\n':
-//line app/vmselect/prometheus/export.qtpl:188
-			qw422016.N().S(`\n`)
-//line app/vmselect/prometheus/export.qtpl:190
-		case '"':
-//line app/vmselect/prometheus/export.qtpl:190
-			qw422016.N().S(`\"`)
-//line app/vmselect/prometheus/export.qtpl:192
 		}
+//line app/vmselect/prometheus/export.qtpl:185
+		qw422016.N().Z(b[:n])
+//line app/vmselect/prometheus/export.qtpl:186
+		switch b[n] {
+//line app/vmselect/prometheus/export.qtpl:187
+		case '\\':
+//line app/vmselect/prometheus/export.qtpl:187
+			qw422016.N().S(`\\`)
+//line app/vmselect/prometheus/export.qtpl:189
+		case '\n':
+//line app/vmselect/prometheus/export.qtpl:189
+			qw422016.N().S(`\n`)
+//line app/vmselect/prometheus/export.qtpl:191
+		case '"':
+//line app/vmselect/prometheus/export.qtpl:191
+			qw422016.N().S(`\"`)
 //line app/vmselect/prometheus/export.qtpl:193
+		}
+//line app/vmselect/prometheus/export.qtpl:194
 		b = b[n+1:]
 
-//line app/vmselect/prometheus/export.qtpl:194
+//line app/vmselect/prometheus/export.qtpl:195
 	}
-//line app/vmselect/prometheus/export.qtpl:194
+//line app/vmselect/prometheus/export.qtpl:195
 	qw422016.N().S(`"`)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 }
 
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 func writeescapePrometheusLabel(qq422016 qtio422016.Writer, b []byte) {
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	streamescapePrometheusLabel(qw422016, b)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 }
 
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 func escapePrometheusLabel(b []byte) string {
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	writeescapePrometheusLabel(qb422016, b)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:196
+//line app/vmselect/prometheus/export.qtpl:197
 }

app/vmselect/prometheus/prometheus.go

@@ -47,15 +47,22 @@ var (
 	maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
 		"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
 
-	maxUniqueTimeseries    = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage")
-	maxFederateSeries      = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
-	maxExportSeries        = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
-	maxTSDBStatusSeries    = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
-	maxSeriesLimit         = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
+	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage")
+	maxFederateSeries   = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
+	maxExportSeries     = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
+	maxTSDBStatusSeries = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
+	maxSeriesLimit      = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
+	maxLabelsAPISeries  = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the the matching time series "+
+		"at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, "+
+		"-search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI")
 	maxPointsPerTimeseries = flag.Int("search.maxPointsPerTimeseries", 30e3, "The maximum points per a single timeseries returned from /api/v1/query_range. "+
 		"This option doesn't limit the number of scanned raw samples in the database. The main purpose of this option is to limit the number of per-series points "+
 		"returned to graphing UI such as VMUI or Grafana. There is no sense in setting this limit to values bigger than the horizontal resolution of the graph. "+
 		"See also -search.maxResponseSeries")
+	ignoreExtraFiltersAtLabelsAPI = flag.Bool("search.ignoreExtraFiltersAtLabelsAPI", false, "Whether to ignore match[], extra_filters[] and extra_label query args at "+
+		"/api/v1/labels and /api/v1/label/.../values . This may be useful for decreasing load on VictoriaMetrics when extra filters "+
+		"match too many time series. The downside is that suprflouos labels or series could be returned, which do not match the extra filters. "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
 )
 
 // Default step used if not set.
@@ -317,19 +324,21 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonPara
 		}
 	} else if format == "promapi" {
 		WriteExportPromAPIHeader(bw)
-		firstLineOnce := uint32(0)
-		firstLineSent := uint32(0)
+		var firstLineOnce atomic.Bool
+		var firstLineSent atomic.Bool
 		writeLineFunc = func(xb *exportBlock, workerID uint) error {
 			bb := sw.getBuffer(workerID)
-			if atomic.CompareAndSwapUint32(&firstLineOnce, 0, 1) {
+			// Use Load() in front of CompareAndSwap() in order to avoid slow inter-CPU synchronization
+			// in fast path after the first line has been already sent.
+			if !firstLineOnce.Load() && firstLineOnce.CompareAndSwap(false, true) {
 				// Send the first line to sw.bw
 				WriteExportPromAPILine(bb, xb)
 				_, err := sw.bw.Write(bb.B)
 				bb.Reset()
-				atomic.StoreUint32(&firstLineSent, 1)
+				firstLineSent.Store(true)
 				return err
 			}
-			for atomic.LoadUint32(&firstLineSent) == 0 {
+			for !firstLineSent.Load() {
 				// Busy wait until the first line is sent to sw.bw
 				runtime.Gosched()
 			}
@@ -489,7 +498,7 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
 	defer labelValuesDuration.UpdateDuration(startTime)
 
-	cp, err := getCommonParamsWithDefaultDuration(r, startTime, false)
+	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
 		return err
 	}
@@ -497,7 +506,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 	if err != nil {
 		return err
 	}
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 	labelValues, err := netstorage.LabelValues(qt, labelName, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
@@ -586,7 +595,7 @@ var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/
 func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer labelsDuration.UpdateDuration(startTime)
 
-	cp, err := getCommonParamsWithDefaultDuration(r, startTime, false)
+	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
 		return err
 	}
@@ -594,7 +603,7 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	if err != nil {
 		return err
 	}
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 	labels, err := netstorage.LabelNames(qt, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain labels: %w", err)
@@ -639,12 +648,12 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer seriesDuration.UpdateDuration(startTime)
 
-	// Do not set start to searchutils.minTimeMsecs by default as Prometheus does,
+	// Do not set start to httputils.minTimeMsecs by default as Prometheus does,
 	// since this leads to fetching and scanning all the data from the storage,
 	// which can take a lot of time for big storages.
 	// It is better setting start as end-defaultStep by default.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
-	cp, err := getCommonParamsWithDefaultDuration(r, startTime, true)
+	cp, err := getCommonParamsForLabelsAPI(r, startTime, true)
 	if err != nil {
 		return err
 	}
@@ -718,9 +727,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		start -= offset
 		end := start
 		start = end - window
-		if start < 0 {
-			start = 0
-		}
 		// Do not include data point with a timestamp matching the lower boundary of the window as Prometheus does.
 		start++
 		if end < start {
@@ -1124,14 +1130,15 @@ func getExportParams(r *http.Request, startTime time.Time) (*commonParams, error
 	return cp, nil
 }
 
-func getCommonParamsWithDefaultDuration(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) {
-	cp, err := getCommonParams(r, startTime, requireNonEmptyMatch)
+func getCommonParamsForLabelsAPI(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) {
+	cp, err := getCommonParamsInternal(r, startTime, requireNonEmptyMatch, true)
 	if err != nil {
 		return nil, err
 	}
 	if cp.start == 0 {
 		cp.start = cp.end - defaultStep
 	}
+	cp.deadline = searchutils.GetDeadlineForExport(r, startTime)
 	return cp, nil
 }
 
@@ -1144,6 +1151,10 @@ func getCommonParamsWithDefaultDuration(r *http.Request, startTime time.Time, re
 // - extra_label
 // - extra_filters[]
 func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) {
+	return getCommonParamsInternal(r, startTime, requireNonEmptyMatch, false)
+}
+
+func getCommonParamsInternal(r *http.Request, startTime time.Time, requireNonEmptyMatch, isLabelsAPI bool) (*commonParams, error) {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	start, err := httputils.GetTime(r, "start", 0)
 	if err != nil {
@@ -1170,15 +1181,20 @@ func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch
 	if requireNonEmptyMatch && len(matches) == 0 {
 		return nil, fmt.Errorf("missing `match[]` arg")
 	}
-	tagFilterss, err := getTagFilterssFromMatches(matches)
-	if err != nil {
-		return nil, err
+
+	var filterss [][]storage.TagFilter
+	if !isLabelsAPI || !*ignoreExtraFiltersAtLabelsAPI {
+		tagFilterss, err := getTagFilterssFromMatches(matches)
+		if err != nil {
+			return nil, err
+		}
+		etfs, err := searchutils.GetExtraTagFilters(r)
+		if err != nil {
+			return nil, err
+		}
+		filterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 	}
-	etfs, err := searchutils.GetExtraTagFilters(r)
-	if err != nil {
-		return nil, err
-	}
-	filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
+
 	cp := &commonParams{
 		deadline:         deadline,
 		start:            start,

app/vmselect/prometheus/query_range_response.qtpl

@@ -33,8 +33,8 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 			// seriesFetched is string instead of int because of historical reasons.
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
-		"seriesFetched": "{%dl qs.SeriesFetched %}",
-		"executionTimeMsec": {%dl qs.ExecutionTimeMsec %}
+		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
+		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_range_response.qtpl.go

@@ -68,11 +68,11 @@ func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result,
 //line app/vmselect/prometheus/query_range_response.qtpl:35
 	qw422016.N().S(`"seriesFetched": "`)
 //line app/vmselect/prometheus/query_range_response.qtpl:36
-	qw422016.N().DL(qs.SeriesFetched)
+	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_range_response.qtpl:36
 	qw422016.N().S(`","executionTimeMsec":`)
 //line app/vmselect/prometheus/query_range_response.qtpl:37
-	qw422016.N().DL(qs.ExecutionTimeMsec)
+	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
 //line app/vmselect/prometheus/query_range_response.qtpl:37
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_range_response.qtpl:40

app/vmselect/prometheus/query_response.qtpl

@@ -35,8 +35,8 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 			// seriesFetched is string instead of int because of historical reasons.
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
-		"seriesFetched": "{%dl qs.SeriesFetched %}",
-		"executionTimeMsec": {%dl qs.ExecutionTimeMsec %}
+		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
+		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/query_response.qtpl.go

@@ -78,11 +78,11 @@ func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *
 //line app/vmselect/prometheus/query_response.qtpl:37
 	qw422016.N().S(`"seriesFetched": "`)
 //line app/vmselect/prometheus/query_response.qtpl:38
-	qw422016.N().DL(qs.SeriesFetched)
+	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_response.qtpl:38
 	qw422016.N().S(`","executionTimeMsec":`)
 //line app/vmselect/prometheus/query_response.qtpl:39
-	qw422016.N().DL(qs.ExecutionTimeMsec)
+	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
 //line app/vmselect/prometheus/query_response.qtpl:39
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_response.qtpl:42

app/vmselect/promql/active_queries.go

@@ -60,7 +60,7 @@ func (aq *activeQueries) Add(ec *EvalConfig, q string) uint64 {
 	aqe.start = ec.Start
 	aqe.end = ec.End
 	aqe.step = ec.Step
-	aqe.qid = atomic.AddUint64(&nextActiveQueryID, 1)
+	aqe.qid = nextActiveQueryID.Add(1)
 	aqe.quotedRemoteAddr = ec.QuotedRemoteAddr
 	aqe.q = q
 	aqe.startTime = time.Now()
@@ -87,4 +87,8 @@ func (aq *activeQueries) GetAll() []activeQueryEntry {
 	return aqes
 }
 
-var nextActiveQueryID = uint64(time.Now().UnixNano())
+var nextActiveQueryID = func() *atomic.Uint64 {
+	var x atomic.Uint64
+	x.Store(uint64(time.Now().UnixNano()))
+	return &x
+}()

app/vmselect/promql/aggr.go

@@ -111,41 +111,48 @@ func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) [
 	modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
 	m := aggrPrepareSeries(argOrig, modifier, maxSeries, keepOriginal)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tss := range m {
-		rv := afe(tss, modifier)
+	for _, tssl := range m {
+		rv := afe(tssl.tss, modifier)
 		rvs = append(rvs, rv...)
 	}
 	return rvs, nil
 }
 
-func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string][]*timeseries {
+func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string]*tssList {
 	// Remove empty time series, e.g. series with all NaN samples,
 	// since such series are ignored by aggregate functions.
 	argOrig = removeEmptySeries(argOrig)
 	arg := copyTimeseriesMetricNames(argOrig, keepOriginal)
 
 	// Perform grouping.
-	m := make(map[string][]*timeseries)
+	m := make(map[string]*tssList)
 	bb := bbPool.Get()
 	for i, ts := range arg {
 		removeGroupTags(&ts.MetricName, modifier)
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		k := string(bb.B)
+		k := bb.B
 		if keepOriginal {
 			ts = argOrig[i]
 		}
-		tss := m[k]
-		if tss == nil && maxSeries > 0 && len(m) >= maxSeries {
-			// We already reached time series limit after grouping. Skip other time series.
-			continue
+		tssl := m[string(k)]
+		if tssl == nil {
+			if maxSeries > 0 && len(m) >= maxSeries {
+				// We already reached time series limit after grouping. Skip other time series.
+				continue
+			}
+			tssl = &tssList{}
+			m[string(k)] = tssl
 		}
-		tss = append(tss, ts)
-		m[k] = tss
+		tssl.tss = append(tssl.tss, ts)
 	}
 	bbPool.Put(bb)
 	return m
 }
 
+type tssList struct {
+	tss []*timeseries
+}
+
 func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	tss, err := getAggrTimeseries(afa.args)
 	if err != nil {
@@ -626,8 +633,8 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 
 	m := aggrPrepareSeries(args[1], &afa.ae.Modifier, afa.ae.Limit, false)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tss := range m {
-		rv, err := afe(tss, modifier)
+	for _, tssl := range m {
+		rv, err := afe(tssl.tss, modifier)
 		if err != nil {
 			return nil, err
 		}

app/vmselect/promql/aggr_incremental.go

@@ -111,8 +111,8 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 	removeGroupTags(&ts.MetricName, &iafc.ae.Modifier)
 	bb := bbPool.Get()
 	bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-	k := string(bb.B)
-	iac := m[k]
+	k := bb.B
+	iac := m[string(k)]
 	if iac == nil {
 		if iafc.ae.Limit > 0 && len(m) >= iafc.ae.Limit {
 			// Skip this time series, since the limit on the number of output time series has been already reached.
@@ -131,7 +131,7 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 			ts:     tsAggr,
 			values: make([]float64, len(ts.Values)),
 		}
-		m[k] = iac
+		m[string(k)] = iac
 	}
 	bbPool.Put(bb)
 	iafc.callbacks.updateAggrFunc(iac, ts.Values)

app/vmselect/promql/binary_op.go

@@ -257,7 +257,7 @@ func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft,
 			var tsCopy timeseries
 			tsCopy.CopyFromShallowTimestamps(tsLeft)
 			tsCopy.MetricName.SetTags(joinTags, joinPrefix, skipTags, &tsRight.MetricName)
-			bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
+			bb.B = marshalMetricNameSorted(bb.B[:0], &tsCopy.MetricName)
 			pair, ok := m[string(bb.B)]
 			if !ok {
 				m[string(bb.B)] = &tsPair{
@@ -522,7 +522,9 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 		mn := storage.GetMetricName()
 		for _, ts := range arg {
 			mn.CopyFrom(&ts.MetricName)
-			mn.ResetMetricGroup()
+			if !be.KeepMetricNames {
+				mn.ResetMetricGroup()
+			}
 			switch groupOp {
 			case "on":
 				mn.RemoveTagsOn(groupTags)
@@ -531,7 +533,7 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 			default:
 				logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
 			}
-			bb.B = marshalMetricTagsSorted(bb.B[:0], mn)
+			bb.B = marshalMetricNameSorted(bb.B[:0], mn)
 			k := string(bb.B)
 			m[k] = append(m[k], ts)
 		}

app/vmselect/promql/eval.go

@@ -29,7 +29,8 @@ import (
 )
 
 var (
-	disableCache                   = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling")
+	disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful when ingesting historical data. "+
+		"See https://docs.victoriametrics.com/#backfilling . See also -search.resetRollupResultCacheOnStartup")
 	maxPointsSubqueryPerTimeseries = flag.Int("search.maxPointsSubqueryPerTimeseries", 100e3, "The maximum number of points per series, which can be generated by subquery. "+
 		"See https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3")
 	maxMemoryPerQuery = flagutil.NewBytes("search.maxMemoryPerQuery", 0, "The maximum amounts of memory a single query may consume. "+
@@ -170,16 +171,17 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 // QueryStats contains various stats for the query.
 type QueryStats struct {
 	// SeriesFetched contains the number of series fetched from storage during the query evaluation.
-	SeriesFetched int64
+	SeriesFetched atomic.Int64
+
 	// ExecutionTimeMsec contains the number of milliseconds the query took to execute.
-	ExecutionTimeMsec int64
+	ExecutionTimeMsec atomic.Int64
 }
 
 func (qs *QueryStats) addSeriesFetched(n int) {
 	if qs == nil {
 		return
 	}
-	atomic.AddInt64(&qs.SeriesFetched, int64(n))
+	qs.SeriesFetched.Add(int64(n))
 }
 
 func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
@@ -187,7 +189,7 @@ func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
 		return
 	}
 	d := time.Since(startTime).Milliseconds()
-	atomic.AddInt64(&qs.ExecutionTimeMsec, d)
+	qs.ExecutionTimeMsec.Add(d)
 }
 
 func (ec *EvalConfig) validate() {
@@ -312,7 +314,7 @@ func evalExprInternal(qt *querytracer.Tracer, ec *EvalConfig, e metricsql.Expr)
 		}
 		rf, err := nrf(args)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("cannot evaluate args for %q: %w", fe.AppendString(nil), err)
 		}
 		rv, err := evalRollupFunc(qt, ec, fe.Name, rf, e, re, nil)
 		if err != nil {
@@ -393,7 +395,7 @@ func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFunc
 			}
 			rf, err := nrf(args)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("cannot evaluate args for aggregate func %q: %w", ae.AppendString(nil), err)
 			}
 			iafc := newIncrementalAggrFuncContext(ae, callbacks)
 			return evalRollupFunc(qt, ec, fe.Name, rf, ae, re, iafc)
@@ -948,7 +950,7 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 		return nil, err
 	}
 
-	var samplesScannedTotal uint64
+	var samplesScannedTotal atomic.Uint64
 	keepMetricNames := getKeepMetricNames(expr)
 	tsw := getTimeseriesByWorkerID()
 	seriesByWorkerID := tsw.byWorkerID
@@ -958,13 +960,13 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, values, timestamps)
-				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+				samplesScannedTotal.Add(samplesScanned)
 				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
-			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+			samplesScannedTotal.Add(samplesScanned)
 			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return values, timestamps
@@ -975,8 +977,8 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	}
 	putTimeseriesByWorkerID(tsw)
 
-	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
-	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal)
+	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
+	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal.Load())
 	return tss, nil
 }
 
@@ -1265,7 +1267,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains bigger values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1327,7 +1329,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains smaller values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1392,7 +1394,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd)
+		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
 		return tss, nil
 	default:
 		qt.Printf("instant rollup optimization isn't implemented for %s()", funcName)
@@ -1419,8 +1421,8 @@ func hasDuplicateSeries(tss []*timeseries) bool {
 	return false
 }
 
-func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	getMin := func(a, b float64) float64 {
@@ -1429,13 +1431,13 @@ func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMin)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMin)
 	qt.Printf("resulting series=%d; ok=%v", len(tss), ok)
 	return tss, ok
 }
 
-func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	getMax := func(a, b float64) float64 {
@@ -1444,12 +1446,12 @@ func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMax)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMax)
 	qt.Printf("resulting series=%d", len(tss))
 	return tss, ok
 }
 
-func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, f func(a, b float64) float64) ([]*timeseries, bool) {
+func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, timestamp int64, f func(a, b float64) float64) ([]*timeseries, bool) {
 	assertInstantValues(tssCached)
 	assertInstantValues(tssStart)
 	assertInstantValues(tssEnd)
@@ -1500,12 +1502,16 @@ func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, f func(a,
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
+
+	setInstantTimestamp(rvs, timestamp)
+
 	return rvs, true
 }
 
-// getSumInstantValues calculates tssCached + tssStart - tssEnd
-func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) []*timeseries {
-	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
+// getSumInstantValues aggregates tssCached, tssStart, tssEnd time series
+// into a new time series with value = tssCached + tssStart - tssEnd
+func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) []*timeseries {
+	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
 	defer qt.Done()
 
 	assertInstantValues(tssCached)
@@ -1550,10 +1556,19 @@ func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
+
+	setInstantTimestamp(rvs, timestamp)
+
 	qt.Printf("resulting series=%d", len(rvs))
 	return rvs
 }
 
+func setInstantTimestamp(tss []*timeseries, timestamp int64) {
+	for _, ts := range tss {
+		ts.Timestamps[0] = timestamp
+	}
+}
+
 func assertInstantValues(tss []*timeseries) {
 	for _, ts := range tss {
 		if len(ts.Values) != 1 {
@@ -1673,7 +1688,7 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	tfss := searchutils.ToTagFilterss(me.LabelFilterss)
 	tfss = searchutils.JoinTagFilterss(tfss, ec.EnforcedTagFilterss)
 	minTimestamp := ec.Start
-	if needSilenceIntervalForRollupFunc(funcName) {
+	if needSilenceIntervalForRollupFunc[funcName] {
 		minTimestamp -= maxSilenceInterval()
 	}
 	if window > ec.Step {
@@ -1681,9 +1696,6 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	} else {
 		minTimestamp -= ec.Step
 	}
-	if minTimestamp < 0 {
-		minTimestamp = 0
-	}
 	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss, ec.MaxSeries)
 	rss, err := netstorage.ProcessSearchQuery(qt, sq, ec.Deadline)
 	if err != nil {
@@ -1777,62 +1789,12 @@ func maxSilenceInterval() int64 {
 	return d
 }
 
-func needSilenceIntervalForRollupFunc(funcName string) bool {
-	// All the rollup functions, which do not rely on the previous sample
-	// before the lookbehind window (aka prevValue and realPrevValue), do not need silence interval.
-	switch strings.ToLower(funcName) {
-	case "default_rollup":
-		// The default_rollup implicitly relies on the previous samples in order to fill gaps.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388
-		return true
-	case
-		"absent_over_time",
-		"avg_over_time",
-		"count_eq_over_time",
-		"count_gt_over_time",
-		"count_le_over_time",
-		"count_ne_over_time",
-		"count_over_time",
-		"first_over_time",
-		"histogram_over_time",
-		"hoeffding_bound_lower",
-		"hoeffding_bound_upper",
-		"last_over_time",
-		"mad_over_time",
-		"max_over_time",
-		"median_over_time",
-		"min_over_time",
-		"predict_linear",
-		"present_over_time",
-		"quantile_over_time",
-		"quantiles_over_time",
-		"range_over_time",
-		"share_gt_over_time",
-		"share_le_over_time",
-		"share_eq_over_time",
-		"stale_samples_over_time",
-		"stddev_over_time",
-		"stdvar_over_time",
-		"sum_over_time",
-		"tfirst_over_time",
-		"timestamp",
-		"timestamp_with_name",
-		"tlast_over_time",
-		"tmax_over_time",
-		"tmin_over_time",
-		"zscore_over_time":
-		return false
-	default:
-		return true
-	}
-}
-
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
-	var samplesScannedTotal uint64
+	var samplesScannedTotal atomic.Uint64
 	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
@@ -1844,12 +1806,12 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 				for _, ts := range tsm.m {
 					iafc.updateTimeseries(ts, workerID)
 				}
-				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+				samplesScannedTotal.Add(samplesScanned)
 				continue
 			}
 			ts.Reset()
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
-			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+			samplesScannedTotal.Add(samplesScanned)
 			iafc.updateTimeseries(ts, workerID)
 
 			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
@@ -1862,8 +1824,8 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 		return nil, err
 	}
 	tss := iafc.finalizeTimeseries()
-	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
-	qt.Printf("series after aggregation with %s(): %d; samplesScanned=%d", iafc.ae.Name, len(tss), samplesScannedTotal)
+	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
+	qt.Printf("series after aggregation with %s(): %d; samplesScanned=%d", iafc.ae.Name, len(tss), samplesScannedTotal.Load())
 	return tss, nil
 }
 
@@ -1872,7 +1834,7 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
-	var samplesScannedTotal uint64
+	var samplesScannedTotal atomic.Uint64
 	tsw := getTimeseriesByWorkerID()
 	seriesByWorkerID := tsw.byWorkerID
 	seriesLen := rss.Len()
@@ -1882,13 +1844,13 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
-				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+				samplesScannedTotal.Add(samplesScanned)
 				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
-			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
+			samplesScannedTotal.Add(samplesScanned)
 			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return nil
@@ -1902,8 +1864,8 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	}
 	putTimeseriesByWorkerID(tsw)
 
-	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
-	qt.Printf("samplesScanned=%d", samplesScannedTotal)
+	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
+	qt.Printf("samplesScanned=%d", samplesScannedTotal.Load())
 	return tss, nil
 }
 

app/vmselect/promql/eval_test.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -85,13 +86,87 @@ func TestQueryStats_addSeriesFetched(t *testing.T) {
 	}
 	ec.QueryStats.addSeriesFetched(1)
 
-	if qs.SeriesFetched != 1 {
-		t.Fatalf("expected to get 1; got %d instead", qs.SeriesFetched)
+	if n := qs.SeriesFetched.Load(); n != 1 {
+		t.Fatalf("expected to get 1; got %d instead", n)
 	}
 
 	ecNew := copyEvalConfig(ec)
 	ecNew.QueryStats.addSeriesFetched(3)
-	if qs.SeriesFetched != 4 {
-		t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched)
+	if n := qs.SeriesFetched.Load(); n != 4 {
+		t.Fatalf("expected to get 4; got %d instead", n)
 	}
 }
+
+func TestGetSumInstantValues(t *testing.T) {
+	f := func(cached, start, end []*timeseries, timestamp int64, expectedResult []*timeseries) {
+		t.Helper()
+
+		result := getSumInstantValues(nil, cached, start, end, timestamp)
+		if !reflect.DeepEqual(result, expectedResult) {
+			t.Errorf("unexpected result; got\n%v\nwant\n%v", result, expectedResult)
+		}
+	}
+	ts := func(name string, timestamp int64, value float64) *timeseries {
+		return &timeseries{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte(name),
+			},
+			Timestamps: []int64{timestamp},
+			Values:     []float64{value},
+		}
+	}
+
+	// start - end + cached = 1
+	f(
+		nil,
+		[]*timeseries{ts("foo", 42, 1)},
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+
+	// start - end + cached = 0
+	f(
+		nil,
+		[]*timeseries{ts("foo", 100, 1)},
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 0)},
+	)
+
+	// start - end + cached = 2
+	f(
+		[]*timeseries{ts("foo", 10, 1)},
+		[]*timeseries{ts("foo", 100, 1)},
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 2)},
+	)
+
+	// start - end + cached = 1
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		[]*timeseries{ts("foo", 100, 1)},
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+
+	// start - end + cached = 0
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		nil,
+		[]*timeseries{ts("foo", 10, 1)},
+		100,
+		[]*timeseries{ts("foo", 100, 0)},
+	)
+
+	// start - end + cached = 1
+	f(
+		[]*timeseries{ts("foo", 50, 1)},
+		nil,
+		nil,
+		100,
+		[]*timeseries{ts("foo", 100, 1)},
+	)
+}

app/vmselect/promql/exec.go

@@ -352,22 +352,19 @@ type parseCacheValue struct {
 }
 
 type parseCache struct {
-	// Move atomic counters to the top of struct for 8-byte alignment on 32-bit arch.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
-
-	requests uint64
-	misses   uint64
+	requests atomic.Uint64
+	misses   atomic.Uint64
 
 	m  map[string]*parseCacheValue
 	mu sync.RWMutex
 }
 
 func (pc *parseCache) Requests() uint64 {
-	return atomic.LoadUint64(&pc.requests)
+	return pc.requests.Load()
 }
 
 func (pc *parseCache) Misses() uint64 {
-	return atomic.LoadUint64(&pc.misses)
+	return pc.misses.Load()
 }
 
 func (pc *parseCache) Len() uint64 {
@@ -378,14 +375,14 @@ func (pc *parseCache) Len() uint64 {
 }
 
 func (pc *parseCache) Get(q string) *parseCacheValue {
-	atomic.AddUint64(&pc.requests, 1)
+	pc.requests.Add(1)
 
 	pc.mu.RLock()
 	pcv := pc.m[q]
 	pc.mu.RUnlock()
 
 	if pcv == nil {
-		atomic.AddUint64(&pc.misses, 1)
+		pc.misses.Add(1)
 	}
 	return pcv
 }

app/vmselect/promql/exec_test.go

@@ -2307,6 +2307,21 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_join dst_label is equal to src_label`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_join(label_join(time(), "bar", "sep1", "a", "b"), "bar", "sep2", "a", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("bar"),
+			Value: []byte("sep2sep1"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`label_value()`, func(t *testing.T) {
 		t.Parallel()
 		q := `with (
@@ -3706,7 +3721,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `(
 		  (label_set(time(), "t1", "v1", "__name__", "q1") or label_set(10, "t2", "v2", "__name__", "q2"))
 		    +
-		  (label_set(100, "t1", "v1", "__name__", "q3") or label_set(time(), "t2", "v3"))
+		  (label_set(100, "t1", "v1", "__name__", "q1") or label_set(time(), "t2", "v3"))
 		) keep_metric_names`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -5822,6 +5837,60 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`count_values_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(
+			count_values_over_time("foo", round(label_set(rand(0), "x", "y"), 0.4)[200s:5s]),
+			"foo",
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 8, 7, 6, 10, 9},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("0"),
+			},
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{20, 13, 19, 18, 14, 13},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("0.4"),
+			},
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{16, 19, 14, 16, 16, 18},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("0.8"),
+			},
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]), "vmrange")`
@@ -6108,6 +6177,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`sum_gt_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_gt_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5.9, 5.2, 8.5, 5.1, 4.9, 4.5},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum_le_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_le_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4.2, 4.9, 3.2, 5.8, 4.1, 5.3},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum_eq_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum_eq_over_time(rand(0)[200s:10s], 0.7), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increases_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `increases_over_time(rand(0)[200s:10s])`
@@ -8050,10 +8152,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate()`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s])`
+		q := `rollup_rate((2200-time())[600s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -8062,7 +8164,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -8071,7 +8173,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{4, 3, 2, 1, 0, -1},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
@@ -8083,10 +8185,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "max")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "max")`
+		q := `rollup_rate((2200-time())[600s], "max")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -8094,10 +8196,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "avg")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "avg")`
+		q := `rollup_rate((2200-time())[600s], "avg")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}

app/vmselect/promql/rollup.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 	"sync"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -31,6 +33,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"count_le_over_time":      newRollupCountLE,
 	"count_ne_over_time":      newRollupCountNE,
 	"count_over_time":         newRollupFuncOneArg(rollupCount),
+	"count_values_over_time":  newRollupCountValues,
 	"decreases_over_time":     newRollupFuncOneArg(rollupDecreases),
 	"default_rollup":          newRollupFuncOneArg(rollupDefault), // default rollup func
 	"delta":                   newRollupFuncOneArg(rollupDelta),
@@ -79,12 +82,15 @@ var rollupFuncs = map[string]newRollupFunc{
 	"rollup_rate":             newRollupFuncOneOrTwoArgs(rollupFake), // + rollupFuncsRemoveCounterResets
 	"rollup_scrape_interval":  newRollupFuncOneOrTwoArgs(rollupFake),
 	"scrape_interval":         newRollupFuncOneArg(rollupScrapeInterval),
+	"share_eq_over_time":      newRollupShareEQ,
 	"share_gt_over_time":      newRollupShareGT,
 	"share_le_over_time":      newRollupShareLE,
-	"share_eq_over_time":      newRollupShareEQ,
 	"stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples),
 	"stddev_over_time":        newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":        newRollupFuncOneArg(rollupStdvar),
+	"sum_eq_over_time":        newRollupSumEQ,
+	"sum_gt_over_time":        newRollupSumGT,
+	"sum_le_over_time":        newRollupSumLE,
 	"sum_over_time":           newRollupFuncOneArg(rollupSum),
 	"sum2_over_time":          newRollupFuncOneArg(rollupSum2),
 	"tfirst_over_time":        newRollupFuncOneArg(rollupTfirst),
@@ -100,6 +106,42 @@ var rollupFuncs = map[string]newRollupFunc{
 	"zscore_over_time":       newRollupFuncOneArg(rollupZScoreOverTime),
 }
 
+// Functions, which need the previous sample before the lookbehind window for proper calculations.
+//
+// All the rollup functions, which do not rely on the previous sample
+// before the lookbehind window (aka prevValue and realPrevValue), do not need silence interval.
+var needSilenceIntervalForRollupFunc = map[string]bool{
+	"ascent_over_time":    true,
+	"changes":             true,
+	"decreases_over_time": true,
+	// The default_rollup implicitly relies on the previous samples in order to fill gaps.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388
+	"default_rollup":         true,
+	"delta":                  true,
+	"deriv_fast":             true,
+	"descent_over_time":      true,
+	"idelta":                 true,
+	"ideriv":                 true,
+	"increase":               true,
+	"increase_pure":          true,
+	"increases_over_time":    true,
+	"integrate":              true,
+	"irate":                  true,
+	"lag":                    true,
+	"lifetime":               true,
+	"rate":                   true,
+	"resets":                 true,
+	"rollup":                 true,
+	"rollup_candlestick":     true,
+	"rollup_delta":           true,
+	"rollup_deriv":           true,
+	"rollup_increase":        true,
+	"rollup_rate":            true,
+	"rollup_scrape_interval": true,
+	"scrape_interval":        true,
+	"tlast_change_over_time": true,
+}
+
 // rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
 var rollupAggrFuncs = map[string]rollupFunc{
 	"absent_over_time":        rollupAbsent,
@@ -570,7 +612,7 @@ type timeseriesMap struct {
 func newTimeseriesMap(funcName string, keepMetricNames bool, sharedTimestamps []int64, mnSrc *storage.MetricName) *timeseriesMap {
 	funcName = strings.ToLower(funcName)
 	switch funcName {
-	case "histogram_over_time", "quantiles_over_time":
+	case "histogram_over_time", "quantiles_over_time", "count_values_over_time":
 	default:
 		return nil
 	}
@@ -604,10 +646,16 @@ func (tsm *timeseriesMap) GetOrCreateTimeseries(labelName, labelValue string) *t
 	if ts != nil {
 		return ts
 	}
+
+	// Make a clone of labelValue in order to use it as map key, since it may point to unsafe string,
+	// which refers some other byte slice, which can change in the future.
+	labelValue = strings.Clone(labelValue)
+
 	ts = &timeseries{}
 	ts.CopyFromShallowTimestamps(tsm.origin)
 	ts.MetricName.RemoveTag(labelName)
 	ts.MetricName.AddTag(labelName, labelValue)
+
 	tsm.m[labelValue] = ts
 	return ts
 }
@@ -859,6 +907,11 @@ func removeCounterResets(values []float64) {
 		}
 		prevValue = v
 		values[i] = v + correction
+		// Check again, there could be precision error in float operations,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571
+		if i > 0 && values[i] < values[i-1] {
+			values[i] = values[i-1]
+		}
 	}
 }
 
@@ -949,7 +1002,7 @@ func newRollupHoltWinters(args []interface{}) (rollupFunc, error) {
 		// before calling rollup funcs.
 		values := rfa.values
 		if len(values) == 0 {
-			return rfa.prevValue
+			return nan
 		}
 		sf := sfs[rfa.idx]
 		if sf < 0 || sf > 1 {
@@ -1084,59 +1137,89 @@ func newRollupDurationOverTime(args []interface{}) (rollupFunc, error) {
 }
 
 func newRollupShareLE(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterLE)
+	return newRollupAvgFilter(args, countFilterLE)
 }
 
-func countFilterLE(values []float64, le float64) int {
+func countFilterLE(values []float64, le float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v <= le {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
 func newRollupShareGT(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterGT)
+	return newRollupAvgFilter(args, countFilterGT)
 }
 
-func countFilterGT(values []float64, gt float64) int {
+func countFilterGT(values []float64, gt float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v > gt {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
 func newRollupShareEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupShareFilter(args, countFilterEQ)
+	return newRollupAvgFilter(args, countFilterEQ)
 }
 
-func countFilterEQ(values []float64, eq float64) int {
+func sumFilterEQ(values []float64, eq float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v == eq {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func sumFilterLE(values []float64, le float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v <= le {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func sumFilterGT(values []float64, gt float64) float64 {
+	var sum float64
+	for _, v := range values {
+		if v > gt {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func countFilterEQ(values []float64, eq float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v == eq {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
-func countFilterNE(values []float64, ne float64) int {
+func countFilterNE(values []float64, ne float64) float64 {
 	n := 0
 	for _, v := range values {
 		if v != ne {
 			n++
 		}
 	}
-	return n
+	return float64(n)
 }
 
-func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
-	rf, err := newRollupCountFilter(args, countFilter)
+func newRollupAvgFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
+	rf, err := newRollupFilter(args, f)
 	if err != nil {
 		return nil, err
 	}
@@ -1146,23 +1229,35 @@ func newRollupShareFilter(args []interface{}, countFilter func(values []float64,
 	}, nil
 }
 
+func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, countFilterEQ)
+}
+
 func newRollupCountLE(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterLE)
+	return newRollupFilter(args, countFilterLE)
 }
 
 func newRollupCountGT(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterGT)
-}
-
-func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterEQ)
+	return newRollupFilter(args, countFilterGT)
 }
 
 func newRollupCountNE(args []interface{}) (rollupFunc, error) {
-	return newRollupCountFilter(args, countFilterNE)
+	return newRollupFilter(args, countFilterNE)
 }
 
-func newRollupCountFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
+func newRollupSumEQ(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterEQ)
+}
+
+func newRollupSumLE(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterLE)
+}
+
+func newRollupSumGT(args []interface{}) (rollupFunc, error) {
+	return newRollupFilter(args, sumFilterGT)
+}
+
+func newRollupFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 2); err != nil {
 		return nil, err
 	}
@@ -1178,7 +1273,7 @@ func newRollupCountFilter(args []interface{}, countFilter func(values []float64,
 			return nan
 		}
 		limit := limits[rfa.idx]
-		return float64(countFilter(values, limit))
+		return f(values, limit)
 	}
 	return rf, nil
 }
@@ -1350,6 +1445,42 @@ func mad(values []float64) float64 {
 	return v
 }
 
+func newRollupCountValues(args []interface{}) (rollupFunc, error) {
+	if err := expectRollupArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	tssLabelNum, ok := args[0].([]*timeseries)
+	if !ok {
+		return nil, fmt.Errorf(`unexpected type for labelName arg; got %T; want %T`, args[0], tssLabelNum)
+	}
+	labelName, err := getString(tssLabelNum, 0)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get labelName: %w", err)
+	}
+	f := func(rfa *rollupFuncArg) float64 {
+		tsm := rfa.tsm
+		idx := rfa.idx
+		bb := bbPool.Get()
+		// Note: the code below may create very big number of time series
+		// if the number of unique values in rfa.values is big.
+		for _, v := range rfa.values {
+			bb.B = strconv.AppendFloat(bb.B[:0], v, 'g', -1, 64)
+			labelValue := bytesutil.ToUnsafeString(bb.B)
+			ts := tsm.GetOrCreateTimeseries(labelName, labelValue)
+			count := ts.Values[idx]
+			if math.IsNaN(count) {
+				count = 1
+			} else {
+				count++
+			}
+			ts.Values[idx] = count
+		}
+		bbPool.Put(bb)
+		return nan
+	}
+	return f, nil
+}
+
 func rollupHistogram(rfa *rollupFuncArg) float64 {
 	values := rfa.values
 	tsm := rfa.tsm
@@ -1536,11 +1667,7 @@ func rollupRateOverSum(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	timestamps := rfa.timestamps
 	if len(timestamps) == 0 {
-		if math.IsNaN(rfa.prevValue) {
-			return nan
-		}
-		// Assume that the value didn't change since rfa.prevValue.
-		return 0
+		return nan
 	}
 	sum := float64(0)
 	for _, v := range rfa.values {
@@ -1560,7 +1687,7 @@ func rollupSum2(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		return rfa.prevValue * rfa.prevValue
+		return nan
 	}
 	var sum2 float64
 	for _, v := range values {
@@ -1574,7 +1701,7 @@ func rollupGeomean(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		return rfa.prevValue
+		return nan
 	}
 	p := 1.0
 	for _, v := range values {
@@ -1911,6 +2038,10 @@ func rollupChangesPrometheus(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values[1:] {
 		if v != prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 			prevValue = v
 		}
@@ -1934,6 +2065,10 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 	}
 	for _, v := range values {
 		if v != prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 			prevValue = v
 		}
@@ -1962,6 +2097,10 @@ func rollupIncreases(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v > prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 		}
 		prevValue = v
@@ -1993,6 +2132,10 @@ func rollupResets(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v < prevValue {
+			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
+				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
+				continue
+			}
 			n++
 		}
 		prevValue = v
@@ -2202,10 +2345,7 @@ func rollupDistinct(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		if math.IsNaN(rfa.prevValue) {
-			return nan
-		}
-		return 0
+		return nan
 	}
 	m := make(map[float64]struct{})
 	for _, v := range values {

app/vmselect/promql/rollup_result_cache.go

@@ -30,6 +30,8 @@ var (
 		"due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset")
 	disableAutoCacheReset = flag.Bool("search.disableAutoCacheReset", false, "Whether to disable automatic response cache reset if a sample with timestamp "+
 		"outside -search.cacheTimestampOffset is inserted into VictoriaMetrics")
+	resetRollupResultCacheOnStartup = flag.Bool("search.resetRollupResultCacheOnStartup", false, "Whether to reset rollup result cache on startup. "+
+		"See https://docs.victoriametrics.com/#rollup-result-cache . See also -search.disableCache")
 )
 
 // ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
@@ -43,6 +45,11 @@ func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
 		rollupResultResetMetricRowSample.Store(&storage.MetricRow{})
 		go checkRollupResultCacheReset()
 	})
+	if needRollupResultCacheReset.Load() {
+		// The cache has been already instructed to reset.
+		return
+	}
+
 	minTimestamp := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds() + checkRollupResultCacheResetInterval.Milliseconds()
 	needCacheReset := false
 	for i := range mrs {
@@ -56,14 +63,14 @@ func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
 	}
 	if needCacheReset {
 		// Do not call ResetRollupResultCache() here, since it may be heavy when frequently called.
-		atomic.StoreUint32(&needRollupResultCacheReset, 1)
+		needRollupResultCacheReset.Store(true)
 	}
 }
 
 func checkRollupResultCacheReset() {
 	for {
 		time.Sleep(checkRollupResultCacheResetInterval)
-		if atomic.SwapUint32(&needRollupResultCacheReset, 0) > 0 {
+		if needRollupResultCacheReset.Swap(false) {
 			mr := rollupResultResetMetricRowSample.Load()
 			d := int64(fasttime.UnixTimestamp()*1000) - mr.Timestamp - cacheTimestampOffset.Milliseconds()
 			logger.Warnf("resetting rollup result cache because the metric %s has a timestamp older than -search.cacheTimestampOffset=%s by %.3fs",
@@ -75,7 +82,7 @@ func checkRollupResultCacheReset() {
 
 const checkRollupResultCacheResetInterval = 5 * time.Second
 
-var needRollupResultCacheReset uint32
+var needRollupResultCacheReset atomic.Bool
 var checkRollupResultCacheResetOnce sync.Once
 var rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
 
@@ -112,12 +119,17 @@ func InitRollupResultCache(cachePath string) {
 	cacheSize := getRollupResultCacheSize()
 	var c *workingsetcache.Cache
 	if len(rollupResultCachePath) > 0 {
-		logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
+		if *resetRollupResultCacheOnStartup {
+			logger.Infof("removing rollupResult cache at %q becasue -search.resetRollupResultCacheOnStartup command-line flag is set", rollupResultCachePath)
+			fs.MustRemoveAll(rollupResultCachePath)
+		} else {
+			logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
+		}
 		c = workingsetcache.Load(rollupResultCachePath, cacheSize)
 		mustLoadRollupResultCacheKeyPrefix(rollupResultCachePath)
 	} else {
 		c = workingsetcache.New(cacheSize)
-		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 	}
 	if *disableCache {
 		c.Reset()
@@ -199,7 +211,7 @@ var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="pr
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
 	rollupResultCacheResets.Inc()
-	atomic.AddUint64(&rollupResultCacheKeyPrefix, 1)
+	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }
 
@@ -426,8 +438,8 @@ func (rrc *rollupResultCache) PutSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	}
 
 	var key rollupResultCacheKey
-	key.prefix = rollupResultCacheKeyPrefix
-	key.suffix = atomic.AddUint64(&rollupResultCacheKeySuffix, 1)
+	key.prefix = rollupResultCacheKeyPrefix.Load()
+	key.suffix = rollupResultCacheKeySuffix.Add(1)
 
 	bb := bbPool.Get()
 	bb.B = key.Marshal(bb.B[:0])
@@ -443,8 +455,12 @@ func (rrc *rollupResultCache) PutSeries(qt *querytracer.Tracer, ec *EvalConfig,
 }
 
 var (
-	rollupResultCacheKeyPrefix uint64
-	rollupResultCacheKeySuffix = uint64(time.Now().UnixNano())
+	rollupResultCacheKeyPrefix atomic.Uint64
+	rollupResultCacheKeySuffix = func() *atomic.Uint64 {
+		var x atomic.Uint64
+		x.Store(uint64(time.Now().UnixNano()))
+		return &x
+	}()
 )
 
 func (rrc *rollupResultCache) getSeriesFromCache(qt *querytracer.Tracer, key []byte) ([]*timeseries, bool) {
@@ -505,26 +521,26 @@ func newRollupResultCacheKeyPrefix() uint64 {
 func mustLoadRollupResultCacheKeyPrefix(path string) {
 	path = path + ".key.prefix"
 	if !fs.IsPathExist(path) {
-		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 		return
 	}
 	data, err := os.ReadFile(path)
 	if err != nil {
 		logger.Errorf("cannot load %s: %s; reset rollupResult cache", path, err)
-		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 		return
 	}
 	if len(data) != 8 {
 		logger.Errorf("unexpected size of %s; want 8 bytes; got %d bytes; reset rollupResult cache", path, len(data))
-		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 		return
 	}
-	rollupResultCacheKeyPrefix = encoding.UnmarshalUint64(data)
+	rollupResultCacheKeyPrefix.Store(encoding.UnmarshalUint64(data))
 }
 
 func mustSaveRollupResultCacheKeyPrefix(path string) {
 	path = path + ".key.prefix"
-	data := encoding.MarshalUint64(nil, rollupResultCacheKeyPrefix)
+	data := encoding.MarshalUint64(nil, rollupResultCacheKeyPrefix.Load())
 	fs.MustWriteAtomic(path, data, true)
 }
 
@@ -540,7 +556,7 @@ const (
 
 func marshalRollupResultCacheKeyForSeries(dst []byte, expr metricsql.Expr, window, step int64, etfs [][]storage.TagFilter) []byte {
 	dst = append(dst, rollupResultCacheVersion)
-	dst = encoding.MarshalUint64(dst, rollupResultCacheKeyPrefix)
+	dst = encoding.MarshalUint64(dst, rollupResultCacheKeyPrefix.Load())
 	dst = append(dst, rollupResultCacheTypeSeries)
 	dst = encoding.MarshalInt64(dst, window)
 	dst = encoding.MarshalInt64(dst, step)
@@ -551,7 +567,7 @@ func marshalRollupResultCacheKeyForSeries(dst []byte, expr metricsql.Expr, windo
 
 func marshalRollupResultCacheKeyForInstantValues(dst []byte, expr metricsql.Expr, window, step int64, etfs [][]storage.TagFilter) []byte {
 	dst = append(dst, rollupResultCacheVersion)
-	dst = encoding.MarshalUint64(dst, rollupResultCacheKeyPrefix)
+	dst = encoding.MarshalUint64(dst, rollupResultCacheKeyPrefix.Load())
 	dst = append(dst, rollupResultCacheTypeInstantValues)
 	dst = encoding.MarshalInt64(dst, window)
 	dst = encoding.MarshalInt64(dst, step)