Merge branch 'master' into streaming-aggregation-ui

# Conflicts:
#	app/vmagent/main.go

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/workflows/check-licenses.yml

@@ -25,7 +25,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
         if: ${{ matrix.language == 'go' }}
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -75,7 +75,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +86,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +100,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -102,7 +102,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -115,6 +115,6 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt

.gitignore

@@ -22,4 +22,3 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata

Makefile

@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
+MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
@@ -178,8 +178,7 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64 \
-	victoria-metrics-windows-amd64
+	victoria-metrics-openbsd-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
@@ -466,7 +465,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.22
+	go mod tidy -compat=1.20
 	go mod vendor
 
 app-local:

SECURITY.md

@@ -2,17 +2,13 @@
 
 ## Supported Versions
 
-The following versions of VictoriaMetrics receive regular security fixes:
-
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/LTS-releases.html) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/LTS-releases.html) | :white_check_mark: |
+| v1.93.x LTS release | :white_check_mark: |
+| v1.87.x LTS release | :white_check_mark: |
 | other releases  | :x:                |
 
-See [this page](https://victoriametrics.com/security/) for more details.
-
 ## Reporting a Vulnerability
 
 Please report any security issues to security@victoriametrics.com

app/victoria-logs/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -21,10 +22,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -32,21 +34,18 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
+	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -54,9 +53,9 @@ func main() {
 	logger.Infof("received signal %s", sig)
 	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-metrics/main.go

@@ -26,8 +26,8 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
@@ -66,11 +66,7 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8428"}
-	}
-	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
@@ -80,7 +76,7 @@ func main() {
 
 	startSelfScraper()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -90,9 +86,9 @@ func main() {
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -123,12 +119,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"expand-with-exprs", "WITH expressions' tutorial"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
-			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -181,7 +180,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, false, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -227,7 +226,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -414,7 +413,6 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
-
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -499,73 +497,3 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
-
-func TestImportJSONLines(t *testing.T) {
-	f := func(labelsCount, labelLen int) {
-		t.Helper()
-
-		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
-		line := generateJSONLine(labelsCount, labelLen)
-		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
-		}
-		if resp.StatusCode != 204 {
-			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
-		}
-	}
-
-	// labels with various lengths
-	for i := 0; i < 500; i++ {
-		f(10, i*5)
-	}
-
-	// Too many labels
-	f(1000, 100)
-
-	// Too long labels
-	f(1, 100_000)
-	f(10, 100_000)
-	f(10, 10_000)
-}
-
-func generateJSONLine(labelsCount, labelLen int) string {
-	m := make(map[string]string, labelsCount)
-	m["__name__"] = generateSizedRandomString(labelLen)
-	for j := 1; j < labelsCount; j++ {
-		labelName := generateSizedRandomString(labelLen)
-		labelValue := generateSizedRandomString(labelLen)
-		m[labelName] = labelValue
-	}
-
-	type jsonLine struct {
-		Metric     map[string]string `json:"metric"`
-		Values     []float64         `json:"values"`
-		Timestamps []int64           `json:"timestamps"`
-	}
-	line := &jsonLine{
-		Metric:     m,
-		Values:     []float64{1.34},
-		Timestamps: []int64{time.Now().UnixNano() / 1e6},
-	}
-	data, err := json.Marshal(&line)
-	if err != nil {
-		panic(fmt.Errorf("cannot marshal JSON: %w", err))
-	}
-	data = append(data, '\n')
-	return string(data)
-}
-
-const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
-
-func generateSizedRandomString(size int) string {
-	dst := make([]byte, size)
-	for i := range dst {
-		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
-	}
-	return string(dst)
-}

app/victoria-metrics/self_scraper.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -50,8 +49,16 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	f := func(currentTime time.Time, sendStaleMarkers bool) {
-		currentTimestamp := currentTime.UnixNano() / 1e6
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -76,27 +83,12 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			if sendStaleMarkers {
-				mr.Value = decimal.StaleNaN
-			} else {
-				mr.Value = r.Value
-			}
+			mr.Value = r.Value
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
-	for {
-		select {
-		case <-selfScraperStopCh:
-			f(time.Now(), true)
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			f(currentTime, false)
-		}
-	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {

app/vlselect/logsql/logsql.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
@@ -18,18 +17,13 @@ var (
 )
 
 // ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}, cancel func()) {
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
 
 	qStr := r.FormValue("query")
 	q, err := logstorage.ParseQuery(qStr)
@@ -40,7 +34,7 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
 
 	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN(), limit)
+	sw.Init(w, maxSortBufferSize.IntN())
 	tenantIDs := []logstorage.TenantID{tenantID}
 	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
 		if len(columns) == 0 {
@@ -52,11 +46,7 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
 			WriteJSONRow(bb, columns, rowIdx)
 		}
-
-		if !sw.TryWrite(bb.B) {
-			cancel()
-		}
-
+		sw.MustWrite(bb.B)
 		blockResultPool.Put(bb)
 	})
 	sw.FinalFlush()

app/vlselect/logsql/sort_writer.go

@@ -36,12 +36,8 @@ var sortWriterPool sync.Pool
 // If the buf isn't empty at FinalFlush() call, then the buffered data
 // is sorted by _time field.
 type sortWriter struct {
-	mu sync.Mutex
-	w  io.Writer
-
-	maxLines     int
-	linesWritten int
-
+	mu         sync.Mutex
+	w          io.Writer
 	maxBufLen  int
 	buf        []byte
 	bufFlushed bool
@@ -51,121 +47,58 @@ type sortWriter struct {
 
 func (sw *sortWriter) reset() {
 	sw.w = nil
-
-	sw.maxLines = 0
-	sw.linesWritten = 0
-
 	sw.maxBufLen = 0
 	sw.buf = sw.buf[:0]
 	sw.bufFlushed = false
 	sw.hasErr = false
 }
 
-// Init initializes sw.
-//
-// If maxLines is set to positive value, then sw accepts up to maxLines
-// and then rejects all the other lines by returning false from TryWrite.
-func (sw *sortWriter) Init(w io.Writer, maxBufLen, maxLines int) {
+func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
 	sw.reset()
 
 	sw.w = w
 	sw.maxBufLen = maxBufLen
-	sw.maxLines = maxLines
 }
 
-// TryWrite writes p to sw.
-//
-// True is returned on successful write, false otherwise.
-//
-// Unsuccessful write may occur on underlying write error or when maxLines lines are already written to sw.
-func (sw *sortWriter) TryWrite(p []byte) bool {
+func (sw *sortWriter) MustWrite(p []byte) {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
 
 	if sw.hasErr {
-		return false
+		return
 	}
 
 	if sw.bufFlushed {
-		if !sw.writeToUnderlyingWriterLocked(p) {
+		if _, err := sw.w.Write(p); err != nil {
 			sw.hasErr = true
-			return false
 		}
-		return true
+		return
 	}
-
 	if len(sw.buf)+len(p) < sw.maxBufLen {
 		sw.buf = append(sw.buf, p...)
-		return true
+		return
 	}
-
 	sw.bufFlushed = true
-	if !sw.writeToUnderlyingWriterLocked(sw.buf) {
-		sw.hasErr = true
-		return false
-	}
-	sw.buf = sw.buf[:0]
-
-	if !sw.writeToUnderlyingWriterLocked(p) {
-		sw.hasErr = true
-		return false
-	}
-	return true
-}
-
-func (sw *sortWriter) writeToUnderlyingWriterLocked(p []byte) bool {
-	if len(p) == 0 {
-		return true
-	}
-	if sw.maxLines > 0 {
-		if sw.linesWritten >= sw.maxLines {
-			return false
+	if len(sw.buf) > 0 {
+		if _, err := sw.w.Write(sw.buf); err != nil {
+			sw.hasErr = true
+			return
 		}
-		var linesLeft int
-		p, linesLeft = trimLines(p, sw.maxLines-sw.linesWritten)
-		println("DEBUG: end trimLines", string(p), linesLeft)
-		sw.linesWritten += linesLeft
+		sw.buf = sw.buf[:0]
 	}
 	if _, err := sw.w.Write(p); err != nil {
-		return false
+		sw.hasErr = true
 	}
-	return true
 }
 
-func trimLines(p []byte, maxLines int) ([]byte, int) {
-	println("DEBUG: start trimLines", string(p), maxLines)
-	if maxLines <= 0 {
-		return nil, 0
-	}
-	n := bytes.Count(p, newline)
-	if n < maxLines {
-		return p, n
-	}
-	for n >= maxLines {
-		idx := bytes.LastIndexByte(p, '\n')
-		p = p[:idx]
-		n--
-	}
-	return p[:len(p)+1], maxLines
-}
-
-var newline = []byte("\n")
-
 func (sw *sortWriter) FinalFlush() {
 	if sw.hasErr || sw.bufFlushed {
 		return
 	}
-
 	rs := getRowsSorter()
 	rs.parseRows(sw.buf)
 	rs.sort()
-
-	rows := rs.rows
-	if sw.maxLines > 0 && len(rows) > sw.maxLines {
-		rows = rows[:sw.maxLines]
-	}
-	WriteJSONRows(sw.w, rows)
-
+	WriteJSONRows(sw.w, rs.rows)
 	putRowsSorter(rs)
 }
 

app/vlselect/logsql/sort_writer_test.go

@@ -7,16 +7,15 @@ import (
 )
 
 func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen, maxLines int, data string, expectedResult string) {
+	f := func(maxBufLen int, data string, expectedResult string) {
 		t.Helper()
 
 		var bb bytes.Buffer
 		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen, maxLines)
+		sw.Init(&bb, maxBufLen)
+
 		for _, s := range strings.Split(data, "\n") {
-			if !sw.TryWrite([]byte(s + "\n")) {
-				break
-			}
+			sw.MustWrite([]byte(s + "\n"))
 		}
 		sw.FinalFlush()
 		putSortWriter(sw)
@@ -27,20 +26,14 @@ func TestSortWriter(t *testing.T) {
 		}
 	}
 
-	f(100, 0, "", "")
-	f(100, 0, "{}", "{}\n")
+	f(100, "", "")
+	f(100, "{}", "{}\n")
 
 	data := `{"_time":"def","_msg":"xxx"}
 {"_time":"abc","_msg":"foo"}`
 	resultExpected := `{"_time":"abc","_msg":"foo"}
 {"_time":"def","_msg":"xxx"}
 `
-	f(100, 0, data, resultExpected)
-	f(10, 0, data, data+"\n")
-
-	// Test with the maxLines
-	f(100, 1, data, `{"_time":"abc","_msg":"foo"}`+"\n")
-	f(10, 1, data, `{"_time":"def","_msg":"xxx"}`+"\n")
-	f(10, 2, data, data+"\n")
-	f(100, 2, data, resultExpected)
+	f(100, data, resultExpected)
+	f(10, data, data+"\n")
 }

app/vlselect/main.go

@@ -1,7 +1,6 @@
 package vlselect
 
 import (
-	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -102,8 +101,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
-	ctx := r.Context()
-	stopCh := ctx.Done()
+	stopCh := r.Context().Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
@@ -141,15 +139,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 	}
 
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-	stopCh = ctxWithCancel.Done()
-
 	switch {
 	case path == "/logsql/query":
 		logsqlQueryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh, cancel)
+		logsql.ProcessQueryRequest(w, r, stopCh)
 		return true
 	default:
 		return false

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.1e22ee10.css",
-    "main.js": "./static/js/main.92cf3903.js",
+    "main.css": "./static/css/main.d1313636.css",
+    "main.js": "./static/js/main.1919fefe.js",
     "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.48b7b7105a48d7775f01.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.1e22ee10.css",
-    "static/js/main.92cf3903.js"
+    "static/css/main.d1313636.css",
+    "static/js/main.1919fefe.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.92cf3903.js"></script><link href="./static/css/main.1e22ee10.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/media/MetricsQL.8644fd7c964802dd34a9.md

@@ -70,7 +70,7 @@ The list of MetricsQL features on top of PromQL:
   It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
 * Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
-  selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
+  selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
 * Support for `group_left(*)` and `group_right(*)` for copying all the labels from time series on the `one` side
   of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches).
@@ -243,7 +243,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time) and [share_eq_over_time](#share_eq_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_gt_over_time
 
@@ -253,7 +253,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_le_over_time
 
@@ -263,7 +263,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_ne_over_time
 
@@ -743,7 +743,7 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
+See also [share_le_over_time](#share_le_over_time).
 
 #### share_le_over_time
 
@@ -756,7 +756,7 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
+See also [share_gt_over_time](#share_gt_over_time).
 
 #### share_eq_over_time
 
@@ -766,8 +766,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_eq_over_time](#count_eq_over_time).
-
 #### stale_samples_over_time
 
 `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
@@ -794,33 +792,6 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
 
-#### sum_eq_over_time
-
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values equal to `eq`
-on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
-
-Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
-
-See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
-
-#### sum_gt_over_time
-
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values bigger than `gt`
-on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
-
-Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
-
-See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
-
-#### sum_le_over_time
-
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values smaller or equal to `le`
-on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
-
-Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
-
-See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
-
 #### sum_over_time
 
 `sum_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the sum of raw sample values
@@ -839,7 +810,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -848,7 +819,7 @@ This function is supported by PromQL. See also [timestamp_with_name](#timestamp_
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
@@ -857,7 +828,7 @@ See also [timestamp](#timestamp).
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -866,7 +837,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -881,7 +852,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -891,7 +862,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -1058,7 +1029,7 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [acosh](#acosh).
+This function is supported by PromQL. This function is supported by PromQL. See also [acosh](#acosh).
 
 #### day_of_month
 
@@ -1078,15 +1049,6 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
-#### day_of_year
-
-`day_of_year(q)` is a [transform function](#transform-functions), which returns the day of year for every point of every time series returned by `q`.
-It is expected that `q` returns unix timestamps. The returned values are in the range `[1...365]` for non-leap years, and `[1 to 366]` in leap years.
-
-Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
-
-This function is supported by PromQL.
-
 #### days_in_month
 
 `days_in_month(q)` is a [transform function](#transform-functions), which returns the number of days in the month identified

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompbmarshal.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompbmarshal.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/main.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
@@ -42,14 +41,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -70,8 +70,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -120,11 +119,7 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
@@ -147,7 +142,9 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -156,11 +153,13 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -231,6 +230,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"-/reload", "reload configuration"},
@@ -369,15 +369,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -432,7 +423,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -441,7 +432,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -451,13 +442,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/stream-agg":
+		streamaggr.WriteHumanReadableState(w, r, remotewrite.GetAggregators())
+		return true
 	case "/ready":
 		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
@@ -613,15 +604,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -678,9 +660,6 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)

app/vmagent/remotewrite/client.go

@@ -18,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -35,7 +34,6 @@ var (
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -123,7 +121,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	tr := &http.Transport{
 		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -397,8 +395,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
+	retryDuration := time.Second
 	retriesCount := 0
 
 again:
@@ -408,8 +405,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -455,8 +452,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()

app/vmagent/remotewrite/pendingseries.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -16,7 +15,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -71,8 +69,7 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
+	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
 	for {
 		select {
@@ -110,12 +107,11 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss     []prompbmarshal.TimeSeries
+	tss []prompbmarshal.TimeSeries
+
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-
-	// buf holds labels data
-	buf []byte
+	buf     []byte
 }
 
 func (wr *writeRequest) reset() {
@@ -226,45 +222,33 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
-var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
-
 func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
-
-	marshalConcurrencyCh <- struct{}{}
-
 	bb := writeRequestBufPool.Get()
 	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := compressBufPool.Get()
+		zb := snappyBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
-
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			zbLen := len(zb.B)
-			ok := tryPushBlock(zb.B)
-			compressBufPool.Put(zb)
-			if ok {
-				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockSizeBytes.Update(float64(zbLen))
+			if !tryPushBlock(zb.B) {
+				return false
 			}
-			return ok
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return true
 		}
-		compressBufPool.Put(zb)
+		snappyBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -310,7 +294,5 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/remotewrite.go

@@ -946,3 +946,24 @@ func CheckStreamAggrConfigs() error {
 	}
 	return nil
 }
+
+// GetAggregators returns aggregators for all the configured remote writes.
+func GetAggregators() map[string]*streamaggr.Aggregators {
+	var result = map[string]*streamaggr.Aggregators{}
+
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for tenant, rwctxs := range rwctxsMap {
+			for rwNum, rw := range rwctxs {
+				result[fmt.Sprintf("rw %d for tenant %v:%v", rwNum, tenant.AccountID, tenant.ProjectID)] = rw.sas.Load()
+			}
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		for rwNum, rw := range rwctxsDefault {
+			result[fmt.Sprintf("remote write %d", rwNum)] = rw.sas.Load()
+		}
+	}
+
+	return result
+}

app/vmalert-tool/unittest/unittest.go

@@ -25,7 +25,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -185,8 +184,7 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	var ab flagutil.ArrayBool
-	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
+	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
 		switch r.URL.Path {
 		case "/prometheus/api/v1/query":
 			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
@@ -227,7 +225,7 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
+	if err := httpserver.Stop(httpListenAddr); err != nil {
 		logger.Errorf("cannot stop the webservice: %s", err)
 	}
 	vmstorage.Stop()

app/vmalert/Makefile

@@ -68,7 +68,6 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
-	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -22,7 +22,6 @@ groups:
               {{ . | first | value }}
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-          link: http://localhost:3000/d/wNf0q_kZk?viewPanel=51&from={{($activeAt.Add (parseDurationTime "1h")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime "-1h")).UnixMilli}}
       - alert: ExampleAlertAlwaysFiring
         update_entries_limit: -1
         expr: sum by(job)

app/vmalert/datasource/init.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -94,7 +93,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
 	}
 
-	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/main.go

@@ -59,8 +59,8 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -178,19 +178,15 @@ func main() {
 
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8880"}
-	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
 	pushmetrics.Stop()
 
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
@@ -252,13 +248,7 @@ func newManager(ctx context.Context) (*manager, error) {
 func getExternalURL(customURL string) (*url.URL, error) {
 	if customURL == "" {
 		// use local hostname as external URL
-		listenAddr := ":8880"
-		if len(*httpListenAddrs) > 0 {
-			listenAddr = (*httpListenAddrs)[0]
-		}
-		isTLS := httpserver.IsTLS(0)
-
-		return getHostnameAsExternalURL(listenAddr, isTLS)
+		return getHostnameAsExternalURL(*httpListenAddr, httpserver.IsTLS())
 	}
 	u, err := url.Parse(customURL)
 	if err != nil {
@@ -270,13 +260,13 @@ func getExternalURL(customURL string) (*url.URL, error) {
 	return u, nil
 }
 
-func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
+func getHostnameAsExternalURL(httpListenAddr string, isSecure bool) (*url.URL, error) {
 	hname, err := os.Hostname()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get hostname: %w", err)
 	}
 	port := ""
-	if ipport := strings.Split(addr, ":"); len(ipport) > 1 {
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]
 	}
 	schema := "http://"

app/vmalert/manager.go

@@ -156,14 +156,11 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
-			// cancel evaluation so the Update will be applied as fast as possible.
-			// it is important to call InterruptEval before the update, because cancel fn
-			// can be re-assigned during the update.
-			item.old.InterruptEval()
 			go func(old *rule.Group, new *rule.Group) {
 				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
+			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alertmanager.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -128,7 +127,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remoteread/init.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -61,7 +60,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
-	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/debug_client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -30,7 +30,7 @@ func NewDebugClient() (*DebugClient, error) {
 		return nil, nil
 	}
 
-	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/init.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -65,7 +64,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, nil
 	}
 
-	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/rule/alerting.go

@@ -324,6 +324,16 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 
+		// if alert is instant, For: 0
+		if ar.For == 0 {
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
+		}
+
+		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
@@ -344,10 +354,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.Start = at
 			}
 			prevT = at
-			if ar.For == 0 {
-				// rules with `for: 0` are always firing when they have Value
-				a.State = notifier.StateFiring
-			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
 			// save alert's state on last iteration, so it can be used on the next execRange call
@@ -440,13 +446,14 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, ts, qFn)
+		a, err := ar.newAlert(m, ls, start, qFn)
 		if err != nil {
 			curState.Err = fmt.Errorf("failed to create alert: %w", err)
 			return nil, curState.Err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
+		a.ActiveAt = ts
 		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
@@ -472,7 +479,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -552,9 +559,9 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 }
 
 const (
-	// alertMetricName is the metric name for time series reflecting the alert state.
+	// alertMetricName is the metric name for synthetic alert timeseries.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
+	// alertForStateMetricName is the metric name for 'for' state of alert.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -569,10 +576,12 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	return []prompbmarshal.TimeSeries{
-		alertToTimeSeries(a, timestamp),
-		alertForToTimeSeries(a, timestamp),
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
+	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {

app/vmalert/rule/alerting_test.go

@@ -28,26 +28,20 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					"job":      "foo",
-					"instance": "bar",
-				}},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -55,33 +49,19 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-						"job":      "foo",
-						"instance": "bar",
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					alertStateLabel: "foo",
-					"__name__":      "bar",
-				}},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__":      alertForStateMetricName,
-						alertStateLabel: "foo",
-					}),
 			},
 		},
 		{
@@ -328,17 +308,14 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &datasource.FakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
-			ts := time.Now()
 			for i, step := range tc.steps {
 				fq.Reset()
 				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
+				if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-
-				// shift the execution timestamp before the next iteration
-				ts = ts.Add(defaultStep)
-
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -390,7 +367,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring},
 			},
 			nil,
 		},
@@ -401,9 +378,8 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels:   map[string]string{"name": "foo"},
-					State:    notifier.StateFiring,
-					ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
 				},
 			},
 			nil,
@@ -414,9 +390,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
 			},
 			nil,
 		},
@@ -484,20 +460,6 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				For:         time.Second,
 			}},
 		},
-		{
-			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
-			[]datasource.Metric{
-				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
-			},
-			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
-				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-			},
-			nil,
-		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
 			[]datasource.Metric{
@@ -572,25 +534,21 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					}},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
 				//
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					}},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
 			},
 			nil,
 		},
@@ -1137,12 +1095,6 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 	return &rule
 }
 
-func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
-	rule := newTestAlertingRule(name, waitFor)
-	rule.EvalInterval = evalInterval
-	return rule
-}
-
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor

app/vmalert/static/js/custom.js

@@ -1,12 +1,5 @@
 function expandAll() {
-    $('.group-heading').each(function () {
-        let style = $(this).attr("style")
-        // display only elements that are currently visible
-        if (style === "display: none;") {
-            return
-        }
-        $(this).next().addClass('show')
-    });
+    $('.collapse').addClass('show');
 }
 
 function collapseAll() {
@@ -22,100 +15,6 @@ function toggleByID(id) {
     }
 }
 
-function debounce(func, delay) {
-    let timer;
-    return function (...args) {
-        clearTimeout(timer);
-        timer = setTimeout(() => {
-            func.apply(this, args);
-        }, delay);
-    };
-}
-
-$('#search').on("keyup", debounce(search, 500));
-
-// search shows or hides groups&rules that satisfy the search phrase.
-// case-insensitive, respects GET param `search`.
-function search() {
-    $(".rule").show();
-
-    let groupHeader = $(".group-heading")
-    let searchPhrase = $("#search").val().toLowerCase()
-    if (searchPhrase.length === 0) {
-        groupHeader.show()
-        setParamURL('search', '')
-        return
-    }
-
-    $(".rule-table").removeClass('show');
-    groupHeader.hide()
-
-    searchPhrase = searchPhrase.toLowerCase()
-    filterRuleByName(searchPhrase);
-    filterRuleByLabels(searchPhrase);
-    filterGroupsByName(searchPhrase);
-
-    setParamURL('search', searchPhrase)
-}
-
-function setParamURL(key, value) {
-    let url = new URL(location.href)
-    url.searchParams.set(key, value);
-    window.history.replaceState(null, null, `?${url.searchParams.toString()}`);
-}
-
-function getParamURL(key) {
-    let url = new URL(location.href)
-    return url.searchParams.get(key)
-}
-
-function filterGroupsByName(searchPhrase) {
-    $(".group-heading").each(function () {
-        const groupName = $(this).attr('data-group-name').toLowerCase();
-        const hasValue = groupName.indexOf(searchPhrase) >= 0
-
-        if (!hasValue) {
-            return
-        }
-
-        const target = $(this).attr("data-bs-target");
-        $(`div[id="${target}"] .rule`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByName(searchPhrase) {
-    $(".rule").each(function () {
-        const ruleName = $(this).attr("data-rule-name").toLowerCase();
-        const hasValue = ruleName.indexOf(searchPhrase) >= 0
-        if (!hasValue) {
-            $(this).hide();
-            return
-        }
-
-        const target = $(this).attr('data-bs-target')
-        $(`#rules-${target}`).addClass('show');
-        $(`div[data-bs-target='rules-${target}']`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByLabels(searchPhrase) {
-    $(".rule").each(function () {
-        const matches = $(".label", this).filter(function () {
-            const label = $(this).text().toLowerCase();
-            return label.indexOf(searchPhrase) >= 0;
-        }).length;
-
-        if (matches > 0) {
-            const target = $(this).attr('data-bs-target')
-            $(`#rules-${target}`).addClass('show');
-            $(`div[data-bs-target='rules-${target}']`).show();
-            $(this).show();
-        }
-    });
-}
-
 $(document).ready(function () {
     $(".group-heading a").click(function (e) {
         e.stopPropagation(); // prevent collapse logic on link click
@@ -133,13 +32,6 @@ $(document).ready(function () {
         });
     });
 
-    // update search element with value from URL, if any
-    let searchPhrase = getParamURL('search')
-    $("#search").val(searchPhrase)
-
-    // apply filtering by search phrase
-    search()
-
     let hash = window.location.hash.substr(1);
     toggleByID(hash);
 });

app/vmalert/utils/tls.go

@@ -1,4 +1,4 @@
-package httputils
+package utils
 
 import (
 	"crypto/tls"

app/vmalert/utils/tls_test.go

@@ -1,4 +1,4 @@
-package httputils
+package utils
 
 import "testing"
 

app/vmalert/web.go

@@ -12,15 +12,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
-var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
-
 var (
 	apiLinks = [][2]string{
 		// api links are relative since they can be used by external clients,
@@ -88,10 +84,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		var data []apiGroup
-		rf := extractRulesFilter(r)
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		WriteListGroups(w, r, rh.groups())
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -102,20 +95,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		var data []apiGroup
-		rf := extractRulesFilter(r)
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		WriteListGroups(w, r, rh.groups())
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		var data []byte
-		var err error
-
-		rf := extractRulesFilter(r)
-		data, err = rh.listGroups(rf)
-
+		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -123,7 +108,6 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
-
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
@@ -167,9 +151,6 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -220,94 +201,26 @@ type listGroupsResponse struct {
 	} `json:"data"`
 }
 
-// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
-type rulesFilter struct {
-	files         []string
-	groupNames    []string
-	ruleNames     []string
-	ruleType      string
-	excludeAlerts bool
-}
-
-func extractRulesFilter(r *http.Request) rulesFilter {
-	rf := rulesFilter{}
-
-	var ruleType string
-	ruleTypeParam := r.URL.Query().Get("type")
-	// for some reason, `type` in filter doesn't match `type` in response,
-	// so we use this matching here
-	if ruleTypeParam == "alert" {
-		ruleType = ruleTypeAlerting
-	} else if ruleTypeParam == "record" {
-		ruleType = ruleTypeRecording
-	}
-	rf.ruleType = ruleType
-
-	rf.excludeAlerts = httputils.GetBool(r, "exclude_alerts")
-	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
-	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
-	rf.files = append([]string{}, r.Form["file[]"]...)
-	return rf
-}
-
-func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
+func (rh *requestHandler) groups() []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	isInList := func(list []string, needle string) bool {
-		if len(list) < 1 {
-			return true
-		}
-		for _, i := range list {
-			if i == needle {
-				return true
-			}
-		}
-		return false
-	}
-
 	groups := make([]apiGroup, 0)
-	for _, group := range rh.m.groups {
-		if !isInList(rf.groupNames, group.Name) {
-			continue
-		}
-		if !isInList(rf.files, group.File) {
-			continue
-		}
-
-		g := groupToAPI(group)
-		// the returned list should always be non-nil
-		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
-		filteredRules := make([]apiRule, 0)
-		for _, r := range g.Rules {
-			if rf.ruleType != "" && rf.ruleType != r.Type {
-				continue
-			}
-			if !isInList(rf.ruleNames, r.Name) {
-				continue
-			}
-			if rf.excludeAlerts {
-				r.Alerts = nil
-			}
-			filteredRules = append(filteredRules, r)
-		}
-		g.Rules = filteredRules
-		groups = append(groups, g)
+	for _, g := range rh.m.groups {
+		groups = append(groups, groupToAPI(g))
 	}
-	// sort list of groups for deterministic output
+
+	// sort list of alerts for deterministic output
 	sort.Slice(groups, func(i, j int) bool {
-		a, b := groups[i], groups[j]
-		if a.Name != b.Name {
-			return a.Name < b.Name
-		}
-		return a.File < b.File
+		return groups[i].Name < groups[j].Name
 	})
+
 	return groups
 }
 
-func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
+func (rh *requestHandler) listGroups() ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups(rf)
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{

app/vmalert/web.qtpl

@@ -70,29 +70,15 @@ btn-primary
                 }
             }
         %}
-        <div class="btn-toolbar mb-3" role="toolbar">
-          <div>
-            <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
-            <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-            <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-            <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
-            <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
-          </div>
-          <div class="col-md-4 col-lg-5">
-            <div class="px-3 input-group">
-              <div class="input-group-prepend">
-                <span class="input-group-text">
-                  <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                </span>
-              </div>
-              <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-            </div>
-          </div>
-        </div>
+         <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
+         <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
         {%  if len(groups) > 0 %}
             {% for _, g := range groups  %}
                   <div
-                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
+                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}">
                     <span class="anchor" id="group-{%s g.ID %}"></span>
                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                      {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
@@ -114,7 +100,7 @@ btn-primary
                         </div>
                     {% endif %}
                 </div>
-                <div class="collapse rule-table" id="rules-{%s g.ID %}">
+                <div class="collapse" id="rules-{%s g.ID %}">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
                             <tr>
@@ -125,7 +111,7 @@ btn-primary
                         </thead>
                         <tbody>
                         {% for _, r := range g.Rules %}
-                            <tr class="rule{% if r.LastError != "" %} alert-danger{% endif %}" data-rule-name="{%s r.Name %}" data-bs-target="{%s g.ID %}">
+                            <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
                                 <td>
                                     <div class="row">
                                         <div class="col-12 mb-2">
@@ -148,7 +134,7 @@ btn-primary
                                         <div class="col-12 mb-2">
                                             {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
                                             {% for k, v := range r.Labels %}
-                                                    <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
+                                                    <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
                                             {% endfor %}
                                         </div>
                                         {% if r.LastError != "" %}
@@ -184,25 +170,11 @@ btn-primary
     {%code prefix := utils.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {% if len(groupAlerts) > 0 %}
-         <div class="btn-toolbar mb-3" role="toolbar">
-              <div>
-                <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-                 <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-              </div>
-              <div class="col-md-4 col-lg-5">
-                <div class="px-3 input-group">
-                  <div class="input-group-prepend">
-                    <span class="input-group-text">
-                      <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                    </span>
-                  </div>
-                  <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-                </div>
-              </div>
-          </div>
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
          {% for _, ga := range groupAlerts %}
             {%code g := ga.Group %}
-            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
                 <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
@@ -220,7 +192,7 @@ btn-primary
                 }
                 sort.Strings(keys)
             %}
-            <div class="collapse rule-table" id="rules-{%s g.ID %}">
+            <div class="collapse" id="rules-{%s g.ID %}">
                 {% for _, ruleID := range keys %}
                     {%code
                         defaultAR := alertsByRule[ruleID][0]
@@ -231,46 +203,45 @@ btn-primary
                         sort.Strings(labelKeys)
                     %}
                     <br>
-                    <div class="rule" data-rule-name="{%s defaultAR.Name %}" data-bs-target="{%s g.ID %}">
-                      <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
-                       | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
-                      <br>
-                      <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
-                      <table class="table table-striped table-hover table-sm">
-                          <thead>
-                              <tr>
-                                  <th scope="col">Labels</th>
-                                  <th scope="col">State</th>
-                                  <th scope="col">Active at</th>
-                                  <th scope="col">Value</th>
-                                  <th scope="col">Link</th>
-                              </tr>
-                          </thead>
-                          <tbody>
-                          {% for _, ar := range alertsByRule[ruleID] %}
-                              <tr>
-                                  <td>
-                                      {% for _, k := range labelKeys %}
-                                          <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
-                                      {% endfor %}
-                                  </td>
-                                  <td>{%= badgeState(ar.State) %}</td>
-                                  <td>
-                                      {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
-                                      {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
-                                      {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
-                                  </td>
-                                  <td>{%s ar.Value %}</td>
-                                  <td>
-                                      <a href="{%s prefix+ar.WebLink() %}">Details</a>
-                                  </td>
-                              </tr>
-                          {% endfor %}
-                       </tbody>
-                      </table>
-                    </div>
+                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                    <br>
+                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr>
+                                <th scope="col">Labels</th>
+                                <th scope="col">State</th>
+                                <th scope="col">Active at</th>
+                                <th scope="col">Value</th>
+                                <th scope="col">Link</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                        {% for _, ar := range alertsByRule[ruleID] %}
+                            <tr>
+                                <td>
+                                    {% for _, k := range labelKeys %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
+                                    {% endfor %}
+                                </td>
+                                <td>{%= badgeState(ar.State) %}</td>
+                                <td>
+                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                    {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
+                                </td>
+                                <td>{%s ar.Value %}</td>
+                                <td>
+                                    <a href="{%s prefix+ar.WebLink() %}">Details</a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                     </tbody>
+                    </table>
                 {% endfor %}
             </div>
+            <br>
         {% endfor %}
 
     {% else %}

app/vmalert/web_test.go

@@ -23,7 +23,6 @@ func TestHandler(t *testing.T) {
 	})
 	g := &rule.Group{
 		Name:        "group",
-		File:        "rules.yaml",
 		Concurrency: 1,
 	}
 	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
@@ -166,74 +165,6 @@ func TestHandler(t *testing.T) {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
-
-	t.Run("/api/v1/rules&filters", func(t *testing.T) {
-		check := func(url string, expGroups, expRules int) {
-			t.Helper()
-			lr := listGroupsResponse{}
-			getResp(ts.URL+url, &lr, 200)
-			if length := len(lr.Data.Groups); length != expGroups {
-				t.Errorf("expected %d groups got %d", expGroups, length)
-			}
-			if len(lr.Data.Groups) < 1 {
-				return
-			}
-			var rulesN int
-			for _, gr := range lr.Data.Groups {
-				rulesN += len(gr.Rules)
-			}
-			if rulesN != expRules {
-				t.Errorf("expected %d rules got %d", expRules, rulesN)
-			}
-		}
-
-		check("/api/v1/rules?type=alert", 1, 1)
-		check("/api/v1/rules?type=record", 1, 1)
-
-		check("/vmalert/api/v1/rules?type=alert", 1, 1)
-		check("/vmalert/api/v1/rules?type=record", 1, 1)
-
-		// no filtering expected due to bad params
-		check("/api/v1/rules?type=badParam", 1, 2)
-		check("/api/v1/rules?foo=bar", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
-	})
-	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
-		// check if response returns active alerts by default
-		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml", &lr, 200)
-		activeAlerts := 0
-		for _, gr := range lr.Data.Groups {
-			for _, r := range gr.Rules {
-				activeAlerts += len(r.Alerts)
-			}
-		}
-		if activeAlerts == 0 {
-			t.Fatalf("expected at least 1 active alert in response; got 0")
-		}
-
-		// disable returning alerts via param
-		lr = listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml&exclude_alerts=true", &lr, 200)
-		activeAlerts = 0
-		for _, gr := range lr.Data.Groups {
-			for _, r := range gr.Rules {
-				activeAlerts += len(r.Alerts)
-			}
-		}
-		if activeAlerts != 0 {
-			t.Fatalf("expected to get 0 active alert in response; got %d", activeAlerts)
-		}
-	})
 }
 
 func TestEmptyResponse(t *testing.T) {

app/vmalert/web_types.go

@@ -193,15 +193,10 @@ func ruleToAPI(r interface{}) apiRule {
 	return apiRule{}
 }
 
-const (
-	ruleTypeRecording = "recording"
-	ruleTypeAlerting  = "alerting"
-)
-
 func recordingToAPI(rr *rule.RecordingRule) apiRule {
 	lastState := rule.GetLastEntry(rr)
 	r := apiRule{
-		Type:              ruleTypeRecording,
+		Type:              "recording",
 		DatasourceType:    rr.Type.String(),
 		Name:              rr.Name,
 		Query:             rr.Expr,
@@ -229,7 +224,7 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 func alertingToAPI(ar *rule.AlertingRule) apiRule {
 	lastState := rule.GetLastEntry(ar)
 	r := apiRule{
-		Type:              ruleTypeAlerting,
+		Type:              "alerting",
 		DatasourceType:    ar.Type.String(),
 		Name:              ar.Name,
 		Query:             ar.Expr,

app/vmauth/auth_config.go

@@ -9,7 +9,6 @@ import (
 	"net/url"
 	"os"
 	"regexp"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -17,13 +16,12 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
@@ -43,9 +41,6 @@ var (
 type AuthConfig struct {
 	Users            []UserInfo `yaml:"users,omitempty"`
 	UnauthorizedUser *UserInfo  `yaml:"unauthorized_user,omitempty"`
-
-	// ms holds all the metrics for the given AuthConfig
-	ms *metrics.Set
 }
 
 // UserInfo is user information read from authConfigPath
@@ -65,15 +60,12 @@ type UserInfo struct {
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
-	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
-
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
-	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -274,10 +266,8 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 		if bu.isBroken() {
 			continue
 		}
-		if atomic.LoadInt32(&bu.concurrentRequests) == 0 {
+		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
 			// Fast path - return the backend with zero concurrently executed requests.
-			// Do not use atomic.CompareAndSwapInt32(), since it is much slower on systems with many CPU cores.
-			atomic.AddInt32(&bu.concurrentRequests, 1)
 			return bu
 		}
 	}
@@ -472,19 +462,17 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var (
-	authConfig   atomic.Pointer[AuthConfig]
-	authUsers    atomic.Pointer[map[string]*UserInfo]
-	authConfigWG sync.WaitGroup
-	stopCh       chan struct{}
-)
+var authConfig atomic.Pointer[AuthConfig]
+var authUsers atomic.Pointer[map[string]*UserInfo]
+var authConfigWG sync.WaitGroup
+var stopCh chan struct{}
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
 func loadAuthConfig() (bool, error) {
-	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
+	data, err := fs.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
@@ -506,11 +494,6 @@ func loadAuthConfig() (bool, error) {
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
-	prevAc := authConfig.Load()
-	if prevAc != nil {
-		metrics.UnregisterSet(prevAc.ms)
-	}
-	metrics.RegisterSet(ac.ms)
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
@@ -523,13 +506,10 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-	ac := &AuthConfig{
-		ms: metrics.NewSet(),
-	}
-	if err = yaml.UnmarshalStrict(data, ac); err != nil {
+	var ac AuthConfig
+	if err = yaml.UnmarshalStrict(data, &ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
-
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		if ui.Username != "" {
@@ -547,30 +527,23 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-
-		metricLabels, err := ui.getMetricLabels()
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
-		}
-		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
-		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
-		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = ac.ms.NewCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
-
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
 		ui.httpTransport = tr
 	}
-	return ac, nil
+	return &ac, nil
 }
 
 func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
@@ -581,47 +554,43 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		if ui.Username != "" && ui.Password == "" {
-			// Do not allow setting username without password if there are other auth configs exist.
-			// This should prevent from typical mis-configuration when access by username without password
-			// remains open if other authorization schemes are defined.
-			if ui.BearerToken != "" {
-				return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
-			}
+		if ui.BearerToken == "" && ui.Username == "" {
+			return nil, fmt.Errorf("either bearer_token or username must be set")
 		}
-		ats := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
-		if len(ats) == 0 {
-			return nil, fmt.Errorf("one of bearer_token, username or mtls must be set")
+		if ui.BearerToken != "" && ui.Username != "" {
+			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
 		}
-		for _, at := range ats {
-			if uiOld := byAuthToken[at]; uiOld != nil {
-				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
-					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
-			}
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+		}
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
 		}
 
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
-		if ui.BearerToken != "" && ui.Password != "" {
-			return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
+		name := ui.name()
+		if ui.BearerToken != "" {
+			if ui.Password != "" {
+				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
-
-		metricLabels, err := ui.getMetricLabels()
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
+		if ui.Username != "" {
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
-		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
-		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
-		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -631,36 +600,12 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		}
 		ui.httpTransport = tr
 
-		for _, at := range ats {
-			byAuthToken[at] = ui
-		}
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
-var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
-
-func (ui *UserInfo) getMetricLabels() (string, error) {
-	name := ui.name()
-	if len(name) == 0 && len(ui.MetricLabels) == 0 {
-		// fast path
-		return "", nil
-	}
-	labels := make([]string, 0, len(ui.MetricLabels)+1)
-	if len(name) > 0 {
-		labels = append(labels, fmt.Sprintf(`username=%q`, name))
-	}
-	for k, v := range ui.MetricLabels {
-		if !labelNameRegexp.MatchString(k) {
-			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
-		}
-		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
-	}
-	sort.Strings(labels)
-	labelsStr := "{" + strings.Join(labels, ",") + "}"
-	return labelsStr, nil
-}
-
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
@@ -731,51 +676,29 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		h := xxhash.Sum64([]byte(ui.BearerToken))
-		return fmt.Sprintf("bearer_token:hash:%016X", h)
+		return "bearer_token"
 	}
 	return ""
 }
 
-func getAuthTokens(bearerToken, username, password string) []string {
-	var ats []string
+func getAuthTokens(bearerToken, username, password string) (string, string) {
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getHTTPAuthBearerToken(bearerToken)
-		at2 := getHTTPAuthBasicToken(bearerToken, "")
-		ats = append(ats, at1, at2)
-	} else if username != "" {
-		at := getHTTPAuthBasicToken(username, password)
-		ats = append(ats, at)
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
 	}
-	return ats
+	at := getAuthToken("", username, password)
+	return at, at
 }
 
-func getHTTPAuthBearerToken(bearerToken string) string {
-	return "http_auth:Bearer " + bearerToken
-}
-
-func getHTTPAuthBasicToken(username, password string) string {
+func getAuthToken(bearerToken, username, password string) string {
+	if bearerToken != "" {
+		return "Bearer " + bearerToken
+	}
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "http_auth:Basic " + token64
-}
-
-func getAuthTokensFromRequest(r *http.Request) []string {
-	var ats []string
-
-	ah := r.Header.Get("Authorization")
-	if ah == "" {
-		return ats
-	}
-	if strings.HasPrefix(ah, "Token ") {
-		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-		// See https://docs.influxdata.com/influxdb/v2.0/api/
-		ah = strings.Replace(ah, "Token", "Bearer", 1)
-	}
-	at := "http_auth:" + ah
-	ats = append(ats, at)
-	return ats
+	return "Basic " + token64
 }
 
 func (up *URLPrefix) sanitize() error {

app/vmauth/auth_config_test.go

@@ -229,14 +229,6 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
-`)
-	// Invalid metric label name
-	f(`
-users:
-- username: foo
-  url_prefix: http://foo.bar
-  metric_labels:
-    not-prometheus-compatible: value
 `)
 }
 
@@ -267,7 +259,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", "bar"): {
+		getAuthToken("", "foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -290,7 +282,7 @@ users:
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", "bar"): {
+		getAuthToken("", "foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
@@ -312,11 +304,11 @@ users:
 - username: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", ""): {
+		getAuthToken("", "foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getHTTPAuthBasicToken("bar", ""): {
+		getAuthToken("", "bar", ""): {
 			Username:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
@@ -336,7 +328,7 @@ users:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getHTTPAuthBearerToken("foo"): {
+		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -365,7 +357,7 @@ users:
 				},
 			},
 		},
-		getHTTPAuthBasicToken("foo", ""): {
+		getAuthToken("", "foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -395,7 +387,7 @@ users:
 			},
 		},
 	})
-	// Multiple users with the same name - this should work, since these users have different passwords
+	// Multiple users with the same name
 	f(`
 users:
 - username: foo-same
@@ -405,18 +397,17 @@ users:
   password: bar
   url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo-same", "baz"): {
+		getAuthToken("", "foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getHTTPAuthBasicToken("foo-same", "bar"): {
+		getAuthToken("", "foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
-
 	// with default url
 	f(`
 users:
@@ -433,7 +424,7 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getHTTPAuthBearerToken("foo"): {
+		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -465,7 +456,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getHTTPAuthBasicToken("foo", ""): {
+		getAuthToken("", "foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -498,41 +489,7 @@ users:
 			}),
 		},
 	})
-	// With metric_labels
-	f(`
-users:
-- username: foo-same
-  password: baz
-  url_prefix: http://foo
-  metric_labels:
-    dc: eu
-    team: dev
-- username: foo-same
-  password: bar
-  url_prefix: https://bar/x///
-  metric_labels:
-    backend_env: test
-    team: accounting
-`, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo-same", "baz"): {
-			Username:  "foo-same",
-			Password:  "baz",
-			URLPrefix: mustParseURL("http://foo"),
-			MetricLabels: map[string]string{
-				"dc":   "eu",
-				"team": "dev",
-			},
-		},
-		getHTTPAuthBasicToken("foo-same", "bar"): {
-			Username:  "foo-same",
-			Password:  "bar",
-			URLPrefix: mustParseURL("https://bar/x"),
-			MetricLabels: map[string]string{
-				"backend_env": "test",
-				"team":        "accounting",
-			},
-		},
-	})
+
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -559,7 +516,7 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getHTTPAuthBasicToken("foo", "bar")]
+	ui := m[getAuthToken("", "foo", "bar")]
 	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
@@ -569,86 +526,6 @@ unauthorized_user:
 	}
 }
 
-func TestUserInfoGetMetricLabels(t *testing.T) {
-	t.Run("empty-labels", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "user1",
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-username", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "user1",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-name", func(t *testing.T) {
-		ui := &UserInfo{
-			Name:        "user1",
-			BearerToken: "abc",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-bearer-token", func(t *testing.T) {
-		ui := &UserInfo{
-			BearerToken: "abc",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("invalid-label", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "foo",
-			MetricLabels: map[string]string{
-				",{": "aaaa",
-			},
-		}
-		_, err := ui.getMetricLabels()
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	})
-}
-
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false

app/vmauth/example_config.yml

@@ -10,11 +10,6 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
-  # Adds labels to the exported metrics for given user section
-  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
-  metric_labels:
-    backend_dc: eu
-    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.

app/vmauth/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -33,8 +33,8 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
@@ -45,7 +45,7 @@ var (
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
@@ -65,14 +65,10 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8427"}
-	}
-	logger.Infof("starting vmauth at %q...", listenAddrs)
+	logger.Infof("starting vmauth at %q...", *httpListenAddr)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -81,8 +77,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -93,7 +89,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *reloadAuthKey, "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -101,9 +97,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
-
-	ats := getAuthTokensFromRequest(r)
-	if len(ats) == 0 {
+	authToken := r.Header.Get("Authorization")
+	if authToken == "" {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
@@ -115,12 +110,18 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
+	if strings.HasPrefix(authToken, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
+	}
 
-	ui := getUserInfoByAuthTokens(ats)
+	ac := *authUsers.Load()
+	ui := ac[authToken]
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
+			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
@@ -136,17 +137,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
-func getUserInfoByAuthTokens(ats []string) *UserInfo {
-	ac := *authUsers.Load()
-	for _, at := range ats {
-		ui := ac[at]
-		if ui != nil {
-			return ui
-		}
-	}
-	return nil
-}
-
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
@@ -160,20 +150,12 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if err := ui.beginConcurrencyLimit(); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
-
-			// Requests failed because of concurrency limit must be counted as errors,
-			// since this usually means the backend cannot keep up with the current load.
-			ui.backendErrors.Inc()
 			return
 		}
 	default:
 		concurrentRequestsLimitReached.Inc()
 		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
 		handleConcurrencyLimitError(w, r, err)
-
-		// Requests failed because of concurrency limit must be counted as errors,
-		// since this usually means the backend cannot keep up with the current load.
-		ui.backendErrors.Inc()
 		return
 	}
 	processRequest(w, r, ui)
@@ -219,7 +201,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
 		bu.put()
 		if ok {
 			return
@@ -231,16 +213,15 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
-	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	req.Host = targetURL.Host
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := ui.httpTransport.RoundTrip(req)
+	res, err := transport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -248,20 +229,15 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-			if errors.Is(err, context.DeadlineExceeded) {
-				// Timed out request must be counted as errors, since this usually means that the backend is slow.
-				ui.backendErrors.Inc()
-			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
+				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
-			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -271,20 +247,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
 		return false
 	}
-	if hasInt(retryStatusCodes, res.StatusCode) {
-		_ = res.Body.Close()
-		if !rtbOK || !rtb.canRetry() {
-			// If we get an error from the retry_status_codes list, but cannot execute retry,
-			// we consider such a request an error as well.
-			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
-					res.StatusCode, targetURL),
-				StatusCode: http.StatusServiceUnavailable,
-			}
-			httpserver.Errorf(w, r, "%s", err)
-			ui.backendErrors.Inc()
-			return true
-		}
+	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
@@ -303,7 +266,6 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
 	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
 	copyBufPool.Put(copyBuf)
-	_ = res.Body.Close()
 	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
@@ -431,10 +393,8 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var (
-	transportMap     = make(map[string]*http.Transport)
-	transportMapLock sync.Mutex
-)
+var transportMap = make(map[string]*http.Transport)
+var transportMapLock sync.Mutex
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)
@@ -462,7 +422,7 @@ func newTransport(insecureSkipVerify bool, caFile string) (*http.Transport, erro
 		tlsCfg.ClientSessionCache = tls.NewLRUClientSessionCache(0)
 		tlsCfg.InsecureSkipVerify = insecureSkipVerify
 		if caFile != "" {
-			data, err := fscore.ReadFileOrHTTP(caFile)
+			data, err := fs.ReadFileOrHTTP(caFile)
 			if err != nil {
 				return nil, fmt.Errorf("cannot read tls_ca_file: %w", err)
 			}

app/vmbackup/main.go

@@ -20,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
 )
 
 var (
@@ -94,8 +93,7 @@ func main() {
 		}
 	}
 
-	listenAddrs := []string{*httpListenAddr}
-	go httpserver.Serve(listenAddrs, nil, nil)
+	go httpserver.Serve(*httpListenAddr, false, nil)
 
 	pushmetrics.Init()
 	err := makeBackup()
@@ -106,8 +104,8 @@ func main() {
 	pushmetrics.Stop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
@@ -170,7 +168,7 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
 }
 
 func newSrcFS() (*fslocal.FS, error) {
-	if err := snapshotutil.Validate(*snapshotName); err != nil {
+	if err := snapshot.Validate(*snapshotName); err != nil {
 		return nil, fmt.Errorf("invalid -snapshotName=%q: %w", *snapshotName, err)
 	}
 	snapshotPath := filepath.Join(*storageDataPath, "snapshots", *snapshotName)

app/vmctl/backoff/backoff_test.go

@@ -15,6 +15,7 @@ func TestRetry_Do(t *testing.T) {
 		backoffFactor      float64
 		backoffMinDuration time.Duration
 		retryableFunc      retryableFunc
+		ctx                context.Context
 		cancelTimeout      time.Duration
 		want               uint64
 		wantErr            bool
@@ -24,6 +25,7 @@ func TestRetry_Do(t *testing.T) {
 			retryableFunc: func() error {
 				return ErrBadRequest
 			},
+			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -33,6 +35,7 @@ func TestRetry_Do(t *testing.T) {
 				time.Sleep(time.Millisecond * 100)
 				return nil
 			},
+			ctx:     context.Background(),
 			want:    0,
 			wantErr: true,
 		},
@@ -55,6 +58,7 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
+			ctx:     context.Background(),
 			want:    1,
 			wantErr: false,
 		},
@@ -71,6 +75,7 @@ func TestRetry_Do(t *testing.T) {
 				}
 				return nil
 			},
+			ctx:     context.Background(),
 			want:    5,
 			wantErr: true,
 		},
@@ -80,8 +85,14 @@ func TestRetry_Do(t *testing.T) {
 			backoffFactor:      1.7,
 			backoffMinDuration: time.Millisecond * 10,
 			retryableFunc: func() error {
-				return fmt.Errorf("got some error")
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					return fmt.Errorf("got some error")
+				}
+				return nil
 			},
+			ctx:           context.Background(),
 			cancelTimeout: time.Millisecond * 40,
 			want:          3,
 			wantErr:       true,
@@ -90,13 +101,12 @@ func TestRetry_Do(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			r := &Backoff{retries: tt.backoffRetries, factor: tt.backoffFactor, minDuration: tt.backoffMinDuration}
-			ctx := context.Background()
 			if tt.cancelTimeout != 0 {
-				newCtx, cancelFn := context.WithTimeout(context.Background(), tt.cancelTimeout)
-				ctx = newCtx
+				newCtx, cancelFn := context.WithTimeout(tt.ctx, tt.cancelTimeout)
+				tt.ctx = newCtx
 				defer cancelFn()
 			}
-			got, err := r.Retry(ctx, tt.retryableFunc)
+			got, err := r.Retry(tt.ctx, tt.retryableFunc)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
 				return

app/vmctl/flags.go

@@ -123,20 +123,15 @@ var (
 )
 
 const (
-	otsdbAddr               = "otsdb-addr"
-	otsdbConcurrency        = "otsdb-concurrency"
-	otsdbQueryLimit         = "otsdb-query-limit"
-	otsdbOffsetDays         = "otsdb-offset-days"
-	otsdbHardTSStart        = "otsdb-hard-ts-start"
-	otsdbRetentions         = "otsdb-retentions"
-	otsdbFilters            = "otsdb-filters"
-	otsdbNormalize          = "otsdb-normalize"
-	otsdbMsecsTime          = "otsdb-msecstime"
-	otsdbCertFile           = "otsdb-cert-file"
-	otsdbKeyFile            = "otsdb-key-file"
-	otsdbCAFile             = "otsdb-CA-file"
-	otsdbServerName         = "otsdb-server-name"
-	otsdbInsecureSkipVerify = "otsdb-insecure-skip-verify"
+	otsdbAddr        = "otsdb-addr"
+	otsdbConcurrency = "otsdb-concurrency"
+	otsdbQueryLimit  = "otsdb-query-limit"
+	otsdbOffsetDays  = "otsdb-offset-days"
+	otsdbHardTSStart = "otsdb-hard-ts-start"
+	otsdbRetentions  = "otsdb-retentions"
+	otsdbFilters     = "otsdb-filters"
+	otsdbNormalize   = "otsdb-normalize"
+	otsdbMsecsTime   = "otsdb-msecstime"
 )
 
 var (
@@ -196,27 +191,6 @@ var (
 			Value: false,
 			Usage: "Whether to normalize all data received to lower case before forwarding to VictoriaMetrics",
 		},
-		&cli.StringFlag{
-			Name:  otsdbCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr",
-		},
-		&cli.StringFlag{
-			Name:  otsdbKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to -otsdb-addr",
-		},
-		&cli.StringFlag{
-			Name:  otsdbCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used",
-		},
-		&cli.StringFlag{
-			Name:  otsdbServerName,
-			Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from otsdbAddr is used",
-		},
-		&cli.BoolFlag{
-			Name:  otsdbInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to -otsdb-addr",
-			Value: false,
-		},
 	}
 )
 
@@ -234,11 +208,6 @@ const (
 	influxMeasurementFieldSeparator = "influx-measurement-field-separator"
 	influxSkipDatabaseLabel         = "influx-skip-database-label"
 	influxPrometheusMode            = "influx-prometheus-mode"
-	influxCertFile                  = "influx-cert-file"
-	influxKeyFile                   = "influx-key-file"
-	influxCAFile                    = "influx-CA-file"
-	influxServerName                = "influx-server-name"
-	influxInsecureSkipVerify        = "influx-insecure-skip-verify"
 )
 
 var (
@@ -303,28 +272,7 @@ var (
 		},
 		&cli.BoolFlag{
 			Name:  influxPrometheusMode,
-			Usage: "Whether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write.",
-			Value: false,
-		},
-		&cli.StringFlag{
-			Name:  influxCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to -influx-addr",
-		},
-		&cli.StringFlag{
-			Name:  influxKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to -influx-addr",
-		},
-		&cli.StringFlag{
-			Name:  influxCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to -influx-addr. By default, system CA is used",
-		},
-		&cli.StringFlag{
-			Name:  influxServerName,
-			Usage: "Optional TLS server name to use for connections to -influx-addr. By default, the server name from -influx-addr is used",
-		},
-		&cli.BoolFlag{
-			Name:  influxInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to -influx-addr",
+			Usage: "Wether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write.",
 			Value: false,
 		},
 	}
@@ -548,10 +496,6 @@ const (
 	remoteReadPassword           = "remote-read-password"
 	remoteReadHTTPTimeout        = "remote-read-http-timeout"
 	remoteReadHeaders            = "remote-read-headers"
-	remoteReadCertFile           = "remote-read-cert-file"
-	remoteReadKeyFile            = "remote-read-key-file"
-	remoteReadCAFile             = "remote-read-CA-file"
-	remoteReadServerName         = "remote-read-server-name"
 	remoteReadInsecureSkipVerify = "remote-read-insecure-skip-verify"
 	remoteReadDisablePathAppend  = "remote-read-disable-path-append"
 )
@@ -630,22 +574,6 @@ var (
 				"For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. \n" +
 				"Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'",
 		},
-		&cli.StringFlag{
-			Name:  remoteReadCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to -remote-read-src-addr",
-		},
-		&cli.StringFlag{
-			Name:  remoteReadKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to -remote-read-src-addr",
-		},
-		&cli.StringFlag{
-			Name:  remoteReadCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to -remote-read-src-addr. By default, system CA is used",
-		},
-		&cli.StringFlag{
-			Name:  remoteReadServerName,
-			Usage: "Optional TLS server name to use for connections to remoteReadSrcAddr. By default, the server name from -remote-read-src-addr is used",
-		},
 		&cli.BoolFlag{
 			Name:  remoteReadInsecureSkipVerify,
 			Usage: "Whether to skip TLS certificate verification when connecting to the remote read address",

app/vmctl/influx/influx.go

@@ -1,7 +1,6 @@
 package influx
 
 import (
-	"crypto/tls"
 	"fmt"
 	"io"
 	"log"
@@ -34,8 +33,7 @@ type Config struct {
 	Retention string
 	ChunkSize int
 
-	Filter    Filter
-	TLSConfig *tls.Config
+	Filter Filter
 }
 
 // Filter contains configuration for filtering
@@ -88,10 +86,10 @@ type LabelPair struct {
 // configured with passed Config
 func NewClient(cfg Config) (*Client, error) {
 	c := influx.HTTPConfig{
-		Addr:      cfg.Addr,
-		Username:  cfg.Username,
-		Password:  cfg.Password,
-		TLSConfig: cfg.TLSConfig,
+		Addr:               cfg.Addr,
+		Username:           cfg.Username,
+		Password:           cfg.Password,
+		InsecureSkipVerify: true,
 	}
 	hc, err := influx.NewHTTPClient(c)
 	if err != nil {

app/vmctl/main.go

@@ -25,7 +25,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 )
@@ -50,20 +49,8 @@ func main() {
 				Action: func(c *cli.Context) error {
 					fmt.Println("OpenTSDB import mode")
 
-					// create Transport with given TLS config
-					certFile := c.String(otsdbCertFile)
-					keyFile := c.String(otsdbKeyFile)
-					caFile := c.String(otsdbCAFile)
-					serverName := c.String(otsdbServerName)
-					insecureSkipVerify := c.Bool(otsdbInsecureSkipVerify)
-					addr := c.String(otsdbAddr)
-
-					tr, err := httputils.Transport(addr, certFile, caFile, keyFile, serverName, insecureSkipVerify)
-					if err != nil {
-						return fmt.Errorf("failed to create Transport: %s", err)
-					}
 					oCfg := opentsdb.Config{
-						Addr:       addr,
+						Addr:       c.String(otsdbAddr),
 						Limit:      c.Int(otsdbQueryLimit),
 						Offset:     c.Int64(otsdbOffsetDays),
 						HardTS:     c.Int64(otsdbHardTSStart),
@@ -71,7 +58,6 @@ func main() {
 						Filters:    c.StringSlice(otsdbFilters),
 						Normalize:  c.Bool(otsdbNormalize),
 						MsecsTime:  c.Bool(otsdbMsecsTime),
-						Transport:  tr,
 					}
 					otsdbClient, err := opentsdb.NewClient(oCfg)
 					if err != nil {
@@ -98,18 +84,6 @@ func main() {
 				Action: func(c *cli.Context) error {
 					fmt.Println("InfluxDB import mode")
 
-					// create TLS config
-					certFile := c.String(influxCertFile)
-					keyFile := c.String(influxKeyFile)
-					caFile := c.String(influxCAFile)
-					serverName := c.String(influxServerName)
-					insecureSkipVerify := c.Bool(influxInsecureSkipVerify)
-
-					tc, err := httputils.TLSConfig(certFile, caFile, keyFile, serverName, insecureSkipVerify)
-					if err != nil {
-						return fmt.Errorf("failed to create TLS Config: %s", err)
-					}
-
 					iCfg := influx.Config{
 						Addr:      c.String(influxAddr),
 						Username:  c.String(influxUser),
@@ -122,9 +96,7 @@ func main() {
 							TimeEnd:   c.String(influxFilterTimeEnd),
 						},
 						ChunkSize: c.Int(influxChunkSize),
-						TLSConfig: tc,
 					}
-
 					influxClient, err := influx.NewClient(iCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create influx client: %s", err)
@@ -162,10 +134,6 @@ func main() {
 						Headers:            c.String(remoteReadHeaders),
 						LabelName:          c.String(remoteReadFilterLabel),
 						LabelValue:         c.String(remoteReadFilterLabelValue),
-						CertFile:           c.String(remoteReadCertFile),
-						KeyFile:            c.String(remoteReadKeyFile),
-						CAFile:             c.String(remoteReadCAFile),
-						ServerName:         c.String(remoteReadServerName),
 						InsecureSkipVerify: c.Bool(remoteReadInsecureSkipVerify),
 						DisablePathAppend:  c.Bool(remoteReadDisablePathAppend),
 					})

app/vmctl/opentsdb/opentsdb.go

@@ -47,8 +47,6 @@ type Client struct {
 	Normalize  bool
 	HardTS     int64
 	MsecsTime  bool
-
-	c *http.Client
 }
 
 // Config contains fields required
@@ -62,7 +60,6 @@ type Config struct {
 	Filters    []string
 	Normalize  bool
 	MsecsTime  bool
-	Transport  *http.Transport
 }
 
 // TimeRange contains data about time ranges to query
@@ -110,8 +107,7 @@ type Metric struct {
 // FindMetrics discovers all metrics that OpenTSDB knows about (given a filter)
 // e.g. /api/suggest?type=metrics&q=system&max=100000
 func (c Client) FindMetrics(q string) ([]string, error) {
-
-	resp, err := c.c.Get(q)
+	resp, err := http.Get(q)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
@@ -135,7 +131,7 @@ func (c Client) FindMetrics(q string) ([]string, error) {
 // e.g. /api/search/lookup?m=system.load5&limit=1000000
 func (c Client) FindSeries(metric string) ([]Meta, error) {
 	q := fmt.Sprintf("%s/api/search/lookup?m=%s&limit=%d", c.Addr, metric, c.Limit)
-	resp, err := c.c.Get(q)
+	resp, err := http.Get(q)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err)
 	}
@@ -188,7 +184,7 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64, m
 		series.Metric, tagStr)
 
 	q := fmt.Sprintf("%s/api/query?%s", c.Addr, queryStr)
-	resp, err := c.c.Get(q)
+	resp, err := http.Get(q)
 	if err != nil {
 		return Metric{}, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
@@ -329,7 +325,6 @@ func NewClient(cfg Config) (*Client, error) {
 		Normalize:  cfg.Normalize,
 		HardTS:     cfg.HardTS,
 		MsecsTime:  cfg.MsecsTime,
-		c:          &http.Client{Transport: cfg.Transport},
 	}
 	return client, nil
 }

app/vmctl/remoteread/remoteread.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/prompb"
@@ -64,13 +64,6 @@ type Config struct {
 	// LabelName, LabelValue stands for label=~value pair used for read requests.
 	// Is optional.
 	LabelName, LabelValue string
-
-	// Optional cert file, key file, CA file and server name for client side TLS configuration
-	CertFile   string
-	KeyFile    string
-	CAFile     string
-	ServerName string
-
 	// TLSSkipVerify defines whether to skip TLS certificate verification when connecting to the remote read address.
 	InsecureSkipVerify bool
 }
@@ -110,15 +103,10 @@ func NewClient(cfg Config) (*Client, error) {
 		}
 	}
 
-	tr, err := httputils.Transport(cfg.Addr, cfg.CertFile, cfg.KeyFile, cfg.CAFile, cfg.ServerName, cfg.InsecureSkipVerify)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %s", err)
-	}
-
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.Timeout,
-			Transport: tr,
+			Transport: utils.Transport(cfg.Addr, cfg.InsecureSkipVerify),
 		},
 		addr:              strings.TrimSuffix(cfg.Addr, "/"),
 		disablePathAppend: cfg.DisablePathAppend,

app/vmctl/utils/time.go

@@ -1,4 +1,4 @@
-package main
+package utils
 
 import (
 	"fmt"
@@ -13,7 +13,8 @@ const (
 	maxTimeMsecs = int64(1<<63-1) / 1e6
 )
 
-func parseTime(s string) (time.Time, error) {
+// GetTime  returns time from the given string.
+func GetTime(s string) (time.Time, error) {
 	secs, err := promutils.ParseTime(s)
 	if err != nil {
 		return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)

app/vmctl/utils/time_test.go

@@ -1,4 +1,4 @@
-package main
+package utils
 
 import (
 	"testing"
@@ -165,7 +165,7 @@ func TestGetTime(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseTime(tt.s)
+			got, err := GetTime(tt.s)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr)
 				return

app/vmctl/utils/tls.go

@@ -0,0 +1,25 @@
+package utils
+
+import (
+	"crypto/tls"
+	"net/http"
+	"strings"
+)
+
+// Transport creates http.Transport object based on provided URL.
+// Returns Transport with TLS configuration if URL contains `https` prefix
+func Transport(URL string, insecureSkipVerify bool) *http.Transport {
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	if !strings.HasPrefix(URL, "https") {
+		return t
+	}
+	t.TLSClientConfig = TLSConfig(insecureSkipVerify)
+	return t
+}
+
+// TLSConfig creates tls.Config object from provided arguments
+func TLSConfig(insecureSkipVerify bool) *tls.Config {
+	return &tls.Config{
+		InsecureSkipVerify: insecureSkipVerify,
+	}
+}

app/vmctl/vm_native.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -53,14 +54,14 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 		startTime: time.Now(),
 	}
 
-	start, err := parseTime(p.filter.TimeStart)
+	start, err := utils.GetTime(p.filter.TimeStart)
 	if err != nil {
 		return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeStart, p.filter.TimeStart, err)
 	}
 
 	end := time.Now().In(start.Location())
 	if p.filter.TimeEnd != "" {
-		end, err = parseTime(p.filter.TimeEnd)
+		end, err = utils.GetTime(p.filter.TimeEnd)
 		if err != nil {
 			return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeEnd, p.filter.TimeEnd, err)
 		}

app/vminsert/common/streamaggr.go

@@ -211,3 +211,8 @@ func pushAggregateSeries(tss []prompbmarshal.TimeSeries) {
 		logger.Errorf("cannot flush aggregate series: %s", err)
 	}
 }
+
+// GetAggregators returns default aggregator for vmsingle.
+func GetAggregators() map[string]*streamaggr.Aggregators {
+	return map[string]*streamaggr.Aggregators{"default": sasGlobal.Load()}
+}

app/vminsert/datadogsketches/request_handler.go

@@ -1,85 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="datadogsketches"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(sketches, extraLabels)
-	})
-}
-
-func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetInsertCtx()
-	defer common.PutInsertCtx(ctx)
-
-	rowsLen := 0
-	for _, sketch := range sketches {
-		rowsLen += sketch.RowsCount()
-	}
-	ctx.Reset(rowsLen)
-	rowsTotal := 0
-	hasRelabeling := relabel.HasRelabeling()
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			ctx.Labels = ctx.Labels[:0]
-			ctx.AddLabel("", m.Name)
-			for _, label := range m.Labels {
-				ctx.AddLabel(label.Name, label.Value)
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				ctx.AddLabel(name, value)
-			}
-			for j := range extraLabels {
-				label := &extraLabels[j]
-				ctx.AddLabel(label.Name, label.Value)
-			}
-			if hasRelabeling {
-				ctx.ApplyRelabeling()
-			}
-			if len(ctx.Labels) == 0 {
-				// Skip metric without labels.
-				continue
-			}
-			ctx.SortLabelsIfNeeded()
-			var metricNameRaw []byte
-			var err error
-			for _, p := range m.Points {
-				metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, p.Timestamp, p.Value)
-				if err != nil {
-					return err
-				}
-			}
-			rowsTotal += len(m.Points)
-		}
-	}
-	rowsInserted.Add(rowsTotal)
-	rowsPerInsert.Update(float64(rowsTotal))
-	return ctx.FlushBufs()
-}

app/vminsert/main.go

@@ -13,7 +13,6 @@ import (
 
 	vminsertCommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -30,7 +29,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
@@ -42,6 +40,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 )
 
 var (
@@ -64,8 +63,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )
@@ -272,15 +270,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -327,7 +316,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -336,7 +325,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -346,12 +335,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
-		w.WriteHeader(http.StatusOK)
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/stream-agg":
+		streamaggr.WriteHumanReadableState(w, r, vminsertCommon.GetAggregators())
 		return true
 	case "/ready":
 		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
@@ -404,9 +393,6 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
-	datadogsketchesWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-
 	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake", protocol="datadog"}`)

app/vmrestore/main.go

@@ -37,8 +37,7 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	listenAddrs := []string{*httpListenAddr}
-	go httpserver.Serve(listenAddrs, nil, nil)
+	go httpserver.Serve(*httpListenAddr, false, nil)
 
 	srcFS, err := newSrcFS()
 	if err != nil {
@@ -63,8 +62,8 @@ func main() {
 	dstFS.MustStop()
 
 	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop http server for metrics: %s", err)
 	}
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())

app/vmselect/main.go

@@ -18,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
@@ -30,13 +29,13 @@ import (
 )
 
 var (
-	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
+	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
+	resetCacheAuthKey    = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
@@ -149,9 +148,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
 					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusTooManyRequests,
+				StatusCode: http.StatusServiceUnavailable,
 			}
-			w.Header().Add("Retry-After", "10")
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -172,7 +170,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	if path == "/internal/resetRollupResultCache" {
-		if !httpserver.CheckAuthFlag(w, r, resetCacheAuthKey.Get(), "resetCacheAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *resetCacheAuthKey, "resetCacheAuthKey") {
 			return true
 		}
 		promql.ResetRollupResultCache()
@@ -369,7 +367,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
-		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
 			return true
 		}
 		graphiteTagsDelSeriesRequests.Inc()
@@ -388,7 +386,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
-		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey.Get(), "deleteAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
 			return true
 		}
 		deleteRequests.Inc()
@@ -427,14 +425,6 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		}
 		return true
 	}
-	if path == "/vmui/timezone" {
-		httpserver.EnableCORS(w, r)
-		if err := handleVMUITimezone(w); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		return true
-	}
 	if strings.HasPrefix(path, "/vmui/") {
 		if strings.HasPrefix(path, "/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.

app/vmselect/netstorage/netstorage.go

@@ -5,12 +5,10 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"reflect"
 	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
-	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -90,6 +88,30 @@ type timeseriesWork struct {
 	rowsProcessed int
 }
 
+func (tsw *timeseriesWork) reset() {
+	tsw.mustStop = nil
+	tsw.rss = nil
+	tsw.pts = nil
+	tsw.f = nil
+	tsw.err = nil
+	tsw.rowsProcessed = 0
+}
+
+func getTimeseriesWork() *timeseriesWork {
+	v := tswPool.Get()
+	if v == nil {
+		v = &timeseriesWork{}
+	}
+	return v.(*timeseriesWork)
+}
+
+func putTimeseriesWork(tsw *timeseriesWork) {
+	tsw.reset()
+	tswPool.Put(tsw)
+}
+
+var tswPool sync.Pool
+
 func (tsw *timeseriesWork) do(r *Result, workerID uint) error {
 	if atomic.LoadUint32(tsw.mustStop) != 0 {
 		return nil
@@ -248,20 +270,22 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	maxWorkers := MaxWorkers()
 	if maxWorkers == 1 || tswsLen == 1 {
 		// It is faster to process time series in the current goroutine.
-		var tsw timeseriesWork
+		tsw := getTimeseriesWork()
 		tmpResult := getTmpResult()
 		rowsProcessedTotal := 0
 		var err error
 		for i := range rss.packedTimeseries {
-			initTimeseriesWork(&tsw, &rss.packedTimeseries[i])
+			initTimeseriesWork(tsw, &rss.packedTimeseries[i])
 			err = tsw.do(&tmpResult.rs, 0)
 			rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 			rowsProcessedTotal += tsw.rowsProcessed
 			if err != nil {
 				break
 			}
+			tsw.reset()
 		}
 		putTmpResult(tmpResult)
+		putTimeseriesWork(tsw)
 
 		return rowsProcessedTotal, err
 	}
@@ -271,9 +295,11 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// which reduces the scalability on systems with many CPU cores.
 
 	// Prepare the work for workers.
-	tsws := make([]timeseriesWork, len(rss.packedTimeseries))
+	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
 	for i := range rss.packedTimeseries {
-		initTimeseriesWork(&tsws[i], &rss.packedTimeseries[i])
+		tsw := getTimeseriesWork()
+		initTimeseriesWork(tsw, &rss.packedTimeseries[i])
+		tsws[i] = tsw
 	}
 
 	// Prepare worker channels.
@@ -288,9 +314,9 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	}
 
 	// Spread work among workers.
-	for i := range tsws {
+	for i, tsw := range tsws {
 		idx := i % len(workChs)
-		workChs[idx] <- &tsws[i]
+		workChs[idx] <- tsw
 	}
 	// Mark worker channels as closed.
 	for _, workCh := range workChs {
@@ -313,14 +339,14 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	// Collect results.
 	var firstErr error
 	rowsProcessedTotal := 0
-	for i := range tsws {
-		tsw := &tsws[i]
+	for _, tsw := range tsws {
 		if tsw.err != nil && firstErr == nil {
 			// Return just the first error, since other errors are likely duplicate the first error.
 			firstErr = tsw.err
 		}
 		rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 		rowsProcessedTotal += tsw.rowsProcessed
+		putTimeseriesWork(tsw)
 	}
 	return rowsProcessedTotal, firstErr
 }
@@ -1018,7 +1044,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 			for xw := range workCh {
 				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
 					errGlobalLock.Lock()
-					if errGlobal == nil {
+					if errGlobal != nil {
 						errGlobal = err
 						atomic.StoreUint32(&mustStop, 1)
 					}
@@ -1143,44 +1169,15 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
-		brs []blockRef
+		brsPrealloc [4]blockRef
+		brs         []blockRef
 	}
-
+	m := make(map[string]*blockRefs, maxSeriesCount)
+	orderedMetricNames := make([]string, 0, maxSeriesCount)
 	blocksRead := 0
 	samples := 0
 	tbf := getTmpBlocksFile()
 	var buf []byte
-	var metricNamePrev []byte
-
-	// metricNamesBuf is used for holding all the loaded unique metric names at m and orderedMetricNames.
-	// It should reduce pressure on Go GC by reducing the number of string allocations
-	// when constructing metricName string from byte slice.
-	metricNamesBufCap := maxSeriesCount * 100
-	if metricNamesBufCap > maxFastAllocBlockSize {
-		metricNamesBufCap = maxFastAllocBlockSize
-	}
-	metricNamesBuf := make([]byte, 0, metricNamesBufCap)
-
-	// brssPool is used for holding all the blockRefs objects across all the loaded time series.
-	// It should reduce pressure on Go GC by reducing the number of blockRefs allocations.
-	brssPool := make([]blockRefs, 0, maxSeriesCount)
-
-	// brsPool is used for holding the most of blockRefs.brs slices across all the loaded time series.
-	// It should reduce pressure on Go GC by reducing the number of allocations for blockRefs.brs slices.
-	brsPoolCap := uintptr(maxSeriesCount)
-	if brsPoolCap > maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) {
-		brsPoolCap = maxFastAllocBlockSize / unsafe.Sizeof(blockRef{})
-	}
-	brsPool := make([]blockRef, 0, brsPoolCap)
-
-	// m maps from metricName to the index of blockRefs inside brssPool
-	m := make(map[string]int, maxSeriesCount)
-
-	// orderedMetricNames contains the list of loaded unique metric names
-	// in the load order. This order is important for triggering sequential data reading.
-	orderedMetricNames := make([]string, 0, maxSeriesCount)
-
-	var brsIdx int
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
@@ -1193,10 +1190,8 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		if *maxSamplesPerQuery > 0 && samples > *maxSamplesPerQuery {
 			putTmpBlocksFile(tbf)
 			putStorageSearch(sr)
-			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: increase the -search.maxSamplesPerQuery; "+
-				"reduce time range for the query; use more specific label filters in order to select fewer series", *maxSamplesPerQuery)
+			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: to increase the -search.maxSamplesPerQuery; to reduce time range for the query; to use more specific label filters in order to select lower number of series", *maxSamplesPerQuery)
 		}
-
 		buf = br.Marshal(buf[:0])
 		addr, err := tbf.WriteBlockRefData(buf)
 		if err != nil {
@@ -1204,59 +1199,24 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-
+		// Do not intern mb.MetricName, since it leads to increased memory usage.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
 		metricName := sr.MetricBlockRef.MetricName
-		if metricNamePrev == nil || string(metricName) != string(metricNamePrev) {
-			idx, ok := m[string(metricName)]
-			if !ok {
-				if cap(brssPool) > len(brssPool) {
-					brssPool = brssPool[:len(brssPool)+1]
-				} else {
-					brssPool = append(brssPool, blockRefs{})
-				}
-				idx = len(brssPool) - 1
-			}
-			brsIdx = idx
-			metricNamePrev = append(metricNamePrev[:0], metricName...)
-		}
-
-		brs := &brssPool[brsIdx]
-		partRef := br.PartRef()
-		if uintptr(cap(brsPool)) >= maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) && len(brsPool) == cap(brsPool) {
-			// Allocate a new brsPool in order to avoid slow allocation of an object
-			// bigger than maxFastAllocBlockSize bytes at append() below.
-			brsPool = make([]blockRef, 0, maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}))
-		}
-		if canAppendToBlockRefPool(brsPool, brs.brs) {
-			// It is safe appending blockRef to brsPool, since there are no other items added there yet.
-			brsPool = append(brsPool, blockRef{
-				partRef: partRef,
-				addr:    addr,
-			})
-			brs.brs = brsPool[len(brsPool)-len(brs.brs)-1 : len(brsPool) : len(brsPool)]
-		} else {
-			// It is unsafe appending blockRef to brsPool, since there are other items added there.
-			// So just append it to brs.brs.
-			brs.brs = append(brs.brs, blockRef{
-				partRef: partRef,
-				addr:    addr,
-			})
+		brs := m[string(metricName)]
+		if brs == nil {
+			brs = &blockRefs{}
+			brs.brs = brs.brsPrealloc[:0]
 		}
+		brs.brs = append(brs.brs, blockRef{
+			partRef: br.PartRef(),
+			addr:    addr,
+		})
 		if len(brs.brs) == 1 {
-			if cap(metricNamesBuf) >= maxFastAllocBlockSize && len(metricNamesBuf)+len(metricName) > cap(metricNamesBuf) {
-				// Allocate a new metricNamesBuf in order to avoid slow allocation of byte slice
-				// bigger than maxFastAllocBlockSize bytes at append() below.
-				metricNamesBuf = make([]byte, 0, maxFastAllocBlockSize)
-			}
-			metricNamesBufLen := len(metricNamesBuf)
-			metricNamesBuf = append(metricNamesBuf, metricName...)
-			metricNameStr := bytesutil.ToUnsafeString(metricNamesBuf[metricNamesBufLen:])
-
+			metricNameStr := string(metricName)
 			orderedMetricNames = append(orderedMetricNames, metricNameStr)
-			m[metricNameStr] = brsIdx
+			m[metricNameStr] = brs
 		}
 	}
-
 	if err := sr.Error(); err != nil {
 		putTmpBlocksFile(tbf)
 		putStorageSearch(sr)
@@ -1279,7 +1239,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	for i, metricName := range orderedMetricNames {
 		pts[i] = packedTimeseries{
 			metricName: metricName,
-			brs:        brssPool[m[metricName]].brs,
+			brs:        m[metricName].brs,
 		}
 	}
 	rss.packedTimeseries = pts
@@ -1295,25 +1255,6 @@ type blockRef struct {
 	addr    tmpBlockAddr
 }
 
-// canAppendToBlockRefPool returns true if a points to the pool and the last item in a is the last item in the pool.
-//
-// In this case it is safe appending an item to the pool and then updating the a, so it refers to the extended slice.
-//
-// True is also returned if a is nil, since in this case it is safe appending an item to the pool and pointing a
-// to the last item in the pool.
-func canAppendToBlockRefPool(pool, a []blockRef) bool {
-	if a == nil {
-		return true
-	}
-	if len(a) > len(pool) {
-		// a doesn't belong to pool
-		return false
-	}
-	shPool := (*reflect.SliceHeader)(unsafe.Pointer(&pool))
-	shA := (*reflect.SliceHeader)(unsafe.Pointer(&a))
-	return shPool.Data+uintptr(shPool.Len)*unsafe.Sizeof(blockRef{}) == shA.Data+uintptr(shA.Len)*unsafe.Sizeof(blockRef{})
-}
-
 func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]storage.TagFilter, maxMetrics int, deadline searchutils.Deadline) ([]*storage.TagFilters, error) {
 	tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
 	for _, tagFilters := range tagFilterss {
@@ -1359,8 +1300,3 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 	}
 	return dst, nil
 }
-
-// Go uses fast allocations for block sizes up to 32Kb.
-//
-// See https://github.com/golang/go/blob/704401ffa06c60e059c9e6e4048045b4ff42530a/src/runtime/malloc.go#L11
-const maxFastAllocBlockSize = 32 * 1024

app/vmselect/netstorage/tmp_blocks_file.go

@@ -136,25 +136,21 @@ func (tbf *tmpBlocksFile) Finalize() error {
 		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
 	}
 	tbf.buf = tbf.buf[:0]
-	r := fs.NewReaderAt(tbf.f)
-
+	r := fs.MustOpenReaderAt(fname)
 	// Hint the OS that the file is read almost sequentiallly.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
-
 	// Collect local stats in order to improve performance on systems with big number of CPU cores.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966
 	r.SetUseLocalStats()
-
 	tbf.r = r
-	tbf.f = nil
 	return nil
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
 	var buf []byte
-	if tbf.r == nil {
+	if tbf.f == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
 	} else {
 		bb := tmpBufPool.Get()
@@ -173,45 +169,21 @@ func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBl
 var tmpBufPool bytesutil.ByteBufferPool
 
 func (tbf *tmpBlocksFile) MustClose() {
-	if tbf.f != nil {
-		// tbf.f could be non-nil if Finalize wasn't called.
-		// In this case tbf.r must be nil.
-		if tbf.r != nil {
-			logger.Panicf("BUG: tbf.r must be nil when tbf.f!=nil")
-		}
-
-		// Try removing the file before closing it in order to prevent from flushing the in-memory data
-		// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
-		// Gracefully handle this case by attempting to remove the file after closing it.
-		fname := tbf.f.Name()
-		errRemove := os.Remove(fname)
-		if err := tbf.f.Close(); err != nil {
-			logger.Panicf("FATAL: cannot close %q: %s", fname, err)
-		}
-		if errRemove != nil {
-			if err := os.Remove(fname); err != nil {
-				logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
-			}
-		}
-		tbf.f = nil
+	if tbf.f == nil {
 		return
 	}
-
-	if tbf.r == nil {
-		// Nothing to do
-		return
+	if tbf.r != nil {
+		// tbf.r could be nil if Finalize wasn't called.
+		tbf.r.MustClose()
 	}
+	fname := tbf.f.Name()
 
-	// Try removing the file before closing it in order to prevent from flushing the in-memory data
-	// from page cache to the disk and save disk write IO. This may fail on non-posix systems such as Windows.
-	// Gracefully handle this case by attempting to remove the file after closing it.
-	fname := tbf.r.Path()
-	errRemove := os.Remove(fname)
-	tbf.r.MustClose()
-	if errRemove != nil {
-		if err := os.Remove(fname); err != nil {
-			logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
-		}
+	if err := tbf.f.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
 	}
-	tbf.r = nil
+	// We cannot remove unclosed at non-posix filesystems, like windows
+	if err := os.Remove(fname); err != nil {
+		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+	}
+	tbf.f = nil
 }

app/vmselect/prometheus/expand-with-exprs.qtpl

@@ -82,7 +82,7 @@ textarea { margin: 1em }
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -146,7 +146,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/expand-with-exprs.qtpl.go

@@ -195,7 +195,7 @@ func streamwithExprsTutorial(qw422016 *qt422016.Writer) {
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -259,7 +259,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/prometheus.go

@@ -321,9 +321,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonPara
 		firstLineSent := uint32(0)
 		writeLineFunc = func(xb *exportBlock, workerID uint) error {
 			bb := sw.getBuffer(workerID)
-			// Use atomic.LoadUint32() in front of atomic.CompareAndSwapUint32() in order to avoid slow inter-CPU synchronization
-			// in fast path after the first line has been already sent.
-			if atomic.LoadUint32(&firstLineOnce) == 0 && atomic.CompareAndSwapUint32(&firstLineOnce, 0, 1) {
+			if atomic.CompareAndSwapUint32(&firstLineOnce, 0, 1) {
 				// Send the first line to sw.bw
 				WriteExportPromAPILine(bb, xb)
 				_, err := sw.bw.Write(bb.B)
@@ -499,10 +497,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 	if err != nil {
 		return err
 	}
-	// Do not limit the number of unique time series, which could be scanned
-	// during the search for matching label values, since users expect this API
-	// must always work.
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, -1)
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
 	labelValues, err := netstorage.LabelValues(qt, labelName, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
@@ -599,10 +594,7 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	if err != nil {
 		return err
 	}
-	// Do not limit the number of unique time series, which could be scanned
-	// during the search for matching label values, since users expect this API
-	// must always work.
-	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, -1)
+	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxUniqueTimeseries)
 	labels, err := netstorage.LabelNames(qt, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain labels: %w", err)
@@ -726,6 +718,9 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		start -= offset
 		end := start
 		start = end - window
+		if start < 0 {
+			start = 0
+		}
 		// Do not include data point with a timestamp matching the lower boundary of the window as Prometheus does.
 		start++
 		if end < start {

app/vmselect/promql/aggr.go

@@ -111,48 +111,41 @@ func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) [
 	modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
 	m := aggrPrepareSeries(argOrig, modifier, maxSeries, keepOriginal)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tssl := range m {
-		rv := afe(tssl.tss, modifier)
+	for _, tss := range m {
+		rv := afe(tss, modifier)
 		rvs = append(rvs, rv...)
 	}
 	return rvs, nil
 }
 
-func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string]*tssList {
+func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string][]*timeseries {
 	// Remove empty time series, e.g. series with all NaN samples,
 	// since such series are ignored by aggregate functions.
 	argOrig = removeEmptySeries(argOrig)
 	arg := copyTimeseriesMetricNames(argOrig, keepOriginal)
 
 	// Perform grouping.
-	m := make(map[string]*tssList)
+	m := make(map[string][]*timeseries)
 	bb := bbPool.Get()
 	for i, ts := range arg {
 		removeGroupTags(&ts.MetricName, modifier)
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		k := bb.B
+		k := string(bb.B)
 		if keepOriginal {
 			ts = argOrig[i]
 		}
-		tssl := m[string(k)]
-		if tssl == nil {
-			if maxSeries > 0 && len(m) >= maxSeries {
-				// We already reached time series limit after grouping. Skip other time series.
-				continue
-			}
-			tssl = &tssList{}
-			m[string(k)] = tssl
+		tss := m[k]
+		if tss == nil && maxSeries > 0 && len(m) >= maxSeries {
+			// We already reached time series limit after grouping. Skip other time series.
+			continue
 		}
-		tssl.tss = append(tssl.tss, ts)
+		tss = append(tss, ts)
+		m[k] = tss
 	}
 	bbPool.Put(bb)
 	return m
 }
 
-type tssList struct {
-	tss []*timeseries
-}
-
 func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	tss, err := getAggrTimeseries(afa.args)
 	if err != nil {
@@ -633,8 +626,8 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 
 	m := aggrPrepareSeries(args[1], &afa.ae.Modifier, afa.ae.Limit, false)
 	rvs := make([]*timeseries, 0, len(m))
-	for _, tssl := range m {
-		rv, err := afe(tssl.tss, modifier)
+	for _, tss := range m {
+		rv, err := afe(tss, modifier)
 		if err != nil {
 			return nil, err
 		}

app/vmselect/promql/aggr_incremental.go

@@ -111,8 +111,8 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 	removeGroupTags(&ts.MetricName, &iafc.ae.Modifier)
 	bb := bbPool.Get()
 	bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-	k := bb.B
-	iac := m[string(k)]
+	k := string(bb.B)
+	iac := m[k]
 	if iac == nil {
 		if iafc.ae.Limit > 0 && len(m) >= iafc.ae.Limit {
 			// Skip this time series, since the limit on the number of output time series has been already reached.
@@ -131,7 +131,7 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 			ts:     tsAggr,
 			values: make([]float64, len(ts.Values)),
 		}
-		m[string(k)] = iac
+		m[k] = iac
 	}
 	bbPool.Put(bb)
 	iafc.callbacks.updateAggrFunc(iac, ts.Values)

app/vmselect/promql/binary_op.go

@@ -257,7 +257,7 @@ func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft,
 			var tsCopy timeseries
 			tsCopy.CopyFromShallowTimestamps(tsLeft)
 			tsCopy.MetricName.SetTags(joinTags, joinPrefix, skipTags, &tsRight.MetricName)
-			bb.B = marshalMetricNameSorted(bb.B[:0], &tsCopy.MetricName)
+			bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
 			pair, ok := m[string(bb.B)]
 			if !ok {
 				m[string(bb.B)] = &tsPair{
@@ -522,9 +522,7 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 		mn := storage.GetMetricName()
 		for _, ts := range arg {
 			mn.CopyFrom(&ts.MetricName)
-			if !be.KeepMetricNames {
-				mn.ResetMetricGroup()
-			}
+			mn.ResetMetricGroup()
 			switch groupOp {
 			case "on":
 				mn.RemoveTagsOn(groupTags)
@@ -533,7 +531,7 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 			default:
 				logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
 			}
-			bb.B = marshalMetricNameSorted(bb.B[:0], mn)
+			bb.B = marshalMetricTagsSorted(bb.B[:0], mn)
 			k := string(bb.B)
 			m[k] = append(m[k], ts)
 		}

app/vmselect/promql/eval.go

@@ -29,8 +29,7 @@ import (
 )
 
 var (
-	disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful when ingesting historical data. "+
-		"See https://docs.victoriametrics.com/#backfilling . See also -search.resetRollupResultCacheOnStartup")
+	disableCache                   = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling")
 	maxPointsSubqueryPerTimeseries = flag.Int("search.maxPointsSubqueryPerTimeseries", 100e3, "The maximum number of points per series, which can be generated by subquery. "+
 		"See https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3")
 	maxMemoryPerQuery = flagutil.NewBytes("search.maxMemoryPerQuery", 0, "The maximum amounts of memory a single query may consume. "+
@@ -111,11 +110,6 @@ type EvalConfig struct {
 	End   int64
 	Step  int64
 
-	// RealStart contains the original start of the interval when executed in subqueries (because Start can be changed  for subqueries)
-	RealStart int64
-	// RealEnd contains the original end of the interval when executed in subqueries (because End can be changed for subqueries)
-	RealEnd int64
-
 	// MaxSeries is the maximum number of time series, which can be scanned by the query.
 	// Zero means 'no limit'
 	MaxSeries int
@@ -157,17 +151,7 @@ type EvalConfig struct {
 func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	var ec EvalConfig
 	ec.Start = src.Start
-	if ec.RealStart > 0 {
-		ec.RealStart = src.RealStart
-	} else {
-		ec.RealStart = src.Start
-	}
 	ec.End = src.End
-	if ec.RealEnd > 0 {
-		ec.RealEnd = src.RealEnd
-	} else {
-		ec.RealEnd = src.End
-	}
 	ec.Step = src.Step
 	ec.MaxSeries = src.MaxSeries
 	ec.MaxPointsPerSeries = src.MaxPointsPerSeries
@@ -1281,7 +1265,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
+		tss, ok := getMaxInstantValues(qtChild, tssCached, tssStart, tssEnd)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains bigger values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1343,7 +1327,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
+		tss, ok := getMinInstantValues(qtChild, tssCached, tssStart, tssEnd)
 		if !ok {
 			qtChild.Printf("cannot apply instant rollup optimization, since tssEnd contains smaller values than tssCached")
 			deleteCachedSeries(qtChild)
@@ -1408,7 +1392,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			return evalAt(qtChild, timestamp, window)
 		}
 		// Calculate the result
-		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd, timestamp)
+		tss := getSumInstantValues(qtChild, tssCached, tssStart, tssEnd)
 		return tss, nil
 	default:
 		qt.Printf("instant rollup optimization isn't implemented for %s()", funcName)
@@ -1435,8 +1419,8 @@ func hasDuplicateSeries(tss []*timeseries) bool {
 	return false
 }
 
-func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
+func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the minimum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
 	defer qt.Done()
 
 	getMin := func(a, b float64) float64 {
@@ -1445,13 +1429,13 @@ func getMinInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMin)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMin)
 	qt.Printf("resulting series=%d; ok=%v", len(tss), ok)
 	return tss, ok
 }
 
-func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) ([]*timeseries, bool) {
-	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
+func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) ([]*timeseries, bool) {
+	qt = qt.NewChild("calculate the maximum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
 	defer qt.Done()
 
 	getMax := func(a, b float64) float64 {
@@ -1460,12 +1444,12 @@ func getMaxInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 		}
 		return b
 	}
-	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, timestamp, getMax)
+	tss, ok := getMinMaxInstantValues(tssCached, tssStart, tssEnd, getMax)
 	qt.Printf("resulting series=%d", len(tss))
 	return tss, ok
 }
 
-func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, timestamp int64, f func(a, b float64) float64) ([]*timeseries, bool) {
+func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, f func(a, b float64) float64) ([]*timeseries, bool) {
 	assertInstantValues(tssCached)
 	assertInstantValues(tssStart)
 	assertInstantValues(tssEnd)
@@ -1516,16 +1500,12 @@ func getMinMaxInstantValues(tssCached, tssStart, tssEnd []*timeseries, timestamp
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
-
-	setInstantTimestamp(rvs, timestamp)
-
 	return rvs, true
 }
 
-// getSumInstantValues aggregates tssCached, tssStart, tssEnd time series
-// into a new time series with value = tssCached + tssStart - tssEnd
-func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries, timestamp int64) []*timeseries {
-	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d, timestamp=%d", len(tssCached), len(tssStart), len(tssEnd), timestamp)
+// getSumInstantValues calculates tssCached + tssStart - tssEnd
+func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*timeseries) []*timeseries {
+	qt = qt.NewChild("calculate the sum for instant values across series; cached=%d, start=%d, end=%d", len(tssCached), len(tssStart), len(tssEnd))
 	defer qt.Done()
 
 	assertInstantValues(tssCached)
@@ -1570,19 +1550,10 @@ func getSumInstantValues(qt *querytracer.Tracer, tssCached, tssStart, tssEnd []*
 	for _, ts := range m {
 		rvs = append(rvs, ts)
 	}
-
-	setInstantTimestamp(rvs, timestamp)
-
 	qt.Printf("resulting series=%d", len(rvs))
 	return rvs
 }
 
-func setInstantTimestamp(tss []*timeseries, timestamp int64) {
-	for _, ts := range tss {
-		ts.Timestamps[0] = timestamp
-	}
-}
-
 func assertInstantValues(tss []*timeseries) {
 	for _, ts := range tss {
 		if len(ts.Values) != 1 {
@@ -1710,6 +1681,9 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	} else {
 		minTimestamp -= ec.Step
 	}
+	if minTimestamp < 0 {
+		minTimestamp = 0
+	}
 	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss, ec.MaxSeries)
 	rss, err := netstorage.ProcessSearchQuery(qt, sq, ec.Deadline)
 	if err != nil {

app/vmselect/promql/eval_test.go

@@ -1,7 +1,6 @@
 package promql
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -96,77 +95,3 @@ func TestQueryStats_addSeriesFetched(t *testing.T) {
 		t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched)
 	}
 }
-
-func TestGetSumInstantValues(t *testing.T) {
-	f := func(cached, start, end []*timeseries, timestamp int64, expectedResult []*timeseries) {
-		t.Helper()
-
-		result := getSumInstantValues(nil, cached, start, end, timestamp)
-		if !reflect.DeepEqual(result, expectedResult) {
-			t.Errorf("unexpected result; got\n%v\nwant\n%v", result, expectedResult)
-		}
-	}
-	ts := func(name string, timestamp int64, value float64) *timeseries {
-		return &timeseries{
-			MetricName: storage.MetricName{
-				MetricGroup: []byte(name),
-			},
-			Timestamps: []int64{timestamp},
-			Values:     []float64{value},
-		}
-	}
-
-	// start - end + cached = 1
-	f(
-		nil,
-		[]*timeseries{ts("foo", 42, 1)},
-		nil,
-		100,
-		[]*timeseries{ts("foo", 100, 1)},
-	)
-
-	// start - end + cached = 0
-	f(
-		nil,
-		[]*timeseries{ts("foo", 100, 1)},
-		[]*timeseries{ts("foo", 10, 1)},
-		100,
-		[]*timeseries{ts("foo", 100, 0)},
-	)
-
-	// start - end + cached = 2
-	f(
-		[]*timeseries{ts("foo", 10, 1)},
-		[]*timeseries{ts("foo", 100, 1)},
-		nil,
-		100,
-		[]*timeseries{ts("foo", 100, 2)},
-	)
-
-	// start - end + cached = 1
-	f(
-		[]*timeseries{ts("foo", 50, 1)},
-		[]*timeseries{ts("foo", 100, 1)},
-		[]*timeseries{ts("foo", 10, 1)},
-		100,
-		[]*timeseries{ts("foo", 100, 1)},
-	)
-
-	// start - end + cached = 0
-	f(
-		[]*timeseries{ts("foo", 50, 1)},
-		nil,
-		[]*timeseries{ts("foo", 10, 1)},
-		100,
-		[]*timeseries{ts("foo", 100, 0)},
-	)
-
-	// start - end + cached = 1
-	f(
-		[]*timeseries{ts("foo", 50, 1)},
-		nil,
-		nil,
-		100,
-		[]*timeseries{ts("foo", 100, 1)},
-	)
-}

app/vmselect/promql/exec_test.go

@@ -3706,7 +3706,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `(
 		  (label_set(time(), "t1", "v1", "__name__", "q1") or label_set(10, "t2", "v2", "__name__", "q2"))
 		    +
-		  (label_set(100, "t1", "v1", "__name__", "q1") or label_set(time(), "t2", "v3"))
+		  (label_set(100, "t1", "v1", "__name__", "q3") or label_set(time(), "t2", "v3"))
 		) keep_metric_names`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -6108,39 +6108,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run(`sum_gt_over_time`, func(t *testing.T) {
-		t.Parallel()
-		q := `round(sum_gt_over_time(rand(0)[200s:10s], 0.7), 0.1)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{5.9, 5.2, 8.5, 5.1, 4.9, 4.5},
-			Timestamps: timestampsExpected,
-		}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`sum_le_over_time`, func(t *testing.T) {
-		t.Parallel()
-		q := `round(sum_le_over_time(rand(0)[200s:10s], 0.7), 0.1)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{4.2, 4.9, 3.2, 5.8, 4.1, 5.3},
-			Timestamps: timestampsExpected,
-		}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`sum_eq_over_time`, func(t *testing.T) {
-		t.Parallel()
-		q := `round(sum_eq_over_time(rand(0)[200s:10s], 0.7), 0.1)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
 	t.Run(`increases_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `increases_over_time(rand(0)[200s:10s])`
@@ -8083,10 +8050,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate()`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2200-time())[600s])`
+		q := `rollup_rate((2000-time())[600s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -8095,7 +8062,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{7, 6, 5, 4, 3, 2},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -8104,7 +8071,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{4, 3, 2, 1, 0, -1},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
@@ -8116,10 +8083,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "max")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2200-time())[600s], "max")`
+		q := `rollup_rate((2000-time())[600s], "max")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{7, 6, 5, 4, 3, 2},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -8127,10 +8094,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "avg")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2200-time())[600s], "avg")`
+		q := `rollup_rate((2000-time())[600s], "avg")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}

app/vmselect/promql/rollup.go

@@ -79,15 +79,12 @@ var rollupFuncs = map[string]newRollupFunc{
 	"rollup_rate":             newRollupFuncOneOrTwoArgs(rollupFake), // + rollupFuncsRemoveCounterResets
 	"rollup_scrape_interval":  newRollupFuncOneOrTwoArgs(rollupFake),
 	"scrape_interval":         newRollupFuncOneArg(rollupScrapeInterval),
-	"share_eq_over_time":      newRollupShareEQ,
 	"share_gt_over_time":      newRollupShareGT,
 	"share_le_over_time":      newRollupShareLE,
+	"share_eq_over_time":      newRollupShareEQ,
 	"stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples),
 	"stddev_over_time":        newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":        newRollupFuncOneArg(rollupStdvar),
-	"sum_eq_over_time":        newRollupSumEQ,
-	"sum_gt_over_time":        newRollupSumGT,
-	"sum_le_over_time":        newRollupSumLE,
 	"sum_over_time":           newRollupFuncOneArg(rollupSum),
 	"sum2_over_time":          newRollupFuncOneArg(rollupSum2),
 	"tfirst_over_time":        newRollupFuncOneArg(rollupTfirst),
@@ -862,11 +859,6 @@ func removeCounterResets(values []float64) {
 		}
 		prevValue = v
 		values[i] = v + correction
-		// Check again, there could be precision error in float operations,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571
-		if i > 0 && values[i] < values[i-1] {
-			values[i] = values[i-1]
-		}
 	}
 }
 
@@ -1092,89 +1084,59 @@ func newRollupDurationOverTime(args []interface{}) (rollupFunc, error) {
 }
 
 func newRollupShareLE(args []interface{}) (rollupFunc, error) {
-	return newRollupAvgFilter(args, countFilterLE)
+	return newRollupShareFilter(args, countFilterLE)
 }
 
-func countFilterLE(values []float64, le float64) float64 {
+func countFilterLE(values []float64, le float64) int {
 	n := 0
 	for _, v := range values {
 		if v <= le {
 			n++
 		}
 	}
-	return float64(n)
+	return n
 }
 
 func newRollupShareGT(args []interface{}) (rollupFunc, error) {
-	return newRollupAvgFilter(args, countFilterGT)
+	return newRollupShareFilter(args, countFilterGT)
 }
 
-func countFilterGT(values []float64, gt float64) float64 {
+func countFilterGT(values []float64, gt float64) int {
 	n := 0
 	for _, v := range values {
 		if v > gt {
 			n++
 		}
 	}
-	return float64(n)
+	return n
 }
 
 func newRollupShareEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupAvgFilter(args, countFilterEQ)
+	return newRollupShareFilter(args, countFilterEQ)
 }
 
-func sumFilterEQ(values []float64, eq float64) float64 {
-	var sum float64
-	for _, v := range values {
-		if v == eq {
-			sum += v
-		}
-	}
-	return sum
-}
-
-func sumFilterLE(values []float64, le float64) float64 {
-	var sum float64
-	for _, v := range values {
-		if v <= le {
-			sum += v
-		}
-	}
-	return sum
-}
-
-func sumFilterGT(values []float64, gt float64) float64 {
-	var sum float64
-	for _, v := range values {
-		if v > gt {
-			sum += v
-		}
-	}
-	return sum
-}
-
-func countFilterEQ(values []float64, eq float64) float64 {
+func countFilterEQ(values []float64, eq float64) int {
 	n := 0
 	for _, v := range values {
 		if v == eq {
 			n++
 		}
 	}
-	return float64(n)
+	return n
 }
 
-func countFilterNE(values []float64, ne float64) float64 {
+func countFilterNE(values []float64, ne float64) int {
 	n := 0
 	for _, v := range values {
 		if v != ne {
 			n++
 		}
 	}
-	return float64(n)
+	return n
 }
 
-func newRollupAvgFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
-	rf, err := newRollupFilter(args, f)
+func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
+	rf, err := newRollupCountFilter(args, countFilter)
 	if err != nil {
 		return nil, err
 	}
@@ -1184,35 +1146,23 @@ func newRollupAvgFilter(args []interface{}, f func(values []float64, limit float
 	}, nil
 }
 
-func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, countFilterEQ)
-}
-
 func newRollupCountLE(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, countFilterLE)
+	return newRollupCountFilter(args, countFilterLE)
 }
 
 func newRollupCountGT(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, countFilterGT)
+	return newRollupCountFilter(args, countFilterGT)
+}
+
+func newRollupCountEQ(args []interface{}) (rollupFunc, error) {
+	return newRollupCountFilter(args, countFilterEQ)
 }
 
 func newRollupCountNE(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, countFilterNE)
+	return newRollupCountFilter(args, countFilterNE)
 }
 
-func newRollupSumEQ(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, sumFilterEQ)
-}
-
-func newRollupSumLE(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, sumFilterLE)
-}
-
-func newRollupSumGT(args []interface{}) (rollupFunc, error) {
-	return newRollupFilter(args, sumFilterGT)
-}
-
-func newRollupFilter(args []interface{}, f func(values []float64, limit float64) float64) (rollupFunc, error) {
+func newRollupCountFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 2); err != nil {
 		return nil, err
 	}
@@ -1228,7 +1178,7 @@ func newRollupFilter(args []interface{}, f func(values []float64, limit float64)
 			return nan
 		}
 		limit := limits[rfa.idx]
-		return f(values, limit)
+		return float64(countFilter(values, limit))
 	}
 	return rf, nil
 }
@@ -1961,10 +1911,6 @@ func rollupChangesPrometheus(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values[1:] {
 		if v != prevValue {
-			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
-				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
-				continue
-			}
 			n++
 			prevValue = v
 		}
@@ -1988,10 +1934,6 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 	}
 	for _, v := range values {
 		if v != prevValue {
-			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
-				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
-				continue
-			}
 			n++
 			prevValue = v
 		}
@@ -2020,10 +1962,6 @@ func rollupIncreases(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v > prevValue {
-			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
-				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
-				continue
-			}
 			n++
 		}
 		prevValue = v
@@ -2055,10 +1993,6 @@ func rollupResets(rfa *rollupFuncArg) float64 {
 	n := 0
 	for _, v := range values {
 		if v < prevValue {
-			if math.Abs(v-prevValue) < 1e-12*math.Abs(v) {
-				// This may be precision error. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767#issuecomment-1650932203
-				continue
-			}
 			n++
 		}
 		prevValue = v

app/vmselect/promql/rollup_result_cache.go

@@ -30,8 +30,6 @@ var (
 		"due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset")
 	disableAutoCacheReset = flag.Bool("search.disableAutoCacheReset", false, "Whether to disable automatic response cache reset if a sample with timestamp "+
 		"outside -search.cacheTimestampOffset is inserted into VictoriaMetrics")
-	resetRollupResultCacheOnStartup = flag.Bool("search.resetRollupResultCacheOnStartup", false, "Whether to reset rollup result cache on startup. "+
-		"See https://docs.victoriametrics.com/#rollup-result-cache . See also -search.disableCache")
 )
 
 // ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
@@ -45,11 +43,6 @@ func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
 		rollupResultResetMetricRowSample.Store(&storage.MetricRow{})
 		go checkRollupResultCacheReset()
 	})
-	if atomic.LoadUint32(&needRollupResultCacheReset) != 0 {
-		// The cache has been already instructed to reset.
-		return
-	}
-
 	minTimestamp := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds() + checkRollupResultCacheResetInterval.Milliseconds()
 	needCacheReset := false
 	for i := range mrs {
@@ -119,12 +112,7 @@ func InitRollupResultCache(cachePath string) {
 	cacheSize := getRollupResultCacheSize()
 	var c *workingsetcache.Cache
 	if len(rollupResultCachePath) > 0 {
-		if *resetRollupResultCacheOnStartup {
-			logger.Infof("removing rollupResult cache at %q becasue -search.resetRollupResultCacheOnStartup command-line flag is set", rollupResultCachePath)
-			fs.MustRemoveAll(rollupResultCachePath)
-		} else {
-			logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
-		}
+		logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
 		c = workingsetcache.Load(rollupResultCachePath, cacheSize)
 		mustLoadRollupResultCacheKeyPrefix(rollupResultCachePath)
 	} else {

app/vmselect/promql/rollup_test.go

@@ -125,7 +125,7 @@ func TestRemoveCounterResets(t *testing.T) {
 	// removeCounterResets doesn't expect negative values, so it doesn't work properly with them.
 	values = []float64{-100, -200, -300, -400}
 	removeCounterResets(values)
-	valuesExpected = []float64{-100, -100, -100, -100}
+	valuesExpected = []float64{-100, -300, -600, -1000}
 	timestampsExpected := []int64{0, 1, 2, 3}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
@@ -136,17 +136,6 @@ func TestRemoveCounterResets(t *testing.T) {
 	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
-
-	// verify results always increase monotonically with possible float operations precision error
-	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
-	removeCounterResets(values)
-	var prev float64
-	for i, v := range values {
-		if v < prev {
-			t.Fatalf("error: unexpected value keep getting bigger %d; cur %v; pre %v\n", i, v, prev)
-		}
-		prev = v
-	}
 }
 
 func TestDeltaValues(t *testing.T) {
@@ -407,75 +396,6 @@ func TestRollupCountNEOverTime(t *testing.T) {
 	f(12, 11)
 }
 
-func TestRollupSumLEOverTime(t *testing.T) {
-	f := func(le, vExpected float64) {
-		t.Helper()
-		les := []*timeseries{{
-			Values:     []float64{le},
-			Timestamps: []int64{123},
-		}}
-		var me metricsql.MetricExpr
-		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
-		testRollupFunc(t, "sum_le_over_time", args, vExpected)
-	}
-
-	f(-123, 0)
-	f(0, 0)
-	f(10, 0)
-	f(12, 12)
-	f(30, 33)
-	f(50, 289)
-	f(100, 442)
-	f(123, 565)
-	f(1000, 565)
-}
-
-func TestRollupSumGTOverTime(t *testing.T) {
-	f := func(le, vExpected float64) {
-		t.Helper()
-		les := []*timeseries{{
-			Values:     []float64{le},
-			Timestamps: []int64{123},
-		}}
-		var me metricsql.MetricExpr
-		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
-		testRollupFunc(t, "sum_gt_over_time", args, vExpected)
-	}
-
-	f(-123, 565)
-	f(0, 565)
-	f(10, 565)
-	f(12, 553)
-	f(30, 532)
-	f(50, 276)
-	f(100, 123)
-	f(123, 0)
-	f(1000, 0)
-}
-
-func TestRollupSumEQOverTime(t *testing.T) {
-	f := func(le, vExpected float64) {
-		t.Helper()
-		les := []*timeseries{{
-			Values:     []float64{le},
-			Timestamps: []int64{123},
-		}}
-		var me metricsql.MetricExpr
-		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
-		testRollupFunc(t, "sum_eq_over_time", args, vExpected)
-	}
-
-	f(-123, 0)
-	f(0, 0)
-	f(10, 0)
-	f(12, 12)
-	f(30, 0)
-	f(50, 0)
-	f(100, 0)
-	f(123, 123)
-	f(1000, 0)
-}
-
 func TestRollupQuantileOverTime(t *testing.T) {
 	f := func(phi, vExpected float64) {
 		t.Helper()

app/vmselect/promql/transform.go

@@ -1248,13 +1248,8 @@ func newTransformFuncRunning(rf func(a, b float64, idx int) float64) transformFu
 			prevValue := values[0]
 			values = values[1:]
 			for i, v := range values {
-				if tfa.ec.RealEnd > 0 && ts.Timestamps[i] > tfa.ec.RealEnd {
-					break
-				}
-				if ts.Timestamps[i] >= tfa.ec.RealStart {
-					if !math.IsNaN(v) {
-						prevValue = rf(prevValue, v, i+1)
-					}
+				if !math.IsNaN(v) {
+					prevValue = rf(prevValue, v, i+1)
 				}
 				values[i] = prevValue
 			}
@@ -1270,7 +1265,7 @@ func newTransformFuncRange(rf func(a, b float64, idx int) float64) transformFunc
 		if err != nil {
 			return nil, err
 		}
-		setLastValues(tfa, rvs)
+		setLastValues(rvs)
 		return rvs, nil
 	}
 }
@@ -1544,7 +1539,7 @@ func transformRangeQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	a.A = values
 	putFloat64s(a)
-	setLastValues(tfa, rvs)
+	setLastValues(rvs)
 	return rvs, nil
 }
 
@@ -1559,14 +1554,9 @@ func transformRangeFirst(tfa *transformFuncArg) ([]*timeseries, error) {
 		if len(values) == 0 {
 			continue
 		}
-		vFirstSet := false
 		vFirst := values[0]
 		for i, v := range values {
-			if !vFirstSet && ts.Timestamps[i] >= tfa.ec.RealStart {
-				vFirst = v
-				vFirstSet = true
-			}
-			if !vFirstSet && math.IsNaN(v) {
+			if math.IsNaN(v) {
 				continue
 			}
 			values[i] = vFirst
@@ -1581,28 +1571,17 @@ func transformRangeLast(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	rvs := args[0]
-	setLastValues(tfa, rvs)
+	setLastValues(rvs)
 	return rvs, nil
 }
 
-func setLastValues(tfa *transformFuncArg, tss []*timeseries) {
+func setLastValues(tss []*timeseries) {
 	for _, ts := range tss {
 		values := skipTrailingNaNs(ts.Values)
 		if len(values) == 0 {
 			continue
 		}
 		vLast := values[len(values)-1]
-		if tfa.ec.RealEnd > 0 {
-			for i := len(values) - 1; i >= 0; i-- {
-				if math.IsNaN(values[i]) {
-					continue
-				}
-				if ts.Timestamps[i] > tfa.ec.RealEnd {
-					continue
-				}
-				vLast = values[i]
-			}
-		}
 		for i, v := range values {
 			if math.IsNaN(v) {
 				continue
@@ -2734,17 +2713,11 @@ func transformStep(tfa *transformFuncArg) float64 {
 }
 
 func transformStart(tfa *transformFuncArg) float64 {
-	if tfa.ec.RealStart == 0 {
-		return float64(tfa.ec.Start) / 1e3
-	}
-	return float64(tfa.ec.RealStart) / 1e3
+	return float64(tfa.ec.Start) / 1e3
 }
 
 func transformEnd(tfa *transformFuncArg) float64 {
-	if tfa.ec.RealEnd == 0 {
-		return float64(tfa.ec.End) / 1e3
-	}
-	return float64(tfa.ec.RealEnd) / 1e3
+	return float64(tfa.ec.End) / 1e3
 }
 
 // copyTimeseries returns a copy of tss.

app/vmselect/vmui.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
@@ -15,8 +14,6 @@ import (
 var (
 	vmuiCustomDashboardsPath = flag.String("vmui.customDashboardsPath", "", "Optional path to vmui dashboards. "+
 		"See https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards")
-	vmuiDefaultTimezone = flag.String("vmui.defaultTimezone", "", "The default timezone to be used in vmui. "+
-		"Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local")
 )
 
 // dashboardSettings represents dashboard settings file struct.
@@ -68,16 +65,6 @@ func handleVMUICustomDashboards(w http.ResponseWriter) error {
 	return nil
 }
 
-func handleVMUITimezone(w http.ResponseWriter) error {
-	tz, err := time.LoadLocation(*vmuiDefaultTimezone)
-	if err != nil {
-		return fmt.Errorf("cannot load timezone %q: %w", *vmuiDefaultTimezone, err)
-	}
-	response := fmt.Sprintf(`{"timezone": %q}`, tz)
-	writeSuccessResponse(w, []byte(response))
-	return nil
-}
-
 func writeSuccessResponse(w http.ResponseWriter, data []byte) {
 	w.WriteHeader(http.StatusOK)
 	w.Header().Set("Content-Type", "application/json")

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.be4fee7a.css",
-    "main.js": "./static/js/main.fd9d9e16.js",
+    "main.css": "./static/css/main.fb353c1e.css",
+    "main.js": "./static/js/main.5bcddddc.js",
     "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8a01ddf56e4e6bc1ccf1.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.b64c4dbf91f4fa581621.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.be4fee7a.css",
-    "static/js/main.fd9d9e16.js"
+    "static/css/main.fb353c1e.css",
+    "static/js/main.5bcddddc.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.fd9d9e16.js"></script><link href="./static/css/main.be4fee7a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.5bcddddc.js"></script><link href="./static/css/main.fb353c1e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmselect/vmui/static/media/MetricsQL.b64c4dbf91f4fa581621.md

@@ -810,7 +810,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -819,7 +819,7 @@ This function is supported by PromQL. See also [timestamp_with_name](#timestamp_
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
@@ -828,7 +828,7 @@ See also [timestamp](#timestamp).
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +837,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +852,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +862,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 

app/vmstorage/main.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -22,14 +20,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	retentionPeriod       = flagutil.NewDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
-	snapshotAuthKey       = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages")
-	forceMergeAuthKey     = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages")
-	forceFlushAuthKey     = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages")
+	snapshotAuthKey       = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
+	forceMergeAuthKey     = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
+	forceFlushAuthKey     = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages")
 	snapshotsMaxAge       = flagutil.NewDuration("snapshotsMaxAge", "0", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
 	snapshotCreateTimeout = flag.Duration("snapshotCreateTimeout", 0, "The timeout for creating new snapshot. If set, make sure that timeout is lower than backup period")
 
@@ -38,10 +36,13 @@ var (
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
-	_ = flag.Duration("finalMergeDelay", 0, "Deprecated: this flag does nothing")
-	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing")
-	_ = flag.Int("smallMergeConcurrency", 0, "Deprecated: this flag does nothing")
-
+	finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
+		"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
+		"Zero value disables final merge")
+	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing. Please use -smallMergeConcurrency "+
+		"for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage")
+	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . "+
+		"It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries")
 	retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+
 		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
 		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
@@ -93,6 +94,8 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
 	storage.SetLogNewSeries(*logNewSeries)
+	storage.SetFinalMergeDelay(*finalMergeDelay)
+	storage.SetMergeWorkersCount(*smallMergeConcurrency)
 	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
@@ -256,7 +259,7 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey.Get(), "forceMergeAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *forceMergeAuthKey, "forceMergeAuthKey") {
 			return true
 		}
 		// Run force merge in background
@@ -274,7 +277,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if path == "/internal/force_flush" {
-		if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey.Get(), "forceFlushAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *forceFlushAuthKey, "forceFlushAuthKey") {
 			return true
 		}
 		logger.Infof("flushing storage to make pending data available for reading")
@@ -290,7 +293,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if !strings.HasPrefix(path, "/snapshot") {
 		return false
 	}
-	if !httpserver.CheckAuthFlag(w, r, snapshotAuthKey.Get(), "snapshotAuthKey") {
+	if !httpserver.CheckAuthFlag(w, r, *snapshotAuthKey, "snapshotAuthKey") {
 		return true
 	}
 	path = path[len("/snapshot"):]
@@ -397,8 +400,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverWG.Add(1)
 	go func() {
 		defer staleSnapshotsRemoverWG.Done()
-		d := timeutil.AddJitterToDuration(time.Second * 11)
-		t := time.NewTicker(d)
+		t := time.NewTicker(11 * time.Second)
 		defer t.Stop()
 		for {
 			select {
@@ -491,8 +493,11 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_composite_filter_success_conversions_total`, idbm.CompositeFilterSuccessConversions)
 	metrics.WriteCounterUint64(w, `vm_composite_filter_missing_conversions_total`, idbm.CompositeFilterMissingConversions)
 
-	// vm_assisted_merges_total name is used for backwards compatibility.
-	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryPartsLimitReachedCount)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/inmemory"}`, tm.InmemoryAssistedMerges)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="storage/small"}`, tm.SmallAssistedMerges)
+
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/inmemory"}`, idbm.InmemoryAssistedMerges)
+	metrics.WriteCounterUint64(w, `vm_assisted_merges_total{type="indexdb/file"}`, idbm.FileAssistedMerges)
 
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_total`, idbm.ItemsAdded)
 	metrics.WriteCounterUint64(w, `vm_indexdb_items_added_size_bytes_total`, idbm.ItemsAddedSizeBytes)
@@ -506,10 +511,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_parts{type="indexdb/inmemory"}`, idbm.InmemoryPartsCount)
 	metrics.WriteGaugeUint64(w, `vm_parts{type="indexdb/file"}`, idbm.FilePartsCount)
 
-	metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/inmemory"}`, tm.LastPartition.InmemoryPartsCount)
-	metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/small"}`, tm.LastPartition.SmallPartsCount)
-	metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/big"}`, tm.LastPartition.BigPartsCount)
-
 	metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/inmemory"}`, tm.InmemoryBlocksCount)
 	metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/small"}`, tm.SmallBlocksCount)
 	metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/big"}`, tm.BigBlocksCount)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.22.0 as build-web-stage
+FROM golang:1.21.6 as build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/README.md

@@ -7,7 +7,6 @@ Web UI for VictoriaMetrics
 * [Updating vmui embedded into VictoriaMetrics](#updating-vmui-embedded-into-victoriametrics)
 * [Predefined dashboards](#predefined-dashboards)
 * [App mode config options](#app-mode-config-options)
-* [Timezone configuration](#timezone-configuration)
 
 ----
 
@@ -247,39 +246,3 @@ vmui can be used to paste into other applications
 ```html
 <div id="root" data-params='{"serverURL":"http://localhost:8428","useTenantID":true,"headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
 ```
-
-----
-
-## Timezone configuration
-
-vmui's timezone setting offers flexibility in displaying time data. It can be set through a configuration flag and is adjustable within the vmui interface. This feature caters to various user preferences and time zones.
-
-### Default Timezone Setting
-
-#### Via Configuration Flag
-
-- Set the default timezone using the `--vmui.defaultTimezone` flag.
-- Accepts a valid IANA Time Zone string (e.g., `America/New_York`, `Europe/Berlin`, `Etc/GMT+3`).
-- If the flag is unset or invalid, vmui defaults to the browser's local timezone.
-
-#### User Interface Adjustments
-
-- Users can change the timezone in the vmui interface.
-- Any changed setting in the interface overrides the flag's default, persisting for the user.
-- The timezone specified in the `--vmui.defaultTimezone` flag is included in the vmui's timezone selection dropdown, aiding user choice.
-
-### Key Points
-
-- **Fallback to Browser's Local Timezone**: If the flag is not set or an invalid timezone is specified, vmui uses the local timezone of the user's browser.
-- **User Preference Priority**: User-selected timezones in vmui take precedence over the default set by the flag.
-- **Cluster Consistency**: Ensure uniform timezone settings across cluster nodes, but individual user interface selections will always override these defaults.
-
-### Examples
-
-Setting a default timezone, with user options to change:
-
-```
-./victoria-metrics --vmui.defaultTimezone="America/New_York"
-```
-
-In this scenario, if a user in Berlin accesses vmui without changing settings, it will default to their browser's local timezone (CET). If they select a different timezone in vmui, this choice will override the `"America/New_York"` setting for that user.

app/vmui/packages/vmui/src/App.tsx

@@ -14,7 +14,6 @@ import PreviewIcons from "./components/Main/Icons/PreviewIcons";
 import WithTemplate from "./pages/WithTemplate";
 import Relabel from "./pages/Relabel";
 import ActiveQueries from "./pages/ActiveQueries";
-import QueryAnalyzer from "./pages/QueryAnalyzer";
 
 const App: FC = () => {
   const [loadedTheme, setLoadedTheme] = useState(false);
@@ -50,10 +49,6 @@ const App: FC = () => {
                   path={router.trace}
                   element={<TracePage/>}
                 />
-                <Route
-                  path={router.queryAnalyzer}
-                  element={<QueryAnalyzer/>}
-                />
                 <Route
                   path={router.dashboards}
                   element={<DashboardsLayout/>}

app/vmui/packages/vmui/src/api/query-range.ts

@@ -4,4 +4,4 @@ export const getQueryRangeUrl = (server: string, query: string, period: TimePara
   `${server}/api/v1/query_range?query=${encodeURIComponent(query)}&start=${period.start}&end=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
 
 export const getQueryUrl = (server: string, query: string, period: TimeParams, nocache: boolean, queryTracing: boolean): string =>
-  `${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
+  `${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;