Merge branch 'master' into streaming-aggregation-ui

# Conflicts:
#	app/vmagent/main.go

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -8,7 +8,7 @@ body:
         Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
         to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -60,12 +60,12 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -24,9 +24,9 @@ body:
       label: Troubleshooting docs
       description: I am familiar with the following troubleshooting docs
       options:
-        - label: General - https://docs.victoriametrics.com/troubleshooting/
+        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
           required: false
-        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
-          required: false
-        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
+        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
           required: false
+        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
+          required: false

.github/pull_request_template.md

@@ -1,9 +0,0 @@
-### Describe Your Changes
-
-Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
-
-### Checklist
-
-The following checks are **mandatory**:
-
-- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.github/workflows/check-licenses.yml

@@ -25,7 +25,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
         if: ${{ matrix.language == 'go' }}
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -75,7 +75,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +86,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +100,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -102,7 +102,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
@@ -115,6 +115,6 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt

.github/workflows/sync-docs.yml

@@ -6,6 +6,9 @@ on:
     paths:
       - 'docs/**'
   workflow_dispatch: {}
+env:
+  PAGEFIND_VERSION: "1.0.4"
+  HUGO_VERSION: "latest"
 permissions:
   contents: read  # This is required for actions/checkout and to commit back image update
   deployments: write
@@ -24,6 +27,16 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
+      - uses: peaceiris/actions-hugo@v2
+        with:
+          hugo-version: ${{env.HUGO_VERSION}}
+          extended: true
+      - name: Install PageFind #install the static search engine for index build
+        uses: supplypike/setup-bin@v3
+        with:
+          uri: "https://github.com/CloudCannon/pagefind/releases/download/v${{env.PAGEFIND_VERSION}}/pagefind-v${{env.PAGEFIND_VERSION}}-x86_64-unknown-linux-musl.tar.gz"
+          name: "pagefind"
+          version: ${{env.PAGEFIND_VERSION}}
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v5
         with:
@@ -38,11 +51,13 @@ jobs:
           calculatedSha=$(git rev-parse --short ${{ github.sha }})
           echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
         working-directory: main
+
       - name: update code and commit
         run: |
           rm -rf content
           cp -r ../main/docs content
           make clean-after-copy
+          make build-search-index
           git config --global user.name "${{ steps.import-gpg.outputs.email }}"
           git config --global user.email "${{ steps.import-gpg.outputs.email }}"
           git add .

.github/workflows/wiki.yml

@@ -0,0 +1,33 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*'
+    branches:
+      - master
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: write  # for Git to git push
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          cp -r docs/* wiki
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add .
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -22,4 +22,3 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata

CONTRIBUTING.md

@@ -1 +1,16 @@
-The document has been moved [here](https://docs.victoriametrics.com/contributing/).
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.

Makefile

@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
+MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
@@ -178,8 +178,7 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64 \
-	victoria-metrics-windows-amd64
+	victoria-metrics-openbsd-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
@@ -466,7 +465,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.22
+	go mod tidy -compat=1.20
 	go mod vendor
 
 app-local:
@@ -492,7 +491,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.55.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...

SECURITY.md

@@ -2,17 +2,13 @@
 
 ## Supported Versions
 
-The following versions of VictoriaMetrics receive regular security fixes:
-
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
+| v1.93.x LTS release | :white_check_mark: |
+| v1.87.x LTS release | :white_check_mark: |
 | other releases  | :x:                |
 
-See [this page](https://victoriametrics.com/security/) for more details.
-
 ## Reporting a Vulnerability
 
 Please report any security issues to security@victoriametrics.com

app/victoria-logs/deployment/Dockerfile

@@ -1,7 +1,7 @@
 ARG base_image
 FROM $base_image
 
-EXPOSE 9428
+EXPOSE 8428
 
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG src_binary

app/victoria-logs/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -21,10 +22,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -32,21 +34,18 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
+	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -54,9 +53,9 @@ func main() {
 	logger.Infof("received signal %s", sig)
 	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-logs/multiarch/Dockerfile

@@ -6,7 +6,7 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
+EXPOSE 8428
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG TARGETARCH
 COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/Makefile

@@ -88,9 +88,6 @@ victoria-metrics-linux-ppc64le:
 victoria-metrics-linux-s390x:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-victoria-metrics-linux-loong64:
-	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 victoria-metrics-linux-386:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-metrics/main.go

@@ -26,12 +26,12 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/#deduplication")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -66,11 +66,7 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8428"}
-	}
-	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
@@ -80,7 +76,7 @@ func main() {
 
 	startSelfScraper()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -90,9 +86,9 @@ func main() {
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -123,12 +119,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"expand-with-exprs", "WITH expressions' tutorial"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
-			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -39,13 +38,11 @@ const (
 )
 
 const (
-	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
-	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
-	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
-
-	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -58,15 +55,14 @@ var (
 )
 
 type test struct {
-	Name                     string   `json:"name"`
-	Data                     []string `json:"data"`
-	InsertQuery              string   `json:"insert_query"`
-	Query                    []string `json:"query"`
-	ResultMetrics            []Metric `json:"result_metrics"`
-	ResultSeries             Series   `json:"result_series"`
-	ResultQuery              Query    `json:"result_query"`
-	Issue                    string   `json:"issue"`
-	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
+	Name          string   `json:"name"`
+	Data          []string `json:"data"`
+	InsertQuery   string   `json:"insert_query"`
+	Query         []string `json:"query"`
+	ResultMetrics []Metric `json:"result_metrics"`
+	ResultSeries  Series   `json:"result_series"`
+	ResultQuery   Query    `json:"result_query"`
+	Issue         string   `json:"issue"`
 }
 
 type Metric struct {
@@ -184,7 +180,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, false, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -230,7 +226,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -241,9 +237,8 @@ func tearDown() {
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
-	time.Sleep(500 * time.Millisecond)
 	vmstorage.Storage.DebugFlush()
-	time.Sleep(1500 * time.Millisecond)
+	time.Sleep(1 * time.Second)
 	t.Run("read", testRead)
 }
 
@@ -265,14 +260,6 @@ func testWrite(t *testing.T) {
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
-	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
-			if test.Data == nil {
-				continue
-			}
-			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
-		}
-	})
 
 	t.Run("influxdb", func(t *testing.T) {
 		for _, x := range readIn("influxdb", t, insertionTime) {
@@ -314,7 +301,7 @@ func testWrite(t *testing.T) {
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
 			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
@@ -325,12 +312,7 @@ func testRead(t *testing.T) {
 						if test.Issue != "" {
 							test.Issue = "\nRegression in " + test.Issue
 						}
-						switch {
-						case strings.HasPrefix(q, "/api/v1/export/csv"):
-							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
-							if len(data) == test.ExpectedResultLinesCount {
-								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
-							}
+						switch true {
 						case strings.HasPrefix(q, "/api/v1/export"):
 							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
 								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
@@ -369,7 +351,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	t.Helper()
 	s := newSuite(t)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -431,7 +413,6 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
-
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -444,20 +425,6 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
-func httpReadData(t *testing.T, address, query string) []byte {
-	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	s.equalInt(resp.StatusCode, 200)
-	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
-	return data
-}
-
 func checkMetricsResult(got, want []Metric) error {
 	for _, r := range append([]Metric(nil), got...) {
 		want = removeIfFoundMetrics(r, want)
@@ -530,73 +497,3 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
-
-func TestImportJSONLines(t *testing.T) {
-	f := func(labelsCount, labelLen int) {
-		t.Helper()
-
-		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
-		line := generateJSONLine(labelsCount, labelLen)
-		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
-		}
-		if resp.StatusCode != 204 {
-			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
-		}
-	}
-
-	// labels with various lengths
-	for i := 0; i < 500; i++ {
-		f(10, i*5)
-	}
-
-	// Too many labels
-	f(1000, 100)
-
-	// Too long labels
-	f(1, 100_000)
-	f(10, 100_000)
-	f(10, 10_000)
-}
-
-func generateJSONLine(labelsCount, labelLen int) string {
-	m := make(map[string]string, labelsCount)
-	m["__name__"] = generateSizedRandomString(labelLen)
-	for j := 1; j < labelsCount; j++ {
-		labelName := generateSizedRandomString(labelLen)
-		labelValue := generateSizedRandomString(labelLen)
-		m[labelName] = labelValue
-	}
-
-	type jsonLine struct {
-		Metric     map[string]string `json:"metric"`
-		Values     []float64         `json:"values"`
-		Timestamps []int64           `json:"timestamps"`
-	}
-	line := &jsonLine{
-		Metric:     m,
-		Values:     []float64{1.34},
-		Timestamps: []int64{time.Now().UnixNano() / 1e6},
-	}
-	data, err := json.Marshal(&line)
-	if err != nil {
-		panic(fmt.Errorf("cannot marshal JSON: %w", err))
-	}
-	data = append(data, '\n')
-	return string(data)
-}
-
-const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
-
-func generateSizedRandomString(size int) string {
-	dst := make([]byte, size)
-	for i := range dst {
-		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
-	}
-	return string(dst)
-}

app/victoria-metrics/self_scraper.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -50,8 +49,16 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	f := func(currentTime time.Time, sendStaleMarkers bool) {
-		currentTimestamp := currentTime.UnixNano() / 1e6
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -76,27 +83,12 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			if sendStaleMarkers {
-				mr.Value = decimal.StaleNaN
-			} else {
-				mr.Value = r.Value
-			}
+			mr.Value = r.Value
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
-	for {
-		select {
-		case <-selfScraperStopCh:
-			f(time.Now(), true)
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			f(currentTime, false)
-		}
-	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {

app/victoria-metrics/testdata/csv/basic.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export",
-  "data": [
-    "rfc3339,4,{TIME_MS}",
-    "rfc3339milli,6,{TIME_MS}",
-    "ts,8,{TIME_MS}",
-    "tsms,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
-  "query": [
-    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export with extra_labels",
-  "data": [
-    "location-1,4,{TIME_MS}",
-    "location-2,6,{TIME_MS}",
-    "location-3,8,{TIME_MS}",
-    "location-4,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
-  "query": [
-    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	processLogMessage := func(_ int64, _ []logstorage.Field) {}
+	processLogMessage := func(timestmap int64, fields []logstorage.Field) {}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))

app/vlinsert/loki/loki_json_test.go

@@ -11,7 +11,7 @@ import (
 func TestParseJSONRequestFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
 			t.Fatalf("unexpected call to parseJSONRequest callback!")
 		})
 		if err == nil {

app/vlinsert/loki/loki_json_timing_test.go

@@ -27,7 +27,7 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -29,7 +29,7 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlogsgenerator/Makefile

@@ -1,7 +0,0 @@
-# All these commands must run from repository root.
-
-vlogsgenerator:
-	APP_NAME=vlogsgenerator $(MAKE) app-local
-
-vlogsgenerator-race:
-	APP_NAME=vlogsgenerator RACE=-race $(MAKE) app-local

app/vlogsgenerator/README.md

@@ -1,156 +0,0 @@
-# vlogsgenerator
-
-Logs generator for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-## How to build vlogsgenerator?
-
-Run `make vlogsgenerator` from the repository root. This builds `bin/vlogsgenerator` binary.
-
-## How run vlogsgenerator?
-
-`vlogsgenerator` generates logs in [JSON line format](https://jsonlines.org/) suitable for the ingestion
-via [`/insert/jsonline` endpoint at VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api).
-
-By default it writes the generated logs into `stdout`. For example, the following command writes generated logs to `stdout`:
-
-```
-bin/vlogsgenerator
-```
-
-It is possible to redirect the generated logs to file. For example, the following command writes the generated logs to `logs.json` file:
-
-```
-bin/vlogsgenerator > logs.json
-```
-
-The generated logs at `logs.json` file can be inspected with the following command:
-
-```
-head logs.json | jq .
-```
-
-Below is an example output:
-
-```json
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 8 and worker 0; ip=185.69.136.129; uuid=b4fe8f1a-c93c-dea3-ba11-5b9f0509291e; u64=8996587920687045253",
-  "host": "host_8",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 8",
-  "const_1": "some value 1 8",
-  "const_2": "some value 2 8",
-  "var_0": "some value 0 12752539384823438260",
-  "dict_0": "warn",
-  "dict_1": "info",
-  "u8_0": "6",
-  "u16_0": "35202",
-  "u32_0": "1964973739",
-  "u64_0": "4810489083243239145",
-  "float_0": "1.868",
-  "ip_0": "250.34.75.125",
-  "timestamp_0": "1799-03-16T01:34:18.311Z"
-}
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 9 and worker 0; ip=164.244.254.194; uuid=7e8373b1-ce0d-1ce7-8e96-4bcab8955598; u64=13949903463741076522",
-  "host": "host_9",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 9",
-  "const_1": "some value 1 9",
-  "const_2": "some value 2 9",
-  "var_0": "some value 0 5371555382075206134",
-  "dict_0": "INFO",
-  "dict_1": "FATAL",
-  "u8_0": "219",
-  "u16_0": "31459",
-  "u32_0": "3918836777",
-  "u64_0": "6593354256620219850",
-  "float_0": "1.085",
-  "ip_0": "253.151.88.158",
-  "timestamp_0": "2042-10-05T16:42:57.082Z"
-}
-```
-
-The `run_id` field uniquely identifies every `vlogsgenerator` invocation.
-
-### How to write logs to VictoriaLogs?
-
-The generated logs can be written directly to VictoriaLogs by passing the address of [`/insert/jsonline` endpoint](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api)
-to `-addr` command-line flag. For example, the following command writes the generated logs to VictoriaLogs running at `localhost`:
-
-```
-bin/vlogsgenerator -addr=http://localhost:9428/insert/jsonline
-```
-
-### Configuration
-
-`vlogsgenerator` accepts various command-line flags, which can be used for configuring the number and the shape of the generated logs.
-These flags can be inspected by running `vlogsgenerator -help`. Below are the most interesting flags:
-
-* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to generate.
-* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
-
-The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
-
-For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields), where every logs stream contains `10_000` log entries,
-and writes them to `http://localhost:9428/insert/jsonline`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-### Churn rate
-
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
-by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
-logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
-but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -totalStreams=1_000 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-In this case the total number of generated logs equals to `-totalStreams` * `-logsPerStream` = `10_000_000`.
-
-### Benchmark tuning
-
-By default `vlogsgenerator` generates and writes logs by a single worker. This may limit the maximum data ingestion rate during benchmarks.
-The number of workers can be changed via `-workers` command-line flag. For example, the following command generates and writes logs with `16` workers:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -workers=16
-```
-
-### Output statistics
-
-Every 10 seconds `vlogsgenerator` writes statistics about the generated logs into `stderr`. The frequency of the generated statistics can be adjusted via `-statInterval` command-line flag.
-For example, the following command writes statistics every 2 seconds:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -statInterval=2s
-```

app/vlogsgenerator/main.go

@@ -1,339 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"os"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-var (
-	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
-	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
-
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields")
-	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
-		"during data generation")
-	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
-
-	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	var remoteWriteURL *url.URL
-	if *addr != "stdout" {
-		urlParsed, err := url.Parse(*addr)
-		if err != nil {
-			logger.Fatalf("cannot parse -addr=%q: %s", *addr, err)
-		}
-		qs, err := url.ParseQuery(urlParsed.RawQuery)
-		if err != nil {
-			logger.Fatalf("cannot parse query string in -addr=%q: %w", *addr, err)
-		}
-		qs.Set("_stream_fields", "host,worker_id")
-		urlParsed.RawQuery = qs.Encode()
-		remoteWriteURL = urlParsed
-	}
-
-	if start.nsec >= end.nsec {
-		logger.Fatalf("-start=%s must be smaller than -end=%s", start, end)
-	}
-	if *activeStreams <= 0 {
-		logger.Fatalf("-activeStreams must be bigger than 0; got %d", *activeStreams)
-	}
-	if *logsPerStream <= 0 {
-		logger.Fatalf("-logsPerStream must be bigger than 0; got %d", *logsPerStream)
-	}
-	if *totalStreams < *activeStreams {
-		*totalStreams = *activeStreams
-	}
-
-	cfg := &workerConfig{
-		url:           remoteWriteURL,
-		activeStreams: *activeStreams,
-		totalStreams:  *totalStreams,
-	}
-
-	// divide total and active streams among workers
-	if *workers <= 0 {
-		logger.Fatalf("-workers must be bigger than 0; got %d", *workers)
-	}
-	if *workers > *activeStreams {
-		logger.Fatalf("-workers=%d cannot exceed -activeStreams=%d", *workers, *activeStreams)
-	}
-	cfg.activeStreams /= *workers
-	cfg.totalStreams /= *workers
-
-	logger.Infof("start -workers=%d workers for ingesting -logsPerStream=%d log entries per each -totalStreams=%d (-activeStreams=%d) on a time range -start=%s, -end=%s to -addr=%s",
-		*workers, *logsPerStream, *totalStreams, *activeStreams, toRFC3339(start.nsec), toRFC3339(end.nsec), *addr)
-
-	startTime := time.Now()
-	var wg sync.WaitGroup
-	for i := 0; i < *workers; i++ {
-		wg.Add(1)
-		go func(workerID int) {
-			defer wg.Done()
-			generateAndPushLogs(cfg, workerID)
-		}(i)
-	}
-
-	go func() {
-		prevEntries := uint64(0)
-		prevBytes := uint64(0)
-		ticker := time.NewTicker(*statInterval)
-		for range ticker.C {
-			currEntries := logEntriesCount.Load()
-			deltaEntries := currEntries - prevEntries
-			rateEntries := float64(deltaEntries) / statInterval.Seconds()
-
-			currBytes := bytesGenerated.Load()
-			deltaBytes := currBytes - prevBytes
-			rateBytes := float64(deltaBytes) / statInterval.Seconds()
-			logger.Infof("generated %dK log entries (%dK total) at %.0fK entries/sec, %dMB (%dMB total) at %.0fMB/sec",
-				deltaEntries/1e3, currEntries/1e3, rateEntries/1e3, deltaBytes/1e6, currBytes/1e6, rateBytes/1e6)
-
-			prevEntries = currEntries
-			prevBytes = currBytes
-		}
-	}()
-
-	wg.Wait()
-
-	dSecs := time.Since(startTime).Seconds()
-	currEntries := logEntriesCount.Load()
-	currBytes := bytesGenerated.Load()
-	rateEntries := float64(currEntries) / dSecs
-	rateBytes := float64(currBytes) / dSecs
-	logger.Infof("ingested %dK log entries (%dMB) in %.3f seconds; avg ingestion rate: %.0fK entries/sec, %.0fMB/sec", currEntries/1e3, currBytes/1e6, dSecs, rateEntries/1e3, rateBytes/1e6)
-}
-
-var logEntriesCount atomic.Uint64
-
-var bytesGenerated atomic.Uint64
-
-type workerConfig struct {
-	url           *url.URL
-	activeStreams int
-	totalStreams  int
-}
-
-type statWriter struct {
-	w io.Writer
-}
-
-func (sw *statWriter) Write(p []byte) (int, error) {
-	bytesGenerated.Add(uint64(len(p)))
-	return sw.w.Write(p)
-}
-
-func generateAndPushLogs(cfg *workerConfig, workerID int) {
-	pr, pw := io.Pipe()
-	sw := &statWriter{
-		w: pw,
-	}
-	bw := bufio.NewWriter(sw)
-	doneCh := make(chan struct{})
-	go func() {
-		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
-		_ = bw.Flush()
-		_ = pw.Close()
-		close(doneCh)
-	}()
-
-	if cfg.url == nil {
-		_, err := io.Copy(os.Stdout, pr)
-		if err != nil {
-			logger.Fatalf("unexpected error when writing logs to stdout: %s", err)
-		}
-		return
-	}
-
-	req, err := http.NewRequest("POST", cfg.url.String(), pr)
-	if err != nil {
-		logger.Fatalf("cannot create request to %q: %s", cfg.url, err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
-	}
-	if resp.StatusCode/100 != 2 {
-		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
-	}
-
-	// Wait until all the generateLogs goroutine is finished.
-	<-doneCh
-}
-
-func generateLogs(bw *bufio.Writer, workerID, activeStreams, totalStreams int) {
-	streamLifetime := int64(float64(end.nsec-start.nsec) * (float64(activeStreams) / float64(totalStreams)))
-	streamStep := int64(float64(end.nsec-start.nsec) / float64(totalStreams-activeStreams+1))
-	step := streamLifetime / (*logsPerStream - 1)
-
-	currNsec := start.nsec
-	for currNsec < end.nsec {
-		firstStreamID := int((currNsec - start.nsec) / streamStep)
-		generateLogsAtTimestamp(bw, workerID, currNsec, firstStreamID, activeStreams)
-		currNsec += step
-	}
-}
-
-var runID = toUUID(rand.Uint64(), rand.Uint64())
-
-func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStreamID, activeStreams int) {
-	streamID := firstStreamID
-	timeStr := toRFC3339(ts)
-	for i := 0; i < activeStreams; i++ {
-		ip := toIPv4(rand.Uint32())
-		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
-			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
-		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
-		for j := 0; j < *constFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"const_%d":"some value %d %d"`, j, j, streamID)
-		}
-		for j := 0; j < *varFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"var_%d":"some value %d %d"`, j, j, rand.Uint64())
-		}
-		for j := 0; j < *dictFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"dict_%d":"%s"`, j, dictValues[rand.Intn(len(dictValues))])
-		}
-		for j := 0; j < *u8FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u8_%d":"%d"`, j, uint8(rand.Uint32()))
-		}
-		for j := 0; j < *u16FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u16_%d":"%d"`, j, uint16(rand.Uint32()))
-		}
-		for j := 0; j < *u32FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u32_%d":"%d"`, j, rand.Uint32())
-		}
-		for j := 0; j < *u64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
-		}
-		for j := 0; j < *floatFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
-		}
-		for j := 0; j < *ipFieldsPerLog; j++ {
-			ip := toIPv4(rand.Uint32())
-			fmt.Fprintf(bw, `,"ip_%d":"%s"`, j, ip)
-		}
-		for j := 0; j < *timestampFieldsPerLog; j++ {
-			timestamp := toISO8601(int64(rand.Uint64()))
-			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
-		}
-		fmt.Fprintf(bw, "}\n")
-
-		logEntriesCount.Add(1)
-		streamID++
-	}
-}
-
-var dictValues = []string{
-	"debug",
-	"info",
-	"warn",
-	"error",
-	"fatal",
-	"ERROR",
-	"FATAL",
-	"INFO",
-}
-
-func newTimeFlag(name, defaultValue, description string) *timeFlag {
-	var tf timeFlag
-	if err := tf.Set(defaultValue); err != nil {
-		logger.Panicf("invalid defaultValue=%q for flag %q: %w", defaultValue, name, err)
-	}
-	flag.Var(&tf, name, description)
-	return &tf
-}
-
-type timeFlag struct {
-	s    string
-	nsec int64
-}
-
-func (tf *timeFlag) Set(s string) error {
-	msec, err := promutils.ParseTimeMsec(s)
-	if err != nil {
-		return fmt.Errorf("cannot parse time from %q: %w", s, err)
-	}
-	tf.s = s
-	tf.nsec = msec * 1e6
-	return nil
-}
-
-func (tf *timeFlag) String() string {
-	return tf.s
-}
-
-func toRFC3339(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format(time.RFC3339Nano)
-}
-
-func toISO8601(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format("2006-01-02T15:04:05.000Z")
-}
-
-func toIPv4(n uint32) string {
-	dst := make([]byte, 0, len("255.255.255.255"))
-	dst = marshalUint64(dst, uint64(n>>24))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>16)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>8)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64(n&0xff))
-	return string(dst)
-}
-
-func toUUID(a, b uint64) string {
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", a&(1<<32-1), (a>>32)&(1<<16-1), (a >> 48), b&(1<<16-1), b>>16)
-}
-
-// marshalUint64 appends string representation of n to dst and returns the result.
-func marshalUint64(dst []byte, n uint64) []byte {
-	return strconv.AppendUint(dst, n, 10)
-}

app/vlselect/logsql/buffered_writer.go

@@ -1,47 +0,0 @@
-package logsql
-
-import (
-	"bufio"
-	"io"
-	"sync"
-)
-
-func getBufferedWriter(w io.Writer) *bufferedWriter {
-	v := bufferedWriterPool.Get()
-	if v == nil {
-		return &bufferedWriter{
-			bw: bufio.NewWriter(w),
-		}
-	}
-	bw := v.(*bufferedWriter)
-	bw.bw.Reset(w)
-	return bw
-}
-
-func putBufferedWriter(bw *bufferedWriter) {
-	bw.reset()
-	bufferedWriterPool.Put(bw)
-}
-
-var bufferedWriterPool sync.Pool
-
-type bufferedWriter struct {
-	mu sync.Mutex
-	bw *bufio.Writer
-}
-
-func (bw *bufferedWriter) reset() {
-	// nothing to do
-}
-
-func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
-	bw.mu.Lock()
-	_, _ = bw.bw.Write(p)
-	bw.mu.Unlock()
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	bw.mu.Lock()
-	_ = bw.bw.Flush()
-	bw.mu.Unlock()
-}

app/vlselect/logsql/logsql.go

@@ -1,22 +1,23 @@
 package logsql
 
 import (
-	"context"
-	"fmt"
-	"math"
 	"net/http"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-// ProcessQueryRequest handles /select/logsql/query request.
-func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+var (
+	maxSortBufferSize = flagutil.NewBytes("select.maxSortBufferSize", 1024*1024, "Query results from /select/logsql/query are automatically sorted by _time "+
+		"if their summary size doesn't exceed this value; otherwise, query results are streamed in the response without sorting; "+
+		"too big value for this flag may result in high memory usage since the sorting is performed in memory")
+)
+
+// ProcessQueryRequest handles /select/logsql/query request
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
 	if err != nil {
@@ -24,86 +25,32 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// Parse query
 	qStr := r.FormValue("query")
 	q, err := logstorage.ParseQuery(qStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse query [%s]: %s", qStr, err)
 		return
 	}
+	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
 
-	// Parse optional start and end args
-	start, okStart, err := getTimeNsec(r, "start")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	end, okEnd, err := getTimeNsec(r, "end")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if okStart || okEnd {
-		if !okStart {
-			start = math.MinInt64
-		}
-		if !okEnd {
-			end = math.MaxInt64
-		}
-		q.AddTimeFilter(start, end)
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit > 0 {
-		q.AddPipeLimit(uint64(limit))
-	}
-	q.Optimize()
-
+	sw := getSortWriter()
+	sw.Init(w, maxSortBufferSize.IntN())
 	tenantIDs := []logstorage.TenantID{tenantID}
-
-	bw := getBufferedWriter(w)
-
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
 		if len(columns) == 0 {
 			return
 		}
+		rowsCount := len(columns[0].Values)
 
 		bb := blockResultPool.Get()
-		for i := range timestamps {
-			WriteJSONRow(bb, columns, i)
+		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
+			WriteJSONRow(bb, columns, rowIdx)
 		}
-		bw.WriteIgnoreErrors(bb.B)
+		sw.MustWrite(bb.B)
 		blockResultPool.Put(bb)
-	}
-
-	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
-	err = vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-
-	bw.FlushIgnoreErrors()
-	putBufferedWriter(bw)
-
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", qStr, err)
-	}
-
+	})
+	sw.FinalFlush()
+	putSortWriter(sw)
 }
 
 var blockResultPool bytesutil.ByteBufferPool
-
-func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
-	s := r.FormValue(argName)
-	if s == "" {
-		return 0, false, nil
-	}
-	currentTimestamp := float64(time.Now().UnixNano()) / 1e9
-	secs, err := promutils.ParseTimeAt(s, currentTimestamp)
-	if err != nil {
-		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
-	}
-	return int64(secs * 1e9), true, nil
-}

app/vlselect/logsql/sort_writer.go

@@ -0,0 +1,225 @@
+package logsql
+
+import (
+	"bytes"
+	"io"
+	"sort"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func getSortWriter() *sortWriter {
+	v := sortWriterPool.Get()
+	if v == nil {
+		return &sortWriter{}
+	}
+	return v.(*sortWriter)
+}
+
+func putSortWriter(sw *sortWriter) {
+	sw.reset()
+	sortWriterPool.Put(sw)
+}
+
+var sortWriterPool sync.Pool
+
+// sortWriter expects JSON line stream to be written to it.
+//
+// It buffers the incoming data until its size reaches maxBufLen.
+// Then it streams the buffered data and all the incoming data to w.
+//
+// The FinalFlush() must be called when all the data is written.
+// If the buf isn't empty at FinalFlush() call, then the buffered data
+// is sorted by _time field.
+type sortWriter struct {
+	mu         sync.Mutex
+	w          io.Writer
+	maxBufLen  int
+	buf        []byte
+	bufFlushed bool
+
+	hasErr bool
+}
+
+func (sw *sortWriter) reset() {
+	sw.w = nil
+	sw.maxBufLen = 0
+	sw.buf = sw.buf[:0]
+	sw.bufFlushed = false
+	sw.hasErr = false
+}
+
+func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
+	sw.reset()
+
+	sw.w = w
+	sw.maxBufLen = maxBufLen
+}
+
+func (sw *sortWriter) MustWrite(p []byte) {
+	sw.mu.Lock()
+	defer sw.mu.Unlock()
+
+	if sw.hasErr {
+		return
+	}
+
+	if sw.bufFlushed {
+		if _, err := sw.w.Write(p); err != nil {
+			sw.hasErr = true
+		}
+		return
+	}
+	if len(sw.buf)+len(p) < sw.maxBufLen {
+		sw.buf = append(sw.buf, p...)
+		return
+	}
+	sw.bufFlushed = true
+	if len(sw.buf) > 0 {
+		if _, err := sw.w.Write(sw.buf); err != nil {
+			sw.hasErr = true
+			return
+		}
+		sw.buf = sw.buf[:0]
+	}
+	if _, err := sw.w.Write(p); err != nil {
+		sw.hasErr = true
+	}
+}
+
+func (sw *sortWriter) FinalFlush() {
+	if sw.hasErr || sw.bufFlushed {
+		return
+	}
+	rs := getRowsSorter()
+	rs.parseRows(sw.buf)
+	rs.sort()
+	WriteJSONRows(sw.w, rs.rows)
+	putRowsSorter(rs)
+}
+
+func getRowsSorter() *rowsSorter {
+	v := rowsSorterPool.Get()
+	if v == nil {
+		return &rowsSorter{}
+	}
+	return v.(*rowsSorter)
+}
+
+func putRowsSorter(rs *rowsSorter) {
+	rs.reset()
+	rowsSorterPool.Put(rs)
+}
+
+var rowsSorterPool sync.Pool
+
+type rowsSorter struct {
+	buf       []byte
+	fieldsBuf []logstorage.Field
+	rows      [][]logstorage.Field
+	times     []string
+}
+
+func (rs *rowsSorter) reset() {
+	rs.buf = rs.buf[:0]
+
+	fieldsBuf := rs.fieldsBuf
+	for i := range fieldsBuf {
+		fieldsBuf[i].Reset()
+	}
+	rs.fieldsBuf = fieldsBuf[:0]
+
+	rows := rs.rows
+	for i := range rows {
+		rows[i] = nil
+	}
+	rs.rows = rows[:0]
+
+	times := rs.times
+	for i := range times {
+		times[i] = ""
+	}
+	rs.times = times[:0]
+}
+
+func (rs *rowsSorter) parseRows(src []byte) {
+	rs.reset()
+
+	buf := rs.buf
+	fieldsBuf := rs.fieldsBuf
+	rows := rs.rows
+	times := rs.times
+
+	p := logjson.GetParser()
+	for len(src) > 0 {
+		var line []byte
+		n := bytes.IndexByte(src, '\n')
+		if n < 0 {
+			line = src
+			src = nil
+		} else {
+			line = src[:n]
+			src = src[n+1:]
+		}
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := p.ParseLogMessage(line); err != nil {
+			logger.Panicf("BUG: unexpected invalid JSON line: %s", err)
+		}
+
+		timeValue := ""
+		fieldsBufLen := len(fieldsBuf)
+		for _, f := range p.Fields {
+			bufLen := len(buf)
+			buf = append(buf, f.Name...)
+			name := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			bufLen = len(buf)
+			buf = append(buf, f.Value...)
+			value := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			fieldsBuf = append(fieldsBuf, logstorage.Field{
+				Name:  name,
+				Value: value,
+			})
+
+			if name == "_time" {
+				timeValue = value
+			}
+		}
+		rows = append(rows, fieldsBuf[fieldsBufLen:])
+		times = append(times, timeValue)
+	}
+	logjson.PutParser(p)
+
+	rs.buf = buf
+	rs.fieldsBuf = fieldsBuf
+	rs.rows = rows
+	rs.times = times
+}
+
+func (rs *rowsSorter) Len() int {
+	return len(rs.rows)
+}
+
+func (rs *rowsSorter) Less(i, j int) bool {
+	times := rs.times
+	return times[i] < times[j]
+}
+
+func (rs *rowsSorter) Swap(i, j int) {
+	times := rs.times
+	rows := rs.rows
+	times[i], times[j] = times[j], times[i]
+	rows[i], rows[j] = rows[j], rows[i]
+}
+
+func (rs *rowsSorter) sort() {
+	sort.Sort(rs)
+}

app/vlselect/logsql/sort_writer_test.go

@@ -0,0 +1,39 @@
+package logsql
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestSortWriter(t *testing.T) {
+	f := func(maxBufLen int, data string, expectedResult string) {
+		t.Helper()
+
+		var bb bytes.Buffer
+		sw := getSortWriter()
+		sw.Init(&bb, maxBufLen)
+
+		for _, s := range strings.Split(data, "\n") {
+			sw.MustWrite([]byte(s + "\n"))
+		}
+		sw.FinalFlush()
+		putSortWriter(sw)
+
+		result := bb.String()
+		if result != expectedResult {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, expectedResult)
+		}
+	}
+
+	f(100, "", "")
+	f(100, "{}", "{}\n")
+
+	data := `{"_time":"def","_msg":"xxx"}
+{"_time":"abc","_msg":"foo"}`
+	resultExpected := `{"_time":"abc","_msg":"foo"}
+{"_time":"def","_msg":"xxx"}
+`
+	f(100, data, resultExpected)
+	f(10, data, data+"\n")
+}

app/vlselect/main.go

@@ -101,8 +101,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
-	ctx := r.Context()
-	stopCh := ctx.Done()
+	stopCh := r.Context().Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
@@ -144,7 +143,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case path == "/logsql/query":
 		logsqlQueryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(ctx, w, r)
+		logsql.ProcessQueryRequest(w, r, stopCh)
 		return true
 	default:
 		return false

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.bc07cc78.css",
-    "main.js": "./static/js/main.8e7757ef.js",
-    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.da86c2db4f0b05e286b0.md",
+    "main.css": "./static/css/main.d1313636.css",
+    "main.js": "./static/js/main.1919fefe.js",
+    "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.bc07cc78.css",
-    "static/js/main.8e7757ef.js"
+    "static/css/main.d1313636.css",
+    "static/js/main.1919fefe.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8e7757ef.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.1919fefe.js.LICENSE.txt

@@ -4,8 +4,10 @@
 	http://jedwatson.github.io/classnames
 */
 
+/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
+
 /**
- * @remix-run/router v1.15.1
+ * @remix-run/router v1.10.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +18,7 @@
  */
 
 /**
- * React Router DOM v6.22.1
+ * React Router DOM v6.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +29,7 @@
  */
 
 /**
- * React Router v6.22.1
+ * React Router v6.17.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlstorage/main.go

@@ -1,11 +1,10 @@
 package vlstorage
 
 import (
-	"context"
 	"flag"
 	"fmt"
-	"io"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -62,16 +61,10 @@ func Init() {
 
 	var ss logstorage.StorageStats
 	strg.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
-		"smallPartSize: %d bytes; bigPartSize: %d bytes",
-		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
-		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
+	logger.Infof("successfully opened storage in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
+		time.Since(startTime).Seconds(), ss.FileParts, ss.FileBlocks, ss.FileRowsCount, ss.CompressedFileSize)
+	storageMetrics = initStorageMetrics(strg)
 
-	// register storage metrics
-	storageMetrics = metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
-	})
 	metrics.RegisterSet(storageMetrics)
 }
 
@@ -106,61 +99,117 @@ func MustAddRows(lr *logstorage.LogRows) {
 	strg.MustAddRows(lr)
 }
 
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock func(workerID uint, timestamps []int64, columns []logstorage.BlockColumn)) error {
-	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
+// RunQuery runs the given q and calls processBlock for the returned data blocks
+func RunQuery(tenantIDs []logstorage.TenantID, q *logstorage.Query, stopCh <-chan struct{}, processBlock func(columns []logstorage.BlockColumn)) {
+	strg.RunQuery(tenantIDs, q, stopCh, processBlock)
 }
 
-func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
+func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
+	ssCache := &logstorage.StorageStats{}
+	var ssCacheLock sync.Mutex
+	var lastUpdateTime time.Time
 
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
-
-	isReadOnly := uint64(0)
-	if ss.IsReadOnly {
-		isReadOnly = 1
+	m := func() *logstorage.StorageStats {
+		ssCacheLock.Lock()
+		defer ssCacheLock.Unlock()
+		if time.Since(lastUpdateTime) < time.Second {
+			return ssCache
+		}
+		var ss logstorage.StorageStats
+		strg.UpdateStats(&ss)
+		ssCache = &ss
+		lastUpdateTime = time.Now()
+		return ssCache
 	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), isReadOnly)
 
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
+	ms := metrics.NewSet()
 
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
+	ms.NewGauge(fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), func() float64 {
+		return float64(fs.MustGetFreeSpace(*storageDataPath))
+	})
+	ms.NewGauge(fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), func() float64 {
+		if m().IsReadOnly {
+			return 1
+		}
+		return 0
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/big"}`, ss.BigPartRowsCount)
+	ms.NewGauge(`vl_active_merges{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryActiveMerges)
+	})
+	ms.NewGauge(`vl_merges_total{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryMergesTotal)
+	})
+	ms.NewGauge(`vl_active_merges{type="file"}`, func() float64 {
+		return float64(m().FileActiveMerges)
+	})
+	ms.NewGauge(`vl_merges_total{type="file"}`, func() float64 {
+		return float64(m().FileMergesTotal)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/inmemory"}`, ss.InmemoryParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/small"}`, ss.SmallParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/big"}`, ss.BigParts)
+	ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryRowsCount)
+	})
+	ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
+		return float64(m().FileRowsCount)
+	})
+	ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryParts)
+	})
+	ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
+		return float64(m().FileParts)
+	})
+	ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryBlocks)
+	})
+	ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
+		return float64(m().FileBlocks)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/inmemory"}`, ss.InmemoryBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
+	ms.NewGauge(`vl_partitions`, func() float64 {
+		return float64(m().PartitionsCount)
+	})
+	ms.NewGauge(`vl_streams_created_total`, func() float64 {
+		return float64(m().StreamsCreatedTotal)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
-	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
+	ms.NewGauge(`vl_indexdb_rows`, func() float64 {
+		return float64(m().IndexdbItemsCount)
+	})
+	ms.NewGauge(`vl_indexdb_parts`, func() float64 {
+		return float64(m().IndexdbPartsCount)
+	})
+	ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
+		return float64(m().IndexdbBlocksCount)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_indexdb_rows`, ss.IndexdbItemsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_parts`, ss.IndexdbPartsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_blocks`, ss.IndexdbBlocksCount)
+	ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
+		return float64(m().IndexdbSizeBytes)
+	})
+	ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
+		dm := m()
+		return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="indexdb"}`, ss.IndexdbSizeBytes)
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="storage"}`, ss.CompressedInmemorySize+ss.CompressedSmallPartSize+ss.CompressedBigPartSize)
+	ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
+		return float64(m().CompressedInmemorySize)
+	})
+	ms.NewGauge(`vl_compressed_data_size_bytes{type="file"}`, func() float64 {
+		return float64(m().CompressedFileSize)
+	})
+	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="inmemory"}`, func() float64 {
+		return float64(m().UncompressedInmemorySize)
+	})
+	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="file"}`, func() float64 {
+		return float64(m().UncompressedFileSize)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/inmemory"}`, ss.CompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/small"}`, ss.CompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/big"}`, ss.CompressedBigPartSize)
+	ms.NewGauge(`vl_rows_dropped_total{reason="too_big_timestamp"}`, func() float64 {
+		return float64(m().RowsDroppedTooBigTimestamp)
+	})
+	ms.NewGauge(`vl_rows_dropped_total{reason="too_small_timestamp"}`, func() float64 {
+		return float64(m().RowsDroppedTooSmallTimestamp)
+	})
 
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/inmemory"}`, ss.UncompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/small"}`, ss.UncompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/big"}`, ss.UncompressedBigPartSize)
-
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
+	return ms
 }

app/vmagent/Makefile

@@ -88,9 +88,6 @@ vmagent-linux-ppc64le:
 vmagent-linux-s390x:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-vmagent-linux-loong64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/README.md

@@ -1,3 +1,3 @@
-See vmagent docs [here](https://docs.victoriametrics.com/vmagent/).
+See vmagent docs [here](https://docs.victoriametrics.com/vmagent.html).
 
-vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).
+vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).

app/vmagent/common/push_ctx.go

@@ -3,15 +3,13 @@ package common
 import (
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // PushCtx is a context used for populating WriteRequest.
 type PushCtx struct {
-	// WriteRequest contains the WriteRequest, which must be pushed later to remote storage.
-	//
-	// The actual labels and samples for the time series are stored in Labels and Samples fields.
 	WriteRequest prompbmarshal.WriteRequest
 
 	// Labels contains flat list of all the labels used in WriteRequest.
@@ -23,7 +21,13 @@ type PushCtx struct {
 
 // Reset resets ctx.
 func (ctx *PushCtx) Reset() {
-	ctx.WriteRequest.Reset()
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
 
 	promrelabel.CleanLabels(ctx.Labels)
 	ctx.Labels = ctx.Labels[:0]
@@ -35,10 +39,15 @@ func (ctx *PushCtx) Reset() {
 //
 // Call PutPushCtx when the ctx is no longer needed.
 func GetPushCtx() *PushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*PushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
 	}
-	return &PushCtx{}
 }
 
 // PutPushCtx returns ctx to the pool.
@@ -46,7 +55,12 @@ func GetPushCtx() *PushCtx {
 // ctx mustn't be used after returning to the pool.
 func PutPushCtx(ctx *PushCtx) {
 	ctx.Reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, cgroup.AvailableCPUs())

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompbmarshal.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompbmarshal.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/influx/request_handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -159,15 +160,25 @@ func (ctx *pushCtx) reset() {
 }
 
 func getPushCtx() *pushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*pushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
 	}
-	return &pushCtx{}
 }
 
 func putPushCtx(ctx *pushCtx) {
 	ctx.reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, cgroup.AvailableCPUs())

app/vmagent/main.go

@@ -8,10 +8,10 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
@@ -24,7 +24,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -37,21 +36,20 @@ import (
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
-	statsdserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -63,10 +61,6 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	statsdListenAddr = flag.String("statsdListenAddr", "", "TCP and UDP address to listen for Statsd plaintext data. Usually :8125 must be set. Doesn't work if empty. "+
-		"See also -statsdListenAddr.useProxyProtocol")
-	statsdUseProxyProtocol = flag.Bool("statsdListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -statsdListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
@@ -76,8 +70,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -86,7 +79,6 @@ var (
 var (
 	influxServer       *influxserver.Server
 	graphiteServer     *graphiteserver.Server
-	statsdServer       *statsdserver.Server
 	opentsdbServer     *opentsdbserver.Server
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
@@ -127,13 +119,8 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
@@ -144,9 +131,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
-	if len(*statsdListenAddr) > 0 {
-		statsdServer = statsdserver.MustStart(*statsdListenAddr, *statsdUseProxyProtocol, statsd.InsertHandler)
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
 		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
@@ -158,21 +142,24 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	remotewrite.StopIngestionRateLimiter()
 	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -182,9 +169,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
-	if len(*statsdListenAddr) > 0 {
-		statsdServer.MustStop()
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdbServer.MustStop()
 	}
@@ -238,7 +222,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
@@ -246,6 +230,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"-/reload", "reload configuration"},
@@ -277,7 +262,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
+	case "/prometheus/api/v1/write", "/api/v1/write":
 		if common.HandleVMProtoServerHandshake(w, r) {
 			return true
 		}
@@ -329,14 +314,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
+	case "/opentelemetry/api/v1/push":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		firehose.WriteSuccessResponse(w, r)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
@@ -384,15 +369,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -447,7 +423,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -456,7 +432,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -466,15 +442,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/stream-agg":
+		streamaggr.WriteHumanReadableState(w, r, remotewrite.GetAggregators())
+		return true
 	case "/ready":
-		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -526,7 +502,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
 	}
 	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -575,14 +551,14 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
+	case "opentelemetry/api/v1/push":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		firehose.WriteSuccessResponse(w, r)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
@@ -628,15 +604,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -693,16 +660,13 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
+	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
 
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
@@ -731,7 +695,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/vmagent/ .
+See the docs at https://docs.victoriametrics.com/vmagent.html .
 `
 	flagutil.Usage(s)
 }

app/vmagent/opentelemetry/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -28,15 +27,10 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
-		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
-			processBody = firehose.ProcessRequestBody
-		} else {
-			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-		}
+		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/remotewrite/client.go

@@ -17,26 +17,23 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
-		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
+		"is sent after temporary unavailability of the remote storage")
 	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -92,7 +89,7 @@ type client struct {
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
 
-	rl *ratelimiter.RateLimiter
+	rl rateLimiter
 
 	bytesSent       *metrics.Counter
 	blocksSent      *metrics.Counter
@@ -113,13 +110,18 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	if err != nil {
 		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
+	tlsCfg, err := authCfg.NewTLSConfig()
+	if err != nil {
+		logger.Fatalf("cannot initialize tls config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
 		DialContext:         statDial,
-		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
+		TLSClientConfig:     tlsCfg,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -137,7 +139,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		tr.Proxy = http.ProxyURL(pu)
 	}
 	hc := &http.Client{
-		Transport: authCfg.NewRoundTripper(tr),
+		Transport: tr,
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
@@ -164,7 +166,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
 		if !useVMProto {
 			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
+				"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol", sanitizedURL)
 		}
 	}
 	c.useVMProto = useVMProto
@@ -173,11 +175,12 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
+		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
@@ -391,9 +394,8 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // The function returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
 	retriesCount := 0
 
 again:
@@ -403,8 +405,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -450,8 +452,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
@@ -474,3 +476,45 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
+type rateLimiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+
+	limitReached *metrics.Counter
+}
+
+func (rl *rateLimiter) register(dataLen int, stopCh <-chan struct{}) {
+	limit := rl.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+
+	for rl.budget <= 0 {
+		if d := time.Until(rl.deadline); d > 0 {
+			rl.limitReached.Inc()
+			t := timerpool.Get(d)
+			select {
+			case <-stopCh:
+				timerpool.Put(t)
+				return
+			case <-t.C:
+				timerpool.Put(t)
+			}
+		}
+		rl.budget += limit
+		rl.deadline = time.Now().Add(time.Second)
+	}
+	rl.budget -= int64(dataLen)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -16,7 +15,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -28,7 +26,7 @@ var (
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
 		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -71,8 +69,7 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
+	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
 	for {
 		select {
@@ -82,7 +79,7 @@ func (ps *pendingSeries) periodicFlusher() {
 			ps.mu.Unlock()
 			return
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
@@ -93,7 +90,8 @@ func (ps *pendingSeries) periodicFlusher() {
 }
 
 type writeRequest struct {
-	lastFlushTime atomic.Uint64
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
+	lastFlushTime uint64
 
 	// The queue to send blocks to.
 	fq *persistentqueue.FastQueue
@@ -109,13 +107,11 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss       []prompbmarshal.TimeSeries
-	labels    []prompbmarshal.Label
-	samples   []prompbmarshal.Sample
-	exemplars []prompbmarshal.Exemplar
+	tss []prompbmarshal.TimeSeries
 
-	// buf holds labels data
-	buf []byte
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
 }
 
 func (wr *writeRequest) reset() {
@@ -123,14 +119,17 @@ func (wr *writeRequest) reset() {
 
 	wr.wr.Timeseries = nil
 
-	clear(wr.tss)
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
 	wr.tss = wr.tss[:0]
 
 	promrelabel.CleanLabels(wr.labels)
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
-	wr.exemplars = wr.exemplars[:0]
 	wr.buf = wr.buf[:0]
 }
 
@@ -152,7 +151,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
-	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
 	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
 		return false
 	}
@@ -202,7 +201,6 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
 	samplesDst := wr.samples
-	exemplarsDst := wr.exemplars
 	buf := wr.buf
 	for i := range src.Labels {
 		labelsDst = append(labelsDst, prompbmarshal.Label{})
@@ -219,61 +217,44 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	samplesDst = append(samplesDst, src.Samples...)
 	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
 
-	exemplarsDst = append(exemplarsDst, src.Exemplars...)
-	dst.Exemplars = exemplarsDst[len(exemplarsDst)-len(src.Exemplars):]
-
 	wr.samples = samplesDst
 	wr.labels = labelsDst
-	wr.exemplars = exemplarsDst
 	wr.buf = buf
 }
 
-// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
-var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
-
 func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
-	marshalConcurrencyCh <- struct{}{}
-
 	bb := writeRequestBufPool.Get()
 	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := compressBufPool.Get()
+		zb := snappyBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
-
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			zbLen := len(zb.B)
-			ok := tryPushBlock(zb.B)
-			compressBufPool.Put(zb)
-			if ok {
-				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockSizeBytes.Update(float64(zbLen))
+			if !tryPushBlock(zb.B) {
+				return false
 			}
-			return ok
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return true
 		}
-		compressBufPool.Put(zb)
+		snappyBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
 	if len(wr.Timeseries) == 1 {
 		// A single time series left. Recursively split its samples into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
-		exemplars := wr.Timeseries[0].Exemplars
-
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
 			return true
@@ -285,16 +266,11 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
-		// We do not want to send exemplars twice
-		wr.Timeseries[0].Exemplars = nil
-
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
-			wr.Timeseries[0].Exemplars = exemplars
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
-		wr.Timeseries[0].Exemplars = exemplars
 		return true
 	}
 	timeseries := wr.Timeseries
@@ -318,7 +294,5 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/pendingseries_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPushWriteRequest(t *testing.T) {
 	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{248, 1952, 17433, 180381, 1861994}
-	expectedBlockLensVM := []int{170, 575, 4748, 44936, 367096}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
 	for i, rowsCount := range rowsCounts {
 		expectedBlockLenProm := expectedBlockLensProm[i]
 		expectedBlockLenVM := expectedBlockLensVM[i]
@@ -59,20 +59,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 				Value: fmt.Sprintf("value_%d_%d", i, j),
 			})
 		}
-		exemplar := prompbmarshal.Exemplar{
-			Labels: []prompbmarshal.Label{
-				{
-					Name:  "trace_id",
-					Value: "123456",
-				},
-				{
-					Name:  "log_id",
-					Value: "987654",
-				},
-			},
-			Value:     float64(i),
-			Timestamp: 1000 * int64(i),
-		}
 		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
 			Labels: labels,
 			Samples: []prompbmarshal.Sample{
@@ -81,10 +67,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 					Timestamp: 1000 * int64(i),
 				},
 			},
-
-			Exemplars: []prompbmarshal.Exemplar{
-				exemplar,
-			},
 		})
 	}
 	return &wr

app/vmagent/remotewrite/relabel.go

@@ -19,10 +19,10 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
 		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -46,11 +46,11 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.global = global
 	}
-	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
-		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url or -remoteWrite.multitenantURL args: %d",
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.

app/vmagent/remotewrite/remotewrite.go

@@ -27,8 +27,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 )
@@ -38,30 +38,25 @@ var (
 		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
 		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
+	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. "+
+		"This flag is deprecated in favor of -enableMultitenantHandlers . See https://docs.victoriametrics.com/vmagent.html#multitenancy")
 	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
-		"https://docs.victoriametrics.com/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
+		"https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format . By default incoming data is processed via single-node insert handlers "+
 		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
-		"See https://docs.victoriametrics.com/vmagent/#multitenancy for details")
-
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details")
 	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
-		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages . "+
-		"See also -remoteWrite.shardByURLReplicas")
-	shardByURLReplicas = flag.Int("remoteWrite.shardByURLReplicas", 1, "How many copies of data to make among remote storage systems enumerated via -remoteWrite.url "+
-		"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
+		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
 	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
-	shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
-		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
-
+		"even distribution of series over the specified -remoteWrite.url systems")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -81,59 +76,51 @@ var (
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
 	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
-	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
-		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation.html")
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
-	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+
-		"for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
-
-	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
+		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
+	disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
+		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence ."+
 		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
-	rwctxs []*remoteWriteCtx
+	// rwctxsDefault contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsDefault []*remoteWriteCtx
 
-	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
+	// rwctxsMap contains dynamically populated entries when -remoteWrite.multitenantURL is specified.
+	rwctxsMap     = make(map[tenantmetrics.TenantID][]*remoteWriteCtx)
+	rwctxsMapLock sync.Mutex
+
+	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
 	defaultAuthToken = &auth.Token{}
 
 	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
 	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
 		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
 			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
-			"see https://docs.victoriametrics.com/vmagent/#disabling-on-disk-persistence"),
+			"see https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence"),
 		StatusCode: http.StatusTooManyRequests,
 	}
-
-	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
-	disableOnDiskQueueAll bool
 )
 
-// MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
+// MultitenancyEnabled returns true if -enableMultitenantHandlers or -remoteWrite.multitenantURL is specified.
 func MultitenancyEnabled() bool {
-	return *enableMultitenantHandlers
+	return *enableMultitenantHandlers || len(*remoteWriteMultitenantURLs) > 0
 }
 
 // Contains the current relabelConfigs.
@@ -153,10 +140,7 @@ func InitSecretFlags() {
 	}
 }
 
-var (
-	shardByURLLabelsMap       map[string]struct{}
-	shardByURLIgnoreLabelsMap map[string]struct{}
-)
+var shardByURLLabelsMap map[string]struct{}
 
 // Init initializes remotewrite.
 //
@@ -164,8 +148,11 @@ var (
 //
 // Stop must be called for graceful shutdown.
 func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-remoteWrite.multitenantURL` command-line flag must be set")
+	}
+	if len(*remoteWriteURLs) > 0 && len(*remoteWriteMultitenantURLs) > 0 {
+		logger.Fatalf("cannot set both `-remoteWrite.url` and `-remoteWrite.multitenantURL` command-line flags")
 	}
 	if *maxHourlySeries > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
@@ -185,21 +172,19 @@ func Init() {
 			return float64(dailySeriesLimiter.CurrentItems())
 		})
 	}
-
 	if *queues > maxQueues {
 		*queues = maxQueues
 	}
 	if *queues <= 0 {
 		*queues = 1
 	}
-
-	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
-		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
-			"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
+	if len(*shardByURLLabels) > 0 {
+		m := make(map[string]struct{}, len(*shardByURLLabels))
+		for _, label := range *shardByURLLabels {
+			m[label] = struct{}{}
+		}
+		shardByURLLabelsMap = m
 	}
-	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
-	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
-
 	initLabelsGlobal()
 
 	// Register SIGHUP handler for config reload before loadRelabelConfigs.
@@ -216,17 +201,8 @@ func Init() {
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
 	if len(*remoteWriteURLs) > 0 {
-		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
-
-	disableOnDiskQueueAll = true
-	for _, v := range *disableOnDiskQueue {
-		if !v {
-			disableOnDiskQueueAll = false
-			break
-		}
-	}
-
 	dropDanglingQueues()
 
 	// Start config reloader.
@@ -249,15 +225,18 @@ func dropDanglingQueues() {
 	if *keepDanglingQueues {
 		return
 	}
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		// Do not drop dangling queues for *remoteWriteMultitenantURLs, since it is impossible to determine
+		// unused queues for multitenant urls - they are created on demand when new sample for the given
+		// tenant is pushed to remote storage.
+		return
+	}
 	// Remove dangling persistent queues, if any.
 	// This is required for the case when the number of queues has been changed or URL have been changed.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
 	//
-	// In case if there were many persistent queues with identical *remoteWriteURLs
-	// the queue with the last index will be dropped.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxs))
-	for _, rwctx := range rwctxs {
+	existingQueues := make(map[string]struct{}, len(rwctxsDefault))
+	for _, rwctx := range rwctxsDefault {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -274,7 +253,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsDefault))
 	}
 }
 
@@ -302,6 +281,18 @@ var (
 )
 
 func reloadStreamAggrConfigs() {
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for _, rwctxs := range rwctxsMap {
+			reinitStreamAggr(rwctxs)
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		reinitStreamAggr(rwctxsDefault)
+	}
+}
+
+func reinitStreamAggr(rwctxs []*remoteWriteCtx) {
 	for _, rwctx := range rwctxs {
 		rwctx.reinitStreamAggr()
 	}
@@ -330,7 +321,7 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		}
 		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if at != nil {
-			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
 			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
 			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
 		}
@@ -345,35 +336,6 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 var configReloaderStopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
-// StartIngestionRateLimiter starts ingestion rate limiter.
-//
-// Ingestion rate limiter must be started before Init() call.
-//
-// StopIngestionRateLimiter must be called before Stop() call in order to unblock all the callers
-// to ingestion rate limiter. Otherwise deadlock may occur at Stop() call.
-func StartIngestionRateLimiter() {
-	if *maxIngestionRate <= 0 {
-		return
-	}
-	ingestionRateLimitReached := metrics.NewCounter(`vmagent_max_ingestion_rate_limit_reached_total`)
-	ingestionRateLimiterStopCh = make(chan struct{})
-	ingestionRateLimiter = ratelimiter.New(int64(*maxIngestionRate), ingestionRateLimitReached, ingestionRateLimiterStopCh)
-}
-
-// StopIngestionRateLimiter stops ingestion rate limiter.
-func StopIngestionRateLimiter() {
-	if ingestionRateLimiterStopCh == nil {
-		return
-	}
-	close(ingestionRateLimiterStopCh)
-	ingestionRateLimiterStopCh = nil
-}
-
-var (
-	ingestionRateLimiter       *ratelimiter.RateLimiter
-	ingestionRateLimiterStopCh chan struct{}
-)
-
 // Stop stops remotewrite.
 //
 // It is expected that nobody calls TryPush during and after the call to this func.
@@ -381,10 +343,18 @@ func Stop() {
 	close(configReloaderStopCh)
 	configReloaderWG.Wait()
 
-	for _, rwctx := range rwctxs {
+	for _, rwctx := range rwctxsDefault {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsDefault = nil
+
+	// There is no need in locking rwctxsMapLock here, since nobody should call TryPush during the Stop call.
+	for _, rwctxs := range rwctxsMap {
+		for _, rwctx := range rwctxs {
+			rwctx.MustStop()
+		}
+	}
+	rwctxsMap = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -394,24 +364,30 @@ func Stop() {
 	}
 }
 
-// PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
+// PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+//
+// If at is nil, then the data is pushed to the configured -remoteWrite.url.
+// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
 //
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
 }
 
-// TryPush tries sending wr to the configured remote storage systems set via -remoteWrite.url
+// TryPush tries sending wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+//
+// If at is nil, then the data is pushed to the configured -remoteWrite.url.
+// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
 //
 // TryPush can modify wr contents, so the caller must re-initialize wr before calling TryPush() after unsuccessful attempt.
 // TryPush may send partial data from wr on unsuccessful attempt, so repeated call for the same wr may send the data multiple times.
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, false)
+	return tryPush(at, wr, *dropSamplesOnOverload)
 }
 
-func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
+func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailure bool) bool {
 	tss := wr.Timeseries
 
 	if at == nil && MultitenancyEnabled() {
@@ -420,25 +396,41 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	}
 
 	var tenantRctx *relabelCtx
-	if at != nil {
+	var rwctxs []*remoteWriteCtx
+	if at == nil {
+		rwctxs = rwctxsDefault
+	} else if len(*remoteWriteMultitenantURLs) == 0 {
 		// Convert at to (vm_account_id, vm_project_id) labels.
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
+		rwctxs = rwctxsDefault
+	} else {
+		rwctxsMapLock.Lock()
+		tenantID := tenantmetrics.TenantID{
+			AccountID: at.AccountID,
+			ProjectID: at.ProjectID,
+		}
+		rwctxs = rwctxsMap[tenantID]
+		if rwctxs == nil {
+			rwctxs = newRemoteWriteCtxs(at, *remoteWriteMultitenantURLs)
+			rwctxsMap[tenantID] = rwctxs
+		}
+		rwctxsMapLock.Unlock()
 	}
+
 	rowsCount := getRowsCount(tss)
 
-	// Quick check whether writes to configured remote storage systems are blocked.
-	// This allows saving CPU time spent on relabeling and block compression
-	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
-	if disableOnDiskQueueAll {
+	if *disableOnDiskQueue {
+		// Quick check whether writes to configured remote storage systems are blocked.
+		// This allows saving CPU time spent on relabeling and block compression
+		// if some of remote storage systems cannot keep up with the data ingestion rate.
 		for _, rwctx := range rwctxs {
 			if rwctx.fq.IsWriteBlocked() {
-				rwctx.pushFailures.Inc()
-				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+				pushFailures.Inc()
+				if dropSamplesOnFailure {
 					// Just drop samples
-					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-					continue
+					samplesDropped.Add(rowsCount)
+					return true
 				}
 				return false
 			}
@@ -470,9 +462,6 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 				break
 			}
 		}
-
-		ingestionRateLimiter.Register(samplesCount)
-
 		tssBlock := tss
 		if i < len(tss) {
 			tssBlock = tss[:i]
@@ -491,14 +480,27 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock) {
+			if !*disableOnDiskQueue {
+				logger.Panicf("BUG: tryPushBlockToRemoteStorages must return true if -remoteWrite.disableOnDiskQueue isn't set")
+			}
+			pushFailures.Inc()
+			if dropSamplesOnFailure {
+				samplesDropped.Add(rowsCount)
+				return true
+			}
 			return false
 		}
 	}
 	return true
 }
 
-func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+var (
+	samplesDropped = metrics.NewCounter(`vmagent_remotewrite_samples_dropped_total`)
+	pushFailures   = metrics.NewCounter(`vmagent_remotewrite_push_failures_total`)
+)
+
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -506,129 +508,70 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 
 	if len(rwctxs) == 1 {
 		// Fast path - just push data to the configured single remote storage
-		return rwctxs[0].TryPush(tssBlock, forceDropSamplesOnFailure)
+		return rwctxs[0].TryPush(tssBlock)
 	}
 
 	// We need to push tssBlock to multiple remote storages.
 	// This is either sharding or replication depending on -remoteWrite.shardByURL command-line flag value.
-	if *shardByURL && *shardByURLReplicas < len(rwctxs) {
-		// Shard tssBlock samples among rwctxs.
-		replicas := *shardByURLReplicas
-		if replicas <= 0 {
-			replicas = 1
+	if *shardByURL {
+		// Shard the data among rwctxs
+		tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs))
+		tmpLabels := promutils.GetLabels()
+		for _, ts := range tssBlock {
+			hashLabels := ts.Labels
+			if len(shardByURLLabelsMap) > 0 {
+				hashLabels = tmpLabels.Labels[:0]
+				for _, label := range ts.Labels {
+					if _, ok := shardByURLLabelsMap[label.Name]; ok {
+						hashLabels = append(hashLabels, label)
+					}
+				}
+			}
+			h := getLabelsHash(hashLabels)
+			idx := h % uint64(len(tssByURL))
+			tssByURL[idx] = append(tssByURL[idx], ts)
 		}
-		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
+		promutils.PutLabels(tmpLabels)
+
+		// Push sharded data to remote storages in parallel in order to reduce
+		// the time needed for sending the data to multiple remote storage systems.
+		var wg sync.WaitGroup
+		wg.Add(len(rwctxs))
+		var anyPushFailed uint64
+		for i, rwctx := range rwctxs {
+			tssShard := tssByURL[i]
+			if len(tssShard) == 0 {
+				continue
+			}
+			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
+				defer wg.Done()
+				if !rwctx.TryPush(tss) {
+					atomic.StoreUint64(&anyPushFailed, 1)
+				}
+			}(rwctx, tssShard)
+		}
+		wg.Wait()
+		return atomic.LoadUint64(&anyPushFailed) == 0
 	}
 
-	// Replicate tssBlock samples among rwctxs.
-	// Push tssBlock to remote storage systems in parallel in order to reduce
+	// Replicate data among rwctxs.
+	// Push block to remote storages in parallel in order to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
-	var anyPushFailed atomic.Bool
+	var anyPushFailed uint64
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			if !rwctx.TryPush(tssBlock, forceDropSamplesOnFailure) {
-				anyPushFailed.Store(true)
+			if !rwctx.TryPush(tssBlock) {
+				atomic.StoreUint64(&anyPushFailed, 1)
 			}
 		}(rwctx)
 	}
 	wg.Wait()
-	return !anyPushFailed.Load()
+	return atomic.LoadUint64(&anyPushFailed) == 0
 }
 
-func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
-	x := getTSSShards(len(rwctxs))
-	defer putTSSShards(x)
-
-	shards := x.shards
-	tmpLabels := promutils.GetLabels()
-	for _, ts := range tssBlock {
-		hashLabels := ts.Labels
-		if len(shardByURLLabelsMap) > 0 {
-			hashLabels = tmpLabels.Labels[:0]
-			for _, label := range ts.Labels {
-				if _, ok := shardByURLLabelsMap[label.Name]; ok {
-					hashLabels = append(hashLabels, label)
-				}
-			}
-			tmpLabels.Labels = hashLabels
-		} else if len(shardByURLIgnoreLabelsMap) > 0 {
-			hashLabels = tmpLabels.Labels[:0]
-			for _, label := range ts.Labels {
-				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
-					hashLabels = append(hashLabels, label)
-				}
-			}
-			tmpLabels.Labels = hashLabels
-		}
-		h := getLabelsHash(hashLabels)
-		idx := h % uint64(len(shards))
-		i := 0
-		for {
-			shards[idx] = append(shards[idx], ts)
-			i++
-			if i >= replicas {
-				break
-			}
-			idx++
-			if idx >= uint64(len(shards)) {
-				idx = 0
-			}
-		}
-	}
-	promutils.PutLabels(tmpLabels)
-
-	// Push sharded samples to remote storage systems in parallel in order to reduce
-	// the time needed for sending the data to multiple remote storage systems.
-	var wg sync.WaitGroup
-	var anyPushFailed atomic.Bool
-	for i, rwctx := range rwctxs {
-		shard := shards[i]
-		if len(shard) == 0 {
-			continue
-		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPush(tss, forceDropSamplesOnFailure) {
-				anyPushFailed.Store(true)
-			}
-		}(rwctx, shard)
-	}
-	wg.Wait()
-	return !anyPushFailed.Load()
-}
-
-type tssShards struct {
-	shards [][]prompbmarshal.TimeSeries
-}
-
-func getTSSShards(n int) *tssShards {
-	v := tssShardsPool.Get()
-	if v == nil {
-		v = &tssShards{}
-	}
-	x := v.(*tssShards)
-	if cap(x.shards) < n {
-		x.shards = make([][]prompbmarshal.TimeSeries, n)
-	}
-	x.shards = x.shards[:n]
-	return x
-}
-
-func putTSSShards(x *tssShards) {
-	shards := x.shards
-	for i := range shards {
-		clear(shards[i])
-		shards[i] = shards[i][:0]
-	}
-	tssShardsPool.Put(x)
-}
-
-var tssShardsPool sync.Pool
-
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
 func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
 	if !*sortLabels {
@@ -722,22 +665,15 @@ type remoteWriteCtx struct {
 	fq  *persistentqueue.FastQueue
 	c   *client
 
-	sas          atomic.Pointer[streamaggr.Aggregators]
-	deduplicator *streamaggr.Deduplicator
-
-	streamAggrKeepInput   bool
-	streamAggrDropInput   bool
-	disableOnDiskQueue    bool
-	dropSamplesOnOverload bool
+	sas                 atomic.Pointer[streamaggr.Aggregators]
+	streamAggrKeepInput bool
+	streamAggrDropInput bool
 
 	pss        []*pendingSeries
-	pssNextIdx atomic.Uint64
+	pssNextIdx uint64
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
-
-	pushFailures             *metrics.Counter
-	rowsDroppedOnPushFailure *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -753,8 +689,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, *disableOnDiskQueue)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
@@ -797,28 +732,15 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
-		disableOnDiskQueue:    isPQDisabled,
-
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 
 	// Initialize sas
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-	ignoreOldSamples := streamAggrIgnoreOldSamples.GetOptionalArg(argIdx)
 	if sasFile != "" {
-		opts := &streamaggr.Options{
-			DedupInterval:        dedupInterval,
-			DropInputLabels:      *streamAggrDropInputLabels,
-			IgnoreOldSamples:     ignoreOldSamples,
-			IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
@@ -827,24 +749,17 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	} else if dedupInterval > 0 {
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels)
 	}
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
-	// sas and deduplicator must be stopped before rwctx is closed
+	// sas must be stopped before rwctx is closed
 	// because sas can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
-	if rwctx.deduplicator != nil {
-		rwctx.deduplicator.MustStop()
-		rwctx.deduplicator = nil
-	}
-
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -861,11 +776,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-// TryPush sends tss series to the configured remote write endpoint
-//
-// TryPush can be called concurrently for multiple remoteWriteCtx,
-// so it shouldn't modify tss entries.
-func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
@@ -887,7 +798,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 	rowsCount := getRowsCount(tss)
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 
-	// Apply stream aggregation or deduplication if they are configured
+	// Apply stream aggregation if any
 	sas := rwctx.sas.Load()
 	if sas != nil {
 		matchIdxs := matchIdxsPool.Get()
@@ -902,9 +813,6 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
-	} else if rwctx.deduplicator != nil {
-		rwctx.deduplicator.Push(tss)
-		tss = tss[:0]
 	}
 
 	// Try pushing the data to remote storage
@@ -917,14 +825,6 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 		putRelabelCtx(rctx)
 	}
 
-	if !ok {
-		rwctx.pushFailures.Inc()
-		if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-			rwctx.rowsDroppedOnPushFailure.Add(len(tss))
-			return true
-		}
-	}
-
 	return ok
 }
 
@@ -941,7 +841,7 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 		}
 	}
 	tail := src[len(dst):]
-	clear(tail)
+	_ = prompbmarshal.ResetTimeSeries(tail)
 	return dst
 }
 
@@ -949,13 +849,13 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.disableOnDiskQueue {
+	if !*disableOnDiskQueue {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
-	rwctx.pushFailures.Inc()
-	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
+	pushFailures.Inc()
+	if *dropSamplesOnOverload {
 		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		samplesDropped.Add(rowsCount)
 	}
 }
 
@@ -972,7 +872,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	}
 
 	pss := rwctx.pss
-	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 
 	ok := pss[idx].TryPush(tss)
 
@@ -994,12 +894,8 @@ func (rwctx *remoteWriteCtx) reinitStreamAggr() {
 
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	opts := &streamaggr.Options{
-		DedupInterval:    streamAggrDedupInterval.GetOptionalArg(rwctx.idx),
-		DropInputLabels:  *streamAggrDropInputLabels,
-		IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(rwctx.idx),
-	}
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
+	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
@@ -1036,17 +932,13 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 
 // CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
 func CheckStreamAggrConfigs() error {
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
 	for idx, sasFile := range *streamAggrConfig {
 		if sasFile == "" {
 			continue
 		}
-		opts := &streamaggr.Options{
-			DedupInterval:    streamAggrDedupInterval.GetOptionalArg(idx),
-			DropInputLabels:  *streamAggrDropInputLabels,
-			IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, opts)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
 		if err != nil {
 			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
 		}
@@ -1055,10 +947,23 @@ func CheckStreamAggrConfigs() error {
 	return nil
 }
 
-func newMapFromStrings(a []string) map[string]struct{} {
-	m := make(map[string]struct{}, len(a))
-	for _, s := range a {
-		m[s] = struct{}{}
+// GetAggregators returns aggregators for all the configured remote writes.
+func GetAggregators() map[string]*streamaggr.Aggregators {
+	var result = map[string]*streamaggr.Aggregators{}
+
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for tenant, rwctxs := range rwctxsMap {
+			for rwNum, rw := range rwctxs {
+				result[fmt.Sprintf("rw %d for tenant %v:%v", rwNum, tenant.AccountID, tenant.ProjectID)] = rw.sas.Load()
+			}
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		for rwNum, rw := range rwctxsDefault {
+			result[fmt.Sprintf("remote write %d", rwNum)] = rw.sas.Load()
+		}
 	}
-	return m
+
+	return result
 }

app/vmagent/remotewrite/remotewrite_test.go

@@ -1,215 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"math"
-	"os"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-func TestGetLabelsHash_Distribution(t *testing.T) {
-	f := func(bucketsCount int) {
-		t.Helper()
-
-		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 1_000 * bucketsCount
-		m := make([]int, bucketsCount)
-		var labels []prompbmarshal.Label
-		for i := 0; i < itemsCount; i++ {
-			labels = append(labels[:0], prompbmarshal.Label{
-				Name:  "__name__",
-				Value: fmt.Sprintf("some_name_%d", i),
-			})
-			for j := 0; j < 10; j++ {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  fmt.Sprintf("label_%d", j),
-					Value: fmt.Sprintf("value_%d_%d", i, j),
-				})
-			}
-			h := getLabelsHash(labels)
-			m[h%uint64(bucketsCount)]++
-		}
-
-		// Verify that the distribution is even
-		expectedItemsPerBucket := itemsCount / bucketsCount
-		for _, n := range m {
-			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
-			}
-		}
-	}
-
-	f(2)
-	f(3)
-	f(4)
-	f(5)
-	f(10)
-}
-
-func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
-		t.Helper()
-		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
-		if err != nil {
-			t.Fatalf("cannot load relabel configs: %s", err)
-		}
-		rcs := &relabelConfigs{
-			perURL: []*promrelabel.ParsedConfigs{
-				perURLRelabel,
-			},
-		}
-		allRelabelConfigs.Store(rcs)
-
-		pss := make([]*pendingSeries, 1)
-		pss[0] = newPendingSeries(nil, true, 0, 100)
-		rwctx := &remoteWriteCtx{
-			idx:                    0,
-			streamAggrKeepInput:    keepInput,
-			streamAggrDropInput:    dropInput,
-			pss:                    pss,
-			rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
-			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
-		}
-		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil)
-		}
-
-		if len(streamAggrConfig) > 0 {
-			f := createFile(t, []byte(streamAggrConfig))
-			sas, err := streamaggr.LoadFromFile(f.Name(), nil, nil)
-			if err != nil {
-				t.Fatalf("cannot load streamaggr configs: %s", err)
-			}
-			rwctx.sas.Store(sas)
-		}
-
-		inputTss := mustParsePromMetrics(input)
-		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
-
-		// copy inputTss to make sure it is not mutated during TryPush call
-		copy(expectedTss, inputTss)
-		rwctx.TryPush(inputTss, false)
-
-		if !reflect.DeepEqual(expectedTss, inputTss) {
-			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
-		}
-	}
-
-	f(`
-- interval: 1m
-  outputs: [sum_samples]
-- interval: 2m
-  outputs: [count_series]
-`, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, 0, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, ``, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="foo"} 20
-metric{env="dev"} 15
-metric{env="foo"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, true, false, `
-metric{env="test"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, false, true, `
-metric{env="foo"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, true, true, `
-metric{env="dev"} 10
-metric{env="test"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-}
-
-func mustParsePromMetrics(s string) []prompbmarshal.TimeSeries {
-	var rows prometheus.Rows
-	errLogger := func(s string) {
-		panic(fmt.Errorf("unexpected error when parsing Prometheus metrics: %s", s))
-	}
-	rows.UnmarshalWithErrLogger(s, errLogger)
-	var tss []prompbmarshal.TimeSeries
-	samples := make([]prompbmarshal.Sample, 0, len(rows.Rows))
-	for _, row := range rows.Rows {
-		labels := make([]prompbmarshal.Label, 0, len(row.Tags)+1)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: row.Metric,
-		})
-		for _, tag := range row.Tags {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samples = append(samples, prompbmarshal.Sample{
-			Value:     row.Value,
-			Timestamp: row.Timestamp,
-		})
-		ts := prompbmarshal.TimeSeries{
-			Labels:  labels,
-			Samples: samples[len(samples)-1:],
-		}
-		tss = append(tss, ts)
-	}
-	return tss
-}
-
-func createFile(t *testing.T, data []byte) *os.File {
-	t.Helper()
-	f, err := os.CreateTemp("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(f.Name(), data, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := f.Sync(); err != nil {
-		t.Fatal(err)
-	}
-	return f
-}

app/vmagent/remotewrite/statconn.go

@@ -3,15 +3,34 @@ package remotewrite
 import (
 	"context"
 	"net"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
 func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
-	conn, err = netutil.DialMaybeSRV(ctx, network, addr)
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
 	dialsTotal.Inc()
 	if err != nil {
 		dialErrors.Inc()
@@ -31,7 +50,7 @@ var (
 )
 
 type statConn struct {
-	closed atomic.Int32
+	closed uint64
 	net.Conn
 }
 
@@ -57,7 +76,7 @@ func (sc *statConn) Write(p []byte) (int, error) {
 
 func (sc *statConn) Close() error {
 	err := sc.Conn.Close()
-	if sc.closed.Add(1) == 1 {
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
 		conns.Dec()
 	}
 	return err

app/vmagent/statsd/request_handler.go

@@ -1,68 +0,0 @@
-package statsd
-
-import (
-	"io"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd/stream"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="statsd"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="statsd"}`)
-)
-
-// InsertHandler processes remote write for statsd plaintext protocol.
-//
-// See https://github.com/statsd/statsd/blob/master/docs/metric_types.md
-func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
-	})
-}
-
-func insertRows(at *auth.Token, rows []parser.Row) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: r.Metric,
-		})
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samples = append(samples, prompbmarshal.Sample{
-			Value:     r.Value,
-			Timestamp: r.Timestamp,
-		})
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[len(samples)-1:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	rowsPerInsert.Update(float64(len(rows)))
-	return nil
-}

app/vmalert-tool/unittest/unittest.go

@@ -25,7 +25,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -185,8 +184,7 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	var ab flagutil.ArrayBool
-	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
+	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
 		switch r.URL.Path {
 		case "/prometheus/api/v1/query":
 			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
@@ -227,7 +225,7 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
+	if err := httpserver.Stop(httpListenAddr); err != nil {
 		logger.Errorf("cannot stop the webservice: %s", err)
 	}
 	vmstorage.Stop()

app/vmalert/Makefile

@@ -68,7 +68,6 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
-	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
@@ -119,9 +118,6 @@ vmalert-linux-ppc64le:
 vmalert-linux-s390x:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-vmalert-linux-loong64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmalert-linux-386:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert/README.md

@@ -1,3 +1,3 @@
-See vmalert docs [here](https://docs.victoriametrics.com/vmalert/).
+See vmalert docs [here](https://docs.victoriametrics.com/vmalert.html).
 
 vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert.md).

app/vmalert/config/testdata/rules/kube-good.rules

@@ -1158,9 +1158,9 @@
           $labels.pod }}.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
       expr: |
-        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace)
+        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
           /
-        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace)
+        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
           > ( 25 / 100 )
       for: 15m
       labels:

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -22,7 +22,6 @@ groups:
               {{ . | first | value }}
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-          link: http://localhost:3000/d/wNf0q_kZk?viewPanel=51&from={{($activeAt.Add (parseDurationTime "1h")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime "-1h")).UnixMilli}}
       - alert: ExampleAlertAlwaysFiring
         update_entries_limit: -1
         expr: sum by(job)

app/vmalert/datasource/init.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -46,7 +45,7 @@ var (
 	oauth2TokenURL = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url")
 	oauth2Scopes   = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
-	lookBack = flag.Duration("datasource.lookback", 0, `Deprecated: please adjust "-search.latencyOffset" at datasource side `+
+	lookBack = flag.Duration("datasource.lookback", 0, `Will be deprecated soon, please adjust "-search.latencyOffset"  at datasource side `+
 		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
 		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
@@ -91,10 +90,10 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		logger.Warnf("flag `-datasource.queryTimeAlignment` is deprecated and will be removed in next releases. Please use `eval_alignment` in rule group instead.")
 	}
 	if *lookBack != 0 {
-		logger.Warnf("flag `-datasource.lookback` is deprecated and will be removed in next releases. Please adjust `-search.latencyOffset` at datasource side or specify `latency_offset` in rule group's params. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
+		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
 	}
 
-	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
@@ -133,6 +132,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
+		lookBack:         *lookBack,
 		queryStep:        *queryStep,
 		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,

app/vmalert/datasource/vm.go

@@ -35,6 +35,7 @@ type VMStorage struct {
 	authCfg          *promauth.Config
 	datasourceURL    string
 	appendTypePrefix bool
+	lookBack         time.Duration
 	queryStep        time.Duration
 	dataSourceType   datasourceType
 
@@ -62,6 +63,7 @@ func (s *VMStorage) Clone() *VMStorage {
 		authCfg:          s.authCfg,
 		datasourceURL:    s.datasourceURL,
 		appendTypePrefix: s.appendTypePrefix,
+		lookBack:         s.lookBack,
 		queryStep:        s.queryStep,
 
 		dataSourceType:     s.dataSourceType,
@@ -120,12 +122,13 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL string, authCfg *promauth.Config, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
 	return &VMStorage{
 		c:                c,
 		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
 		appendTypePrefix: appendTypePrefix,
+		lookBack:         lookBack,
 		queryStep:        queryStep,
 		dataSourceType:   datasourcePrometheus,
 		extraParams:      url.Values{},
@@ -134,11 +137,11 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, queryStep time.Durat
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := s.newQueryRequest(ctx, query, ts)
+	req, err := s.newQueryRequest(query, ts)
 	if err != nil {
 		return Result{}, nil, err
 	}
-	resp, err := s.do(req)
+	resp, err := s.do(ctx, req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -146,11 +149,11 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRequest(ctx, query, ts)
+		req, err = s.newQueryRequest(query, ts)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(req)
+		resp, err = s.do(ctx, req)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
@@ -179,11 +182,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	req, err := s.newQueryRangeRequest(ctx, query, start, end)
+	req, err := s.newQueryRangeRequest(query, start, end)
 	if err != nil {
 		return res, err
 	}
-	resp, err := s.do(req)
+	resp, err := s.do(ctx, req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -191,11 +194,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRangeRequest(ctx, query, start, end)
+		req, err = s.newQueryRangeRequest(query, start, end)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(req)
+		resp, err = s.do(ctx, req)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
@@ -207,7 +210,7 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	return res, err
 }
 
-func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
+func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
 	ru := req.URL.Redacted()
 	if *showDatasourceURL {
 		ru = req.URL.String()
@@ -215,7 +218,7 @@ func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
 	if s.debug {
 		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru)
 	}
-	resp, err := s.c.Do(req)
+	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", ru, err)
 	}
@@ -227,8 +230,8 @@ func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
-func (s *VMStorage) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) {
-	req, err := s.newRequest(ctx)
+func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*http.Request, error) {
+	req, err := s.newRequest()
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -236,8 +239,8 @@ func (s *VMStorage) newQueryRangeRequest(ctx context.Context, query string, star
 	return req, nil
 }
 
-func (s *VMStorage) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) {
-	req, err := s.newRequest(ctx)
+func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request, error) {
+	req, err := s.newRequest()
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -245,15 +248,15 @@ func (s *VMStorage) newQueryRequest(ctx context.Context, query string, ts time.T
 	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
 	case datasourceGraphite:
-		s.setGraphiteReqParams(req, query)
+		s.setGraphiteReqParams(req, query, ts)
 	default:
 		logger.Panicf("BUG: engine not found: %q", s.dataSourceType)
 	}
 	return req, nil
 }
 
-func (s *VMStorage) newRequest(ctx context.Context) (*http.Request, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.datasourceURL, nil)
+func (s *VMStorage) newRequest() (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}

app/vmalert/datasource/vm_graphite_api.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
+	"time"
 )
 
 type graphiteResponse []graphiteResponseTarget
@@ -46,13 +48,17 @@ const (
 	graphitePrefix = "/graphite"
 )
 
-func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string) {
+func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestamp time.Time) {
 	if s.appendTypePrefix {
 		r.URL.Path += graphitePrefix
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
 	from := "-5min"
+	if s.lookBack > 0 {
+		lookBack := timestamp.Add(-s.lookBack)
+		from = strconv.FormatInt(lookBack.Unix(), 10)
+	}
 	q.Set("from", from)
 	q.Set("format", "json")
 	q.Set("target", query)

app/vmalert/datasource/vm_prom_api.go

@@ -161,6 +161,9 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
+	if s.lookBack > 0 {
+		timestamp = timestamp.Add(-s.lookBack)
+	}
 	q.Set("time", timestamp.Format(time.RFC3339))
 	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older

app/vmalert/datasource/vm_test.go

@@ -71,7 +71,7 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
 		}
 	})
-	mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
+	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
 		c++
 		switch c {
 		case 8:
@@ -86,7 +86,7 @@ func TestVMInstantQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, 0, false, srv.Client())
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
 	p := datasourcePrometheus
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
@@ -225,7 +225,7 @@ func TestVMInstantQueryWithRetry(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, nil, 0, false, srv.Client())
+	s := NewVMStorage(srv.URL, nil, time.Minute, 0, false, srv.Client())
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus)})
 
 	expErr := func(err string) {
@@ -334,7 +334,7 @@ func TestVMRangeQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, *queryStep, false, srv.Client())
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, *queryStep, false, srv.Client())
 
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})
 
@@ -487,6 +487,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, "bar", p)
 			},
 		},
+		{
+			"lookback",
+			false,
+			&VMStorage{
+				lookBack: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "time": {timestamp.Add(-time.Minute).Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
 		{
 			"evaluation interval",
 			false,
@@ -499,6 +510,20 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
+		{
+			"lookback + evaluation interval",
+			false,
+			&VMStorage{
+				lookBack:           time.Minute,
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				tt := timestamp.Add(-time.Minute)
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
 		{
 			"step override",
 			false,
@@ -612,7 +637,7 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequest(ctx)
+			req, err := tc.vm.newRequest()
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -624,7 +649,7 @@ func TestRequestParams(t *testing.T) {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
 			case datasourceGraphite:
-				tc.vm.setGraphiteReqParams(req, query)
+				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
 		})
@@ -710,7 +735,7 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
+			req, err := vm.newQueryRequest("foo", time.Now())
 			if err != nil {
 				t.Fatal(err)
 			}

app/vmalert/main.go

@@ -44,7 +44,7 @@ Enterprise version of vmalert supports S3 and GCS paths to rules.
 For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
 S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
 all files with prefix rule_ in folder dir.
-See https://docs.victoriametrics.com/vmalert/#reading-rules-from-object-storage
+See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
 `)
 
 	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions `+
@@ -59,8 +59,8 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -71,7 +71,7 @@ absolute path to all .tpl files in root.
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier. By default, hostname is used as address.")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
-		`Supports templating - see https://docs.victoriametrics.com/vmalert/#templating . `+
+		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
 		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
@@ -178,19 +178,15 @@ func main() {
 
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8880"}
-	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
 	pushmetrics.Stop()
 
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
@@ -252,13 +248,7 @@ func newManager(ctx context.Context) (*manager, error) {
 func getExternalURL(customURL string) (*url.URL, error) {
 	if customURL == "" {
 		// use local hostname as external URL
-		listenAddr := ":8880"
-		if len(*httpListenAddrs) > 0 {
-			listenAddr = (*httpListenAddrs)[0]
-		}
-		isTLS := httpserver.IsTLS(0)
-
-		return getHostnameAsExternalURL(listenAddr, isTLS)
+		return getHostnameAsExternalURL(*httpListenAddr, httpserver.IsTLS())
 	}
 	u, err := url.Parse(customURL)
 	if err != nil {
@@ -270,13 +260,13 @@ func getExternalURL(customURL string) (*url.URL, error) {
 	return u, nil
 }
 
-func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
+func getHostnameAsExternalURL(httpListenAddr string, isSecure bool) (*url.URL, error) {
 	hname, err := os.Hostname()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get hostname: %w", err)
 	}
 	port := ""
-	if ipport := strings.Split(addr, ":"); len(ipport) > 1 {
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]
 	}
 	schema := "http://"
@@ -304,7 +294,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		qFn := func(_ string) ([]datasource.Metric, error) {
+		qFn := func(query string) ([]datasource.Metric, error) {
 			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
 		}
 		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
@@ -319,7 +309,7 @@ func usage() {
 	const s = `
 vmalert processes alerts and recording rules.
 
-See the docs at https://docs.victoriametrics.com/vmalert/ .
+See the docs at https://docs.victoriametrics.com/vmalert.html .
 `
 	flagutil.Usage(s)
 }

app/vmalert/manager.go

@@ -156,14 +156,11 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
-			// cancel evaluation so the Update will be applied as fast as possible.
-			// it is important to call InterruptEval before the update, because cancel fn
-			// can be re-assigned during the update.
-			item.old.InterruptEval()
 			go func(old *rule.Group, new *rule.Group) {
 				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
+			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alert_test.go

@@ -178,7 +178,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		},
 	}
 
-	qFn := func(_ string) ([]datasource.Metric, error) {
+	qFn := func(q string) ([]datasource.Metric, error) {
 		return []datasource.Metric{
 			{
 				Labels: []datasource.Label{

app/vmalert/notifier/alertmanager.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -105,7 +104,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 	if *showNotifierURL {
 		amURL = am.addr.String()
 	}
-	if resp.StatusCode/100 != 2 {
+	if resp.StatusCode != http.StatusOK {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return fmt.Errorf("failed to read response from %q: %w", amURL, err)
@@ -128,7 +127,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remoteread/init.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -61,7 +60,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
-	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
@@ -79,5 +78,5 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, authCfg, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -151,22 +151,12 @@ func (c *Client) run(ctx context.Context) {
 	ticker := time.NewTicker(c.flushInterval)
 	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
-		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
-		logger.Infof("shutting down remote write client and flushing remained series")
-
-		shutdownFlushCnt := 0
 		for ts := range c.input {
 			wr.Timeseries = append(wr.Timeseries, ts)
-			if len(wr.Timeseries) >= c.maxBatchSize {
-				shutdownFlushCnt += len(wr.Timeseries)
-				c.flush(lastCtx, wr)
-			}
 		}
-		// flush the last batch. `flush` will re-check and avoid flushing empty batch.
-		shutdownFlushCnt += len(wr.Timeseries)
+		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
+		logger.Infof("shutting down remote write client and flushing remained %d series", len(wr.Timeseries))
 		c.flush(lastCtx, wr)
-
-		logger.Infof("shutting down remote write client flushed %d series", shutdownFlushCnt)
 		cancel()
 	}
 	c.wg.Add(1)
@@ -289,7 +279,7 @@ L:
 
 func (c *Client) send(ctx context.Context, data []byte) error {
 	r := bytes.NewReader(data)
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.addr, r)
+	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
@@ -312,7 +302,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
-	resp, err := c.c.Do(req)
+	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
 			req.URL.Redacted(), err, len(data), r.Size())

app/vmalert/remotewrite/client_test.go

@@ -84,70 +84,6 @@ func TestClient_Push(t *testing.T) {
 	}
 }
 
-func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
-	batchSize := 20
-
-	testTable := []struct {
-		name     string // name of the test case
-		pushCnt  int    // how many time series is pushed to the client
-		batchCnt int    // the expected batch count sent by the client
-	}{
-		{
-			name:     "pushCnt % batchSize == 0",
-			pushCnt:  batchSize * 40,
-			batchCnt: 40,
-		},
-		{
-			name:     "pushCnt % batchSize != 0",
-			pushCnt:  batchSize*40 + 1,
-			batchCnt: 40 + 1,
-		},
-	}
-
-	for _, tt := range testTable {
-		t.Run(tt.name, func(t *testing.T) {
-			// run new server
-			bcServer := newBatchCntRWServer()
-
-			// run new client
-			rwClient, err := NewClient(context.Background(), Config{
-				MaxBatchSize: batchSize,
-
-				// Set everything to 1 to simplify the calculation.
-				Concurrency:   1,
-				MaxQueueSize:  1000,
-				FlushInterval: time.Minute,
-
-				// batch count server
-				Addr: bcServer.URL,
-			})
-			if err != nil {
-				t.Fatalf("new remote write client failed, err: %v", err)
-			}
-
-			// push time series to the client.
-			for i := 0; i < tt.pushCnt; i++ {
-				if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
-					t.Fatalf("push time series to the client failed, err: %v", err)
-				}
-			}
-
-			// close the client so the rest ts will be flushed in `shutdown`
-			if err = rwClient.Close(); err != nil {
-				t.Fatalf("shutdown client failed, err: %v", err)
-			}
-
-			// finally check how many batches is sent.
-			if tt.batchCnt != bcServer.acceptedBatches() {
-				t.Errorf("client sent batch count incorrect, want: %d, get: %d", tt.batchCnt, bcServer.acceptedBatches())
-			}
-			if tt.pushCnt != bcServer.accepted() {
-				t.Errorf("client sent time series count incorrect, want: %d, get: %d", tt.pushCnt, bcServer.accepted())
-			}
-		})
-	}
-}
-
 func newRWServer() *rwServer {
 	rw := &rwServer{}
 	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
@@ -155,12 +91,14 @@ func newRWServer() *rwServer {
 }
 
 type rwServer struct {
-	acceptedRows atomic.Uint64
+	// WARN: ordering of fields is important for alignment!
+	// see https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	acceptedRows uint64
 	*httptest.Server
 }
 
 func (rw *rwServer) accepted() int {
-	return int(rw.acceptedRows.Load())
+	return int(atomic.LoadUint64(&rw.acceptedRows))
 }
 
 func (rw *rwServer) err(w http.ResponseWriter, err error) {
@@ -206,7 +144,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))
 		return
 	}
-	rw.acceptedRows.Add(uint64(len(wr.Timeseries)))
+	atomic.AddUint64(&rw.acceptedRows, uint64(len(wr.Timeseries)))
 	w.WriteHeader(http.StatusNoContent)
 }
 
@@ -248,27 +186,3 @@ func (frw *faultyRWServer) handler(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("server overloaded"))
 	}
 }
-
-type batchCntRWServer struct {
-	*rwServer
-
-	batchCnt atomic.Int64 // accepted batch count, which also equals to request count
-}
-
-func newBatchCntRWServer() *batchCntRWServer {
-	bc := &batchCntRWServer{
-		rwServer: &rwServer{},
-	}
-
-	bc.Server = httptest.NewServer(http.HandlerFunc(bc.handler))
-	return bc
-}
-
-func (bc *batchCntRWServer) handler(w http.ResponseWriter, r *http.Request) {
-	bc.batchCnt.Add(1)
-	bc.rwServer.handler(w, r)
-}
-
-func (bc *batchCntRWServer) acceptedBatches() int {
-	return int(bc.batchCnt.Load())
-}

app/vmalert/remotewrite/debug_client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -30,7 +30,7 @@ func NewDebugClient() (*DebugClient, error) {
 		return nil, nil
 	}
 
-	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/remotewrite/init.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -65,7 +64,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, nil
 	}
 
-	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}

app/vmalert/rule/alerting.go

@@ -310,24 +310,37 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 	}
 	var result []prompbmarshal.TimeSeries
 	holdAlertState := make(map[uint64]*notifier.Alert)
-	qFn := func(_ string) ([]datasource.Metric, error) {
+	qFn := func(query string) ([]datasource.Metric, error) {
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range res.Data {
-		ls, as, err := ar.expandTemplates(s, qFn, time.Time{})
+		ls, err := ar.toLabels(s, qFn)
 		if err != nil {
-			return nil, fmt.Errorf("failed to expand templates: %s", err)
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+		h := hash(ls.processed)
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
+		if err != nil {
+			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
-		alertID := hash(ls.processed)
-		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
 
+		// if alert is instant, For: 0
+		if ar.For == 0 {
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
+		}
+
+		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
 			// try to restore alert's state on the first iteration
 			if at.Equal(start) {
-				if _, ok := ar.alerts[alertID]; ok {
-					a = ar.alerts[alertID]
+				if _, ok := ar.alerts[h]; ok {
+					a = ar.alerts[h]
 					prevT = at
 				}
 			}
@@ -341,15 +354,11 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.Start = at
 			}
 			prevT = at
-			if ar.For == 0 {
-				// rules with `for: 0` are always firing when they have Value
-				a.State = notifier.StateFiring
-			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
 			// save alert's state on last iteration, so it can be used on the next execRange call
 			if at.Equal(end) {
-				holdAlertState[alertID] = a
+				holdAlertState[h] = a
 			}
 		}
 	}
@@ -383,34 +392,15 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		}
 	}()
 
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
+
 	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.Samples, curState.Duration)
 
-	qFn := func(query string) ([]datasource.Metric, error) {
-		res, _, err := ar.q.Query(ctx, query, ts)
-		return res.Data, err
-	}
-
-	// template labels and annotations before updating ar.alerts,
-	// since they could use `query` function which takes a while to execute,
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079.
-	expandedLabels := make([]*labelSet, len(res.Data))
-	expandedAnnotations := make([]map[string]string, len(res.Data))
-	for i, m := range res.Data {
-		ls, as, err := ar.expandTemplates(m, qFn, ts)
-		if err != nil {
-			curState.Err = fmt.Errorf("failed to expand templates: %w", err)
-			return nil, curState.Err
-		}
-		expandedLabels[i] = ls
-		expandedAnnotations[i] = as
-	}
-
-	ar.alertsMu.Lock()
-	defer ar.alertsMu.Unlock()
-
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
 		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
@@ -419,18 +409,26 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		}
 	}
 
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res.Data, err
+	}
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
-	for i, m := range res.Data {
-		labels, annotations := expandedLabels[i], expandedAnnotations[i]
-		alertID := hash(labels.processed)
-		if _, ok := updated[alertID]; ok {
-			// duplicate may be caused the removal of `__name__` label
-			curState.Err = fmt.Errorf("labels %v: %w", labels.processed, errDuplicate)
+	for _, m := range res.Data {
+		ls, err := ar.toLabels(m, qFn)
+		if err != nil {
+			curState.Err = fmt.Errorf("failed to expand labels: %w", err)
 			return nil, curState.Err
 		}
-		updated[alertID] = struct{}{}
-		if a, ok := ar.alerts[alertID]; ok {
+		h := hash(ls.processed)
+		if _, ok := updated[h]; ok {
+			// duplicate may be caused the removal of `__name__` label
+			curState.Err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, curState.Err
+		}
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
 			if a.State == notifier.StateInactive {
 				// alert could be in inactive state for resolvedRetention
 				// so when we again receive metrics for it - we switch it
@@ -440,17 +438,23 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
 			a.Value = m.Values[0]
-			a.Annotations = annotations
+			// re-exec template since Value or query can be used in annotations
+			a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 			if err != nil {
 				return nil, err
 			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a := ar.newAlert(m, ts, labels.processed, annotations)
-		a.ID = alertID
+		a, err := ar.newAlert(m, ls, start, qFn)
+		if err != nil {
+			curState.Err = fmt.Errorf("failed to create alert: %w", err)
+			return nil, curState.Err
+		}
+		a.ID = h
 		a.State = notifier.StatePending
-		ar.alerts[alertID] = a
+		a.ActiveAt = ts
+		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
 	var numActivePending int
@@ -475,7 +479,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -500,28 +504,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	return ar.toTimeSeries(ts.Unix()), nil
 }
 
-func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
-	ls, err := ar.toLabels(m, qFn)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
-	}
-
-	tplData := notifier.AlertTplData{
-		Value:    m.Values[0],
-		Labels:   ls.origin,
-		Expr:     ar.Expr,
-		AlertID:  hash(ls.processed),
-		GroupID:  ar.GroupID,
-		ActiveAt: ts,
-		For:      ar.For,
-	}
-	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
-	}
-	return ls, as, nil
-}
-
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
@@ -555,31 +537,31 @@ func hash(labels map[string]string) uint64 {
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, annotations map[string]string) *notifier.Alert {
-	as := make(map[string]string)
-	if annotations != nil {
-		as = annotations
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
+	var err error
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %w", err)
+		}
 	}
-	ls := make(map[string]string)
-	if labels != nil {
-		ls = labels
-	}
-	return &notifier.Alert{
-		GroupID:     ar.GroupID,
-		Name:        ar.Name,
-		Expr:        ar.Expr,
-		For:         ar.For,
-		ActiveAt:    start,
-		Value:       m.Values[0],
-		Labels:      ls,
-		Annotations: as,
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+		For:      ar.For,
 	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+	return a, err
 }
 
 const (
-	// alertMetricName is the metric name for time series reflecting the alert state.
+	// alertMetricName is the metric name for synthetic alert timeseries.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
+	// alertForStateMetricName is the metric name for 'for' state of alert.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -594,10 +576,12 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	return []prompbmarshal.TimeSeries{
-		alertToTimeSeries(a, timestamp),
-		alertForToTimeSeries(a, timestamp),
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
+	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
@@ -629,6 +613,9 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 		return nil
 	}
 
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
 	if len(ar.alerts) < 1 {
 		return nil
 	}
@@ -653,10 +640,6 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 		ar.logDebugf(ts, nil, "no response was received from restore query")
 		return nil
 	}
-
-	ar.alertsMu.Lock()
-	defer ar.alertsMu.Unlock()
-
 	for _, series := range res.Data {
 		series.DelLabel("__name__")
 		labelSet := make(map[string]string, len(series.Labels))
@@ -681,19 +664,15 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 // alertsToSend walks through the current alerts of AlertingRule
 // and returns only those which should be sent to notifier.
 // Isn't concurrent safe.
-func (ar *AlertingRule) alertsToSend(resolveDuration, resendDelay time.Duration) []notifier.Alert {
-	currentTime := time.Now()
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
 	needsSending := func(a *notifier.Alert) bool {
 		if a.State == notifier.StatePending {
 			return false
 		}
-		if a.State == notifier.StateFiring && a.End.Before(a.LastSent) {
+		if a.ResolvedAt.After(a.LastSent) {
 			return true
 		}
-		if a.State == notifier.StateInactive && a.ResolvedAt.After(a.LastSent) {
-			return true
-		}
-		return a.LastSent.Add(resendDelay).Before(currentTime)
+		return a.LastSent.Add(resendDelay).Before(ts)
 	}
 
 	var alerts []notifier.Alert
@@ -701,11 +680,11 @@ func (ar *AlertingRule) alertsToSend(resolveDuration, resendDelay time.Duration)
 		if !needsSending(a) {
 			continue
 		}
-		a.End = currentTime.Add(resolveDuration)
+		a.End = ts.Add(resolveDuration)
 		if a.State == notifier.StateInactive {
 			a.End = a.ResolvedAt
 		}
-		a.LastSent = currentTime
+		a.LastSent = ts
 		alerts = append(alerts, *a)
 	}
 	return alerts

app/vmalert/rule/alerting_test.go

@@ -28,28 +28,20 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{
-				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					"job":      "foo",
-					"instance": "bar",
-				},
-			},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -57,35 +49,19 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-						"job":      "foo",
-						"instance": "bar",
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{
-				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					alertStateLabel: "foo",
-					"__name__":      "bar",
-				},
-			},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__":      alertForStateMetricName,
-						alertStateLabel: "foo",
-					}),
 			},
 		},
 		{
@@ -332,17 +308,14 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &datasource.FakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
-			ts := time.Now()
 			for i, step := range tc.steps {
 				fq.Reset()
 				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
+				if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-
-				// shift the execution timestamp before the next iteration
-				ts = ts.Add(defaultStep)
-
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -394,7 +367,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring},
 			},
 			nil,
 		},
@@ -405,9 +378,8 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels:   map[string]string{"name": "foo"},
-					State:    notifier.StateFiring,
-					ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
 				},
 			},
 			nil,
@@ -418,9 +390,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
 			},
 			nil,
 		},
@@ -488,20 +460,6 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				For:         time.Second,
 			}},
 		},
-		{
-			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
-			[]datasource.Metric{
-				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
-			},
-			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
-				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-			},
-			nil,
-		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
 			[]datasource.Metric{
@@ -576,33 +534,21 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					},
-				},
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					},
-				},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
 				//
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					},
-				},
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					},
-				},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
 			},
 			nil,
 		},
@@ -1054,7 +1000,7 @@ func TestAlertsToSend(t *testing.T) {
 		for i, a := range alerts {
 			ar.alerts[uint64(i)] = a
 		}
-		gotAlerts := ar.alertsToSend(resolveDuration, resendDelay)
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
 		if gotAlerts == nil && expAlerts == nil {
 			return
 		}
@@ -1070,36 +1016,60 @@ func TestAlertsToSend(t *testing.T) {
 		})
 		for i, exp := range expAlerts {
 			got := gotAlerts[i]
-			if got.Name != exp.Name {
-				t.Fatalf("expected Name to be %v; got %v", exp.Name, got.Name)
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
 			}
 		}
 	}
 
-	f( // check if firing alerts need to be sent with non-zero resendDelay
-		[]*notifier.Alert{
-			{Name: "a", State: notifier.StateFiring, Start: ts},
-			// no need to resend firing
-			{Name: "b", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(5 * time.Minute)},
-			// last message is for resolved, send firing message this time
-			{Name: "c", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(-1 * time.Minute)},
-			// resend firing
-			{Name: "d", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-1 * time.Minute)},
-		},
-		[]*notifier.Alert{{Name: "a"}, {Name: "c"}, {Name: "d"}},
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
 		5*time.Minute, time.Minute,
 	)
-	f( // check if resolved alerts need to be sent with non-zero resendDelay
-		[]*notifier.Alert{
-			{Name: "a", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts.Add(-30 * time.Second)},
-			// no need to resend resolved
-			{Name: "b", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts},
-			// resend resolved
-			{Name: "c", State: notifier.StateInactive, ResolvedAt: ts.Add(-1 * time.Minute), LastSent: ts.Add(-1 * time.Minute)},
-		},
-		[]*notifier.Alert{{Name: "a"}, {Name: "c"}},
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
 		5*time.Minute, time.Minute,
 	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
 }
 
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
@@ -1125,12 +1095,6 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 	return &rule
 }
 
-func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
-	rule := newTestAlertingRule(name, waitFor)
-	rule.EvalInterval = evalInterval
-	return rule
-}
-
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor

app/vmalert/rule/group.go

@@ -9,11 +9,10 @@ import (
 	"hash/fnv"
 	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-
 	"github.com/cheggaaa/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -705,7 +704,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return nil
 	}
 
-	alerts := ar.alertsToSend(resolveDuration, *resendDelay)
+	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
@@ -725,19 +724,13 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	return errGr.Err()
 }
 
-var bbPool bytesutil.ByteBufferPool
-
-// getStaleSeries checks whether there are stale series from previously sent ones.
+// getStaledSeries checks whether there are stale series from previously sent ones.
 func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
-	bb := bbPool.Get()
-	defer bbPool.Put(bb)
-
 	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
 	for _, ts := range tss {
-		// convert labels to strings, so we can compare with previously sent series
-		bb.B = labelsToString(bb.B, ts.Labels)
-		ruleLabels[string(bb.B)] = ts.Labels
-		bb.Reset()
+		// convert labels to strings so we can compare with previously sent series
+		key := labelsToString(ts.Labels)
+		ruleLabels[key] = ts.Labels
 	}
 
 	rID := r.ID()
@@ -783,20 +776,21 @@ func (e *executor) purgeStaleSeries(activeRules []Rule) {
 	e.previouslySentSeriesToRWMu.Unlock()
 }
 
-func labelsToString(dst []byte, labels []prompbmarshal.Label) []byte {
-	dst = append(dst, '{')
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b strings.Builder
+	b.WriteRune('{')
 	for i, label := range labels {
 		if len(label.Name) == 0 {
-			dst = append(dst, "__name__"...)
+			b.WriteString("__name__")
 		} else {
-			dst = append(dst, label.Name...)
+			b.WriteString(label.Name)
 		}
-		dst = append(dst, '=')
-		dst = strconv.AppendQuote(dst, label.Value)
+		b.WriteRune('=')
+		b.WriteString(strconv.Quote(label.Value))
 		if i < len(labels)-1 {
-			dst = append(dst, ',')
+			b.WriteRune(',')
 		}
 	}
-	dst = append(dst, '}')
-	return dst
+	b.WriteRune('}')
+	return b.String()
 }

app/vmalert/rule/group_test.go

@@ -217,21 +217,20 @@ func TestGroupStart(t *testing.T) {
 
 	const evalInterval = time.Millisecond
 	g := NewGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
+	g.Concurrency = 2
 
 	const inst1, inst2, job = "foo", "bar", "baz"
 	m1 := metricWithLabels(t, "instance", inst1, "job", job)
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
 	r := g.Rules[0].(*AlertingRule)
-	alert1 := r.newAlert(m1, time.Now(), nil, nil)
+	alert1, err := r.newAlert(m1, nil, time.Now(), nil)
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
 	alert1.State = notifier.StateFiring
-	// add annotations
-	alert1.Annotations["summary"] = "1"
 	// add external label
 	alert1.Labels["cluster"] = "east-1"
-	// add labels from response
-	alert1.Labels["job"] = job
-	alert1.Labels["instance"] = inst1
 	// add rule labels
 	alert1.Labels["label"] = "bar"
 	alert1.Labels["host"] = inst1
@@ -240,15 +239,13 @@ func TestGroupStart(t *testing.T) {
 	alert1.Labels[alertGroupNameLabel] = g.Name
 	alert1.ID = hash(alert1.Labels)
 
-	alert2 := r.newAlert(m2, time.Now(), nil, nil)
+	alert2, err := r.newAlert(m2, nil, time.Now(), nil)
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
 	alert2.State = notifier.StateFiring
-	// add annotations
-	alert2.Annotations["summary"] = "1"
 	// add external label
 	alert2.Labels["cluster"] = "east-1"
-	// add labels from response
-	alert2.Labels["job"] = job
-	alert2.Labels["instance"] = inst2
 	// add rule labels
 	alert2.Labels["label"] = "bar"
 	alert2.Labels["host"] = inst2
@@ -265,25 +262,8 @@ func TestGroupStart(t *testing.T) {
 		close(finished)
 	}()
 
-	waitForIterations := func(n int, interval time.Duration) {
-		t.Helper()
-
-		var cur uint64
-		prev := g.metrics.iterationTotal.Get()
-		for i := 0; ; i++ {
-			if i > 40 {
-				t.Fatalf("group wasn't able to perform %d evaluations during %d eval intervals", n, i)
-			}
-			cur = g.metrics.iterationTotal.Get()
-			if int(cur-prev) >= n {
-				return
-			}
-			time.Sleep(interval)
-		}
-	}
-
-	// wait for multiple evaluation iterations
-	waitForIterations(4, evalInterval)
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
 
 	gotAlerts := fn.GetAlerts()
 	expectedAlerts := []notifier.Alert{*alert1, *alert2}
@@ -300,8 +280,8 @@ func TestGroupStart(t *testing.T) {
 	// and set only one datapoint for response
 	fs.Add(m1)
 
-	// wait for multiple evaluation iterations
-	waitForIterations(4, evalInterval)
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
 
 	gotAlerts = fn.GetAlerts()
 	alert2.State = notifier.StateInactive

app/vmalert/rule/group_timing_test.go

@@ -1,36 +0,0 @@
-package rule
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-)
-
-func BenchmarkGetStaleSeries(b *testing.B) {
-	ts := time.Now()
-	n := 100
-	payload := make([]prompbmarshal.TimeSeries, n)
-	for i := 0; i < n; i++ {
-		s := fmt.Sprintf("%d", i)
-		labels := toPromLabels(b,
-			"__name__", "foo", ""+
-				"instance", s,
-			"job", s,
-			"state", s,
-		)
-		payload = append(payload, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels))
-	}
-
-	e := &executor{
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-	}
-	ar := &AlertingRule{RuleID: 1}
-
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		e.getStaleSeries(ar, payload, ts)
-	}
-}

app/vmalert/rule/rule.go

@@ -32,7 +32,7 @@ type Rule interface {
 	close()
 }
 
-var errDuplicate = errors.New("result contains metrics with the same labelset during evaluation. See https://docs.victoriametrics.com/vmalert/#series-with-the-same-labelset for details")
+var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels. See https://docs.victoriametrics.com/vmalert.html#series-with-the-same-labelset for details")
 
 type ruleState struct {
 	sync.RWMutex

app/vmalert/rule/test_helpers.go

@@ -95,7 +95,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
-func toPromLabels(t testing.TB, labels ...string) []prompbmarshal.Label {
+func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
 	t.Helper()
 	if len(labels) == 0 || len(labels)%2 != 0 {
 		t.Fatalf("expected to get even number of labels")

app/vmalert/static/js/custom.js

@@ -1,32 +1,11 @@
 function expandAll() {
-    $('.group-heading').each(function () {
-        let style = $(this).attr("style")
-        // display only elements that are currently visible
-        if (style === "display: none;") {
-            return
-        }
-        $(this).next().addClass('show')
-    });
+    $('.collapse').addClass('show');
 }
 
 function collapseAll() {
     $('.collapse').removeClass('show');
 }
 
-function showByID(id) {
-    if (!id) {
-        return
-    }
-    let parent = $("#" + id).parent();
-    if (!parent) {
-        return
-    }
-    let target = $("#" + parent.attr("data-bs-target"));
-    if (target.length > 0) {
-        target.addClass('show');
-    }
-}
-
 function toggleByID(id) {
     if (id) {
         let el = $("#" + id);
@@ -36,100 +15,6 @@ function toggleByID(id) {
     }
 }
 
-function debounce(func, delay) {
-    let timer;
-    return function (...args) {
-        clearTimeout(timer);
-        timer = setTimeout(() => {
-            func.apply(this, args);
-        }, delay);
-    };
-}
-
-$('#search').on("keyup", debounce(search, 500));
-
-// search shows or hides groups&rules that satisfy the search phrase.
-// case-insensitive, respects GET param `search`.
-function search() {
-    $(".rule").show();
-
-    let groupHeader = $(".group-heading")
-    let searchPhrase = $("#search").val().toLowerCase()
-    if (searchPhrase.length === 0) {
-        groupHeader.show()
-        setParamURL('search', '')
-        return
-    }
-
-    $(".rule-table").removeClass('show');
-    groupHeader.hide()
-
-    searchPhrase = searchPhrase.toLowerCase()
-    filterRuleByName(searchPhrase);
-    filterRuleByLabels(searchPhrase);
-    filterGroupsByName(searchPhrase);
-
-    setParamURL('search', searchPhrase)
-}
-
-function setParamURL(key, value) {
-    let url = new URL(location.href)
-    url.searchParams.set(key, value);
-    window.history.replaceState(null, null, `?${url.searchParams.toString()}${url.hash}`);
-}
-
-function getParamURL(key) {
-    let url = new URL(location.href)
-    return url.searchParams.get(key)
-}
-
-function filterGroupsByName(searchPhrase) {
-    $(".group-heading").each(function () {
-        const groupName = $(this).attr('data-group-name').toLowerCase();
-        const hasValue = groupName.indexOf(searchPhrase) >= 0
-
-        if (!hasValue) {
-            return
-        }
-
-        const target = $(this).attr("data-bs-target");
-        $(`div[id="${target}"] .rule`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByName(searchPhrase) {
-    $(".rule").each(function () {
-        const ruleName = $(this).attr("data-rule-name").toLowerCase();
-        const hasValue = ruleName.indexOf(searchPhrase) >= 0
-        if (!hasValue) {
-            $(this).hide();
-            return
-        }
-
-        const target = $(this).attr('data-bs-target')
-        $(`#rules-${target}`).addClass('show');
-        $(`div[data-bs-target='rules-${target}']`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByLabels(searchPhrase) {
-    $(".rule").each(function () {
-        const matches = $(".label", this).filter(function () {
-            const label = $(this).text().toLowerCase();
-            return label.indexOf(searchPhrase) >= 0;
-        }).length;
-
-        if (matches > 0) {
-            const target = $(this).attr('data-bs-target')
-            $(`#rules-${target}`).addClass('show');
-            $(`div[data-bs-target='rules-${target}']`).show();
-            $(this).show();
-        }
-    });
-}
-
 $(document).ready(function () {
     $(".group-heading a").click(function (e) {
         e.stopPropagation(); // prevent collapse logic on link click
@@ -147,15 +32,8 @@ $(document).ready(function () {
         });
     });
 
-    // update search element with value from URL, if any
-    let searchPhrase = getParamURL('search')
-    $("#search").val(searchPhrase)
-
-    // apply filtering by search phrase
-    search()
-
     let hash = window.location.hash.substr(1);
-    showByID(hash);
+    toggleByID(hash);
 });
 
 $(document).ready(function () {

app/vmalert/templates/template.go

@@ -476,7 +476,7 @@ func templateFuncs() textTpl.FuncMap {
 		// For example, {{ query "foo" | first | value }} will
 		// execute "/api/v1/query?query=foo" request and will return
 		// the first value in response.
-		"query": func(_ string) ([]metric, error) {
+		"query": func(q string) ([]metric, error) {
 			// query function supposed to be substituted at FuncsWithQuery().
 			// it is present here only for validation purposes, when there is no
 			// provided datasource.

app/vmalert/utils/tls.go

@@ -1,4 +1,4 @@
-package httputils
+package utils
 
 import (
 	"crypto/tls"
@@ -11,12 +11,12 @@ import (
 
 // Transport creates http.Transport object based on provided URL.
 // Returns Transport with TLS configuration if URL contains `https` prefix
-func Transport(URL, certFile, keyFile, caFile, serverName string, insecureSkipVerify bool) (*http.Transport, error) {
+func Transport(URL, certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*http.Transport, error) {
 	t := http.DefaultTransport.(*http.Transport).Clone()
 	if !strings.HasPrefix(URL, "https") {
 		return t, nil
 	}
-	tlsCfg, err := TLSConfig(certFile, keyFile, caFile, serverName, insecureSkipVerify)
+	tlsCfg, err := TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
 	if err != nil {
 		return nil, err
 	}
@@ -25,7 +25,7 @@ func Transport(URL, certFile, keyFile, caFile, serverName string, insecureSkipVe
 }
 
 // TLSConfig creates tls.Config object from provided arguments
-func TLSConfig(certFile, keyFile, caFile, serverName string, insecureSkipVerify bool) (*tls.Config, error) {
+func TLSConfig(certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*tls.Config, error) {
 	var certs []tls.Certificate
 	if certFile != "" {
 		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
@@ -37,15 +37,15 @@ func TLSConfig(certFile, keyFile, caFile, serverName string, insecureSkipVerify
 	}
 
 	var rootCAs *x509.CertPool
-	if caFile != "" {
-		pem, err := os.ReadFile(caFile)
+	if CAFile != "" {
+		pem, err := os.ReadFile(CAFile)
 		if err != nil {
-			return nil, fmt.Errorf("cannot read `ca_file` %q: %w", caFile, err)
+			return nil, fmt.Errorf("cannot read `ca_file` %q: %w", CAFile, err)
 		}
 
 		rootCAs = x509.NewCertPool()
 		if !rootCAs.AppendCertsFromPEM(pem) {
-			return nil, fmt.Errorf("cannot parse data from `ca_file` %q", caFile)
+			return nil, fmt.Errorf("cannot parse data from `ca_file` %q", CAFile)
 		}
 	}
 

app/vmalert/utils/tls_test.go

@@ -1,4 +1,4 @@
-package httputils
+package utils
 
 import "testing"
 

app/vmalert/web.go

@@ -12,15 +12,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
-var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
-
 var (
 	apiLinks = [][2]string{
 		// api links are relative since they can be used by external clients,
@@ -39,7 +35,7 @@ var (
 		{Name: "Groups", Url: "groups"},
 		{Name: "Alerts", Url: "alerts"},
 		{Name: "Notifiers", Url: "notifiers"},
-		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert/"},
+		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
 	}
 )
 
@@ -88,10 +84,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		var data []apiGroup
-		rf := extractRulesFilter(r)
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		WriteListGroups(w, r, rh.groups())
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -102,20 +95,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		var data []apiGroup
-		rf := extractRulesFilter(r)
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		WriteListGroups(w, r, rh.groups())
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		var data []byte
-		var err error
-
-		rf := extractRulesFilter(r)
-		data, err = rh.listGroups(rf)
-
+		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -123,7 +108,6 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
-
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
@@ -167,9 +151,6 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -220,94 +201,26 @@ type listGroupsResponse struct {
 	} `json:"data"`
 }
 
-// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
-type rulesFilter struct {
-	files         []string
-	groupNames    []string
-	ruleNames     []string
-	ruleType      string
-	excludeAlerts bool
-}
-
-func extractRulesFilter(r *http.Request) rulesFilter {
-	rf := rulesFilter{}
-
-	var ruleType string
-	ruleTypeParam := r.URL.Query().Get("type")
-	// for some reason, `type` in filter doesn't match `type` in response,
-	// so we use this matching here
-	if ruleTypeParam == "alert" {
-		ruleType = ruleTypeAlerting
-	} else if ruleTypeParam == "record" {
-		ruleType = ruleTypeRecording
-	}
-	rf.ruleType = ruleType
-
-	rf.excludeAlerts = httputils.GetBool(r, "exclude_alerts")
-	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
-	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
-	rf.files = append([]string{}, r.Form["file[]"]...)
-	return rf
-}
-
-func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
+func (rh *requestHandler) groups() []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	isInList := func(list []string, needle string) bool {
-		if len(list) < 1 {
-			return true
-		}
-		for _, i := range list {
-			if i == needle {
-				return true
-			}
-		}
-		return false
-	}
-
 	groups := make([]apiGroup, 0)
-	for _, group := range rh.m.groups {
-		if !isInList(rf.groupNames, group.Name) {
-			continue
-		}
-		if !isInList(rf.files, group.File) {
-			continue
-		}
-
-		g := groupToAPI(group)
-		// the returned list should always be non-nil
-		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
-		filteredRules := make([]apiRule, 0)
-		for _, r := range g.Rules {
-			if rf.ruleType != "" && rf.ruleType != r.Type {
-				continue
-			}
-			if !isInList(rf.ruleNames, r.Name) {
-				continue
-			}
-			if rf.excludeAlerts {
-				r.Alerts = nil
-			}
-			filteredRules = append(filteredRules, r)
-		}
-		g.Rules = filteredRules
-		groups = append(groups, g)
+	for _, g := range rh.m.groups {
+		groups = append(groups, groupToAPI(g))
 	}
-	// sort list of groups for deterministic output
+
+	// sort list of alerts for deterministic output
 	sort.Slice(groups, func(i, j int) bool {
-		a, b := groups[i], groups[j]
-		if a.Name != b.Name {
-			return a.Name < b.Name
-		}
-		return a.File < b.File
+		return groups[i].Name < groups[j].Name
 	})
+
 	return groups
 }
 
-func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
+func (rh *requestHandler) listGroups() ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups(rf)
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{

app/vmalert/web.qtpl

@@ -70,29 +70,15 @@ btn-primary
                 }
             }
         %}
-        <div class="btn-toolbar mb-3" role="toolbar">
-          <div>
-            <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
-            <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-            <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-            <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
-            <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
-          </div>
-          <div class="col-md-4 col-lg-5">
-            <div class="px-3 input-group">
-              <div class="input-group-prepend">
-                <span class="input-group-text">
-                  <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                </span>
-              </div>
-              <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-            </div>
-          </div>
-        </div>
+         <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
+         <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
         {%  if len(groups) > 0 %}
             {% for _, g := range groups  %}
                   <div
-                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
+                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}">
                     <span class="anchor" id="group-{%s g.ID %}"></span>
                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                      {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
@@ -114,7 +100,7 @@ btn-primary
                         </div>
                     {% endif %}
                 </div>
-                <div class="collapse rule-table" id="rules-{%s g.ID %}">
+                <div class="collapse" id="rules-{%s g.ID %}">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
                             <tr>
@@ -125,7 +111,7 @@ btn-primary
                         </thead>
                         <tbody>
                         {% for _, r := range g.Rules %}
-                            <tr class="rule{% if r.LastError != "" %} alert-danger{% endif %}" data-rule-name="{%s r.Name %}" data-bs-target="{%s g.ID %}">
+                            <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
                                 <td>
                                     <div class="row">
                                         <div class="col-12 mb-2">
@@ -148,7 +134,7 @@ btn-primary
                                         <div class="col-12 mb-2">
                                             {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
                                             {% for k, v := range r.Labels %}
-                                                    <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
+                                                    <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
                                             {% endfor %}
                                         </div>
                                         {% if r.LastError != "" %}
@@ -184,25 +170,11 @@ btn-primary
     {%code prefix := utils.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {% if len(groupAlerts) > 0 %}
-         <div class="btn-toolbar mb-3" role="toolbar">
-              <div>
-                <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-                 <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-              </div>
-              <div class="col-md-4 col-lg-5">
-                <div class="px-3 input-group">
-                  <div class="input-group-prepend">
-                    <span class="input-group-text">
-                      <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                    </span>
-                  </div>
-                  <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-                </div>
-              </div>
-          </div>
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
          {% for _, ga := range groupAlerts %}
             {%code g := ga.Group %}
-            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
                 <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
@@ -220,7 +192,7 @@ btn-primary
                 }
                 sort.Strings(keys)
             %}
-            <div class="collapse rule-table" id="rules-{%s g.ID %}">
+            <div class="collapse" id="rules-{%s g.ID %}">
                 {% for _, ruleID := range keys %}
                     {%code
                         defaultAR := alertsByRule[ruleID][0]
@@ -231,46 +203,45 @@ btn-primary
                         sort.Strings(labelKeys)
                     %}
                     <br>
-                    <div class="rule" data-rule-name="{%s defaultAR.Name %}" data-bs-target="{%s g.ID %}">
-                      <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
-                       | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
-                      <br>
-                      <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
-                      <table class="table table-striped table-hover table-sm">
-                          <thead>
-                              <tr>
-                                  <th scope="col">Labels</th>
-                                  <th scope="col">State</th>
-                                  <th scope="col">Active at</th>
-                                  <th scope="col">Value</th>
-                                  <th scope="col">Link</th>
-                              </tr>
-                          </thead>
-                          <tbody>
-                          {% for _, ar := range alertsByRule[ruleID] %}
-                              <tr>
-                                  <td>
-                                      {% for _, k := range labelKeys %}
-                                          <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
-                                      {% endfor %}
-                                  </td>
-                                  <td>{%= badgeState(ar.State) %}</td>
-                                  <td>
-                                      {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
-                                      {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
-                                      {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
-                                  </td>
-                                  <td>{%s ar.Value %}</td>
-                                  <td>
-                                      <a href="{%s prefix+ar.WebLink() %}">Details</a>
-                                  </td>
-                              </tr>
-                          {% endfor %}
-                       </tbody>
-                      </table>
-                    </div>
+                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                    <br>
+                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr>
+                                <th scope="col">Labels</th>
+                                <th scope="col">State</th>
+                                <th scope="col">Active at</th>
+                                <th scope="col">Value</th>
+                                <th scope="col">Link</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                        {% for _, ar := range alertsByRule[ruleID] %}
+                            <tr>
+                                <td>
+                                    {% for _, k := range labelKeys %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
+                                    {% endfor %}
+                                </td>
+                                <td>{%= badgeState(ar.State) %}</td>
+                                <td>
+                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                    {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
+                                </td>
+                                <td>{%s ar.Value %}</td>
+                                <td>
+                                    <a href="{%s prefix+ar.WebLink() %}">Details</a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                     </tbody>
+                    </table>
                 {% endfor %}
             </div>
+            <br>
         {% endfor %}
 
     {% else %}

app/vmalert/web_test.go

@@ -23,7 +23,6 @@ func TestHandler(t *testing.T) {
 	})
 	g := &rule.Group{
 		Name:        "group",
-		File:        "rules.yaml",
 		Concurrency: 1,
 	}
 	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
@@ -36,7 +35,7 @@ func TestHandler(t *testing.T) {
 	}}
 	rh := &requestHandler{m: m}
 
-	getResp := func(t *testing.T, url string, to interface{}, code int) {
+	getResp := func(url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
@@ -60,43 +59,43 @@ func TestHandler(t *testing.T) {
 	defer ts.Close()
 
 	t.Run("/", func(t *testing.T) {
-		getResp(t, ts.URL, nil, 200)
-		getResp(t, ts.URL+"/vmalert", nil, 200)
-		getResp(t, ts.URL+"/vmalert/alerts", nil, 200)
-		getResp(t, ts.URL+"/vmalert/groups", nil, 200)
-		getResp(t, ts.URL+"/vmalert/notifiers", nil, 200)
-		getResp(t, ts.URL+"/rules", nil, 200)
+		getResp(ts.URL, nil, 200)
+		getResp(ts.URL+"/vmalert", nil, 200)
+		getResp(ts.URL+"/vmalert/alerts", nil, 200)
+		getResp(ts.URL+"/vmalert/groups", nil, 200)
+		getResp(ts.URL+"/vmalert/notifiers", nil, 200)
+		getResp(ts.URL+"/rules", nil, 200)
 	})
 
 	t.Run("/vmalert/rule", func(t *testing.T) {
 		a := ruleToAPI(ar)
-		getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
 		r := ruleToAPI(rr)
-		getResp(t, ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
+		getResp(ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
 	})
 	t.Run("/vmalert/alert", func(t *testing.T) {
 		alerts := ruleToAPIAlert(ar)
 		for _, a := range alerts {
-			getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+			getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
 		}
 	})
 	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)
-		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
 
 		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramRuleID)
-		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
 	})
 
 	t.Run("/api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
-		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 
 		lr = listAlertsResponse{}
-		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
 			t.Errorf("expected 1 alert got %d", length)
 		}
@@ -104,13 +103,13 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
 		expAlert := newAlertAPI(ar, ar.GetAlerts()[0])
 		alert := &apiAlert{}
-		getResp(t, ts.URL+"/"+expAlert.APILink(), alert, 200)
+		getResp(ts.URL+"/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
 
 		alert = &apiAlert{}
-		getResp(t, ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
+		getResp(ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
@@ -118,28 +117,28 @@ func TestHandler(t *testing.T) {
 
 	t.Run("/api/v1/alert?badParams", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramAlertID)
-		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
-		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
 		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramAlertID)
-		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
-		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
 		// bad request, alertID is missing
 		params = fmt.Sprintf("?%s=1", paramGroupID)
-		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 400)
-		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 400)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
 	})
 
 	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}
 
 		lr = listGroupsResponse{}
-		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}
@@ -147,93 +146,25 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
 		expRule := ruleToAPI(ar)
 		gotRule := apiRule{}
-		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRule, 200)
+		getResp(ts.URL+"/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRule = apiRule{}
-		getResp(t, ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
+		getResp(ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
-		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
+		getResp(ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
 		if gotRuleWithUpdates.StateUpdates == nil || len(gotRuleWithUpdates.StateUpdates) < 1 {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
-
-	t.Run("/api/v1/rules&filters", func(t *testing.T) {
-		check := func(url string, expGroups, expRules int) {
-			t.Helper()
-			lr := listGroupsResponse{}
-			getResp(t, ts.URL+url, &lr, 200)
-			if length := len(lr.Data.Groups); length != expGroups {
-				t.Errorf("expected %d groups got %d", expGroups, length)
-			}
-			if len(lr.Data.Groups) < 1 {
-				return
-			}
-			var rulesN int
-			for _, gr := range lr.Data.Groups {
-				rulesN += len(gr.Rules)
-			}
-			if rulesN != expRules {
-				t.Errorf("expected %d rules got %d", expRules, rulesN)
-			}
-		}
-
-		check("/api/v1/rules?type=alert", 1, 1)
-		check("/api/v1/rules?type=record", 1, 1)
-
-		check("/vmalert/api/v1/rules?type=alert", 1, 1)
-		check("/vmalert/api/v1/rules?type=record", 1, 1)
-
-		// no filtering expected due to bad params
-		check("/api/v1/rules?type=badParam", 1, 2)
-		check("/api/v1/rules?foo=bar", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
-
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
-	})
-	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
-		// check if response returns active alerts by default
-		lr := listGroupsResponse{}
-		getResp(t, ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml", &lr, 200)
-		activeAlerts := 0
-		for _, gr := range lr.Data.Groups {
-			for _, r := range gr.Rules {
-				activeAlerts += len(r.Alerts)
-			}
-		}
-		if activeAlerts == 0 {
-			t.Fatalf("expected at least 1 active alert in response; got 0")
-		}
-
-		// disable returning alerts via param
-		lr = listGroupsResponse{}
-		getResp(t, ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml&exclude_alerts=true", &lr, 200)
-		activeAlerts = 0
-		for _, gr := range lr.Data.Groups {
-			for _, r := range gr.Rules {
-				activeAlerts += len(r.Alerts)
-			}
-		}
-		if activeAlerts != 0 {
-			t.Fatalf("expected to get 0 active alert in response; got %d", activeAlerts)
-		}
-	})
 }
 
 func TestEmptyResponse(t *testing.T) {
@@ -241,7 +172,7 @@ func TestEmptyResponse(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithNoGroups.handler(w, r) }))
 	defer ts.Close()
 
-	getResp := func(t *testing.T, url string, to interface{}, code int) {
+	getResp := func(url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
@@ -264,13 +195,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("no groups /api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
-		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
 			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
 
 		lr = listAlertsResponse{}
-		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
 			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
@@ -278,13 +209,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("no groups /api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
-		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
@@ -295,13 +226,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("empty group /api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
-		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}

app/vmalert/web_types.go

@@ -193,15 +193,10 @@ func ruleToAPI(r interface{}) apiRule {
 	return apiRule{}
 }
 
-const (
-	ruleTypeRecording = "recording"
-	ruleTypeAlerting  = "alerting"
-)
-
 func recordingToAPI(rr *rule.RecordingRule) apiRule {
 	lastState := rule.GetLastEntry(rr)
 	r := apiRule{
-		Type:              ruleTypeRecording,
+		Type:              "recording",
 		DatasourceType:    rr.Type.String(),
 		Name:              rr.Name,
 		Query:             rr.Expr,
@@ -229,7 +224,7 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 func alertingToAPI(ar *rule.AlertingRule) apiRule {
 	lastState := rule.GetLastEntry(ar)
 	r := apiRule{
-		Type:              ruleTypeAlerting,
+		Type:              "alerting",
 		DatasourceType:    ar.Type.String(),
 		Name:              ar.Name,
 		Query:             ar.Expr,

app/vmauth/Makefile

@@ -87,9 +87,6 @@ vmauth-linux-ppc64le:
 vmauth-linux-s390x:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-vmauth-linux-loong64:
-	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmauth-linux-386:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmauth/README.md

@@ -1,3 +1,3 @@
-See vmauth docs [here](https://docs.victoriametrics.com/vmauth/).
+See vmauth docs [here](https://docs.victoriametrics.com/vmauth.html).
 
 vmauth docs can be edited at [docs/vmauth.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmauth.md).

app/vmauth/auth_config.go

@@ -2,70 +2,54 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"math"
 	"net/http"
 	"net/url"
 	"os"
 	"regexp"
-	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
 var (
 	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmauth/ for details on the format of this auth config")
+		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "interval for config file re-read. "+
 		"Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.")
 	defaultRetryStatusCodes = flagutil.NewArrayInt("retryStatusCodes", 0, "Comma-separated list of default HTTP response status codes when vmauth re-tries the request on other backends. "+
-		"See https://docs.victoriametrics.com/vmauth/#load-balancing for details")
+		"See https://docs.victoriametrics.com/vmauth.html#load-balancing for details")
 	defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+
-		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth/#load-balancing")
-	discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+
-		"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/vmauth/#discovering-backend-ips")
-	discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+
-		"Too low value may lead to DNS errors")
-	httpAuthHeader = flagutil.NewArrayString("httpAuthHeader", "HTTP request header to use for obtaining authorization tokens. By default auth tokens are read from Authorization request header")
+		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing for more details")
 )
 
 // AuthConfig represents auth config.
 type AuthConfig struct {
 	Users            []UserInfo `yaml:"users,omitempty"`
 	UnauthorizedUser *UserInfo  `yaml:"unauthorized_user,omitempty"`
-
-	// ms holds all the metrics for the given AuthConfig
-	ms *metrics.Set
 }
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name string `yaml:"name,omitempty"`
-
-	BearerToken string `yaml:"bearer_token,omitempty"`
-	AuthToken   string `yaml:"auth_token,omitempty"`
-	Username    string `yaml:"username,omitempty"`
-	Password    string `yaml:"password,omitempty"`
-
+	Name                   string      `yaml:"name,omitempty"`
+	BearerToken            string      `yaml:"bearer_token,omitempty"`
+	Username               string      `yaml:"username,omitempty"`
+	Password               string      `yaml:"password,omitempty"`
 	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
-	DiscoverBackendIPs     *bool       `yaml:"discover_backend_ips,omitempty"`
 	URLMaps                []URLMap    `yaml:"url_map,omitempty"`
 	HeadersConf            HeadersConf `yaml:",inline"`
 	MaxConcurrentRequests  int         `yaml:"max_concurrent_requests,omitempty"`
@@ -73,28 +57,22 @@ type UserInfo struct {
 	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
 	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
 	DropSrcPathPrefixParts *int        `yaml:"drop_src_path_prefix_parts,omitempty"`
-	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
-	TLSCertFile            string      `yaml:"tls_cert_file,omitempty"`
-	TLSKeyFile             string      `yaml:"tls_key_file,omitempty"`
-	TLSServerName          string      `yaml:"tls_server_name,omitempty"`
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
-
-	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
-	rt http.RoundTripper
+	httpTransport *http.Transport
 
 	requests         *metrics.Counter
-	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
-	RequestHeaders  []*Header `yaml:"headers,omitempty"`
-	ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
+	RequestHeaders  []Header `yaml:"headers,omitempty"`
+	ResponseHeaders []Header `yaml:"response_headers,omitempty"`
 }
 
 func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -123,8 +101,6 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 type Header struct {
 	Name  string
 	Value string
-
-	sOriginal string
 }
 
 // UnmarshalYAML unmarshals h from f.
@@ -133,8 +109,6 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 	if err := f(&s); err != nil {
 		return err
 	}
-	h.sOriginal = s
-
 	n := strings.IndexByte(s, ':')
 	if n < 0 {
 		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
@@ -146,29 +120,21 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 
 // MarshalYAML marshals h to yaml.
 func (h *Header) MarshalYAML() (interface{}, error) {
-	return h.sOriginal, nil
+	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
+	return s, nil
 }
 
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	// SrcPaths is an optional list of regular expressions, which must match the request path.
-	SrcPaths []*Regex `yaml:"src_paths,omitempty"`
-
-	// SrcHosts is an optional list of regular expressions, which must match the request hostname.
+	// SrcHosts is the list of regular expressions, which match the request hostname.
 	SrcHosts []*Regex `yaml:"src_hosts,omitempty"`
 
-	// SrcQueryArgs is an optional list of query args, which must match request URL query args.
-	SrcQueryArgs []*QueryArg `yaml:"src_query_args,omitempty"`
-
-	// SrcHeaders is an optional list of headers, which must match request headers.
-	SrcHeaders []*Header `yaml:"src_headers,omitempty"`
+	// SrcPaths is the list of regular expressions, which match the request path.
+	SrcPaths []*Regex `yaml:"src_paths,omitempty"`
 
 	// UrlPrefix contains backend url prefixes for the proxied request url.
 	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
 
-	// DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS.
-	DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
-
 	// HeadersConf is the config for augumenting request and response headers.
 	HeadersConf HeadersConf `yaml:",inline"`
 
@@ -182,78 +148,27 @@ type URLMap struct {
 	DropSrcPathPrefixParts *int `yaml:"drop_src_path_prefix_parts,omitempty"`
 }
 
-// QueryArg represents HTTP query arg
-type QueryArg struct {
-	Name  string
-	Value *Regex
-
+// Regex represents a regex
+type Regex struct {
 	sOriginal string
-}
-
-// UnmarshalYAML unmarshals qa from yaml.
-func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
-	var s string
-	if err := f(&s); err != nil {
-		return err
-	}
-	qa.sOriginal = s
-
-	n := strings.IndexByte(s, '=')
-	if n < 0 {
-		return nil
-	}
-
-	qa.Name = s[:n]
-	expr := s[n+1:]
-	if !strings.HasPrefix(expr, "~") {
-		expr = regexp.QuoteMeta(expr)
-	} else {
-		expr = expr[1:]
-	}
-
-	var re Regex
-	if err := yaml.Unmarshal([]byte(expr), &re); err != nil {
-		return fmt.Errorf("cannot unmarshal regex for %q query arg: %w", qa.Name, err)
-	}
-	qa.Value = &re
-	return nil
-}
-
-// MarshalYAML marshals qa to yaml.
-func (qa *QueryArg) MarshalYAML() (interface{}, error) {
-	return qa.sOriginal, nil
+	re        *regexp.Regexp
 }
 
 // URLPrefix represents passed `url_prefix`
 type URLPrefix struct {
+	n uint32
+
+	// the list of backend urls
+	bus []*backendURL
+
 	// requests are re-tried on other backend urls for these http response status codes
 	retryStatusCodes []int
 
 	// load balancing policy used
 	loadBalancingPolicy string
 
-	// how many request path prefix parts to drop before routing the request to backendURL
+	// how many request path prefix parts to drop before routing the request to backendURL.
 	dropSrcPathPrefixParts int
-
-	// busOriginal contains the original list of backends specified in yaml config.
-	busOriginal []*url.URL
-
-	// n is an atomic counter, which is used for balancing load among available backends.
-	n atomic.Uint32
-
-	// the list of backend urls
-	//
-	// the list can be dynamically updated if `discover_backend_ips` option is set.
-	bus atomic.Pointer[[]*backendURL]
-
-	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
-	discoverBackendIPs bool
-
-	// The next deadline for DNS-based discovery of backend IPs
-	nextDiscoveryDeadline atomic.Uint64
-
-	// vOriginal contains the original yaml value for URLPrefix.
-	vOriginal interface{}
 }
 
 func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
@@ -269,160 +184,49 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 }
 
 type backendURL struct {
-	brokenDeadline     atomic.Uint64
-	concurrentRequests atomic.Int32
-
-	url *url.URL
+	brokenDeadline     uint64
+	concurrentRequests int32
+	url                *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
 	ct := fasttime.UnixTimestamp()
-	return ct < bu.brokenDeadline.Load()
+	return ct < atomic.LoadUint64(&bu.brokenDeadline)
 }
 
 func (bu *backendURL) setBroken() {
 	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	bu.brokenDeadline.Store(deadline)
+	atomic.StoreUint64(&bu.brokenDeadline, deadline)
 }
 
 func (bu *backendURL) get() {
-	bu.concurrentRequests.Add(1)
+	atomic.AddInt32(&bu.concurrentRequests, 1)
 }
 
 func (bu *backendURL) put() {
-	bu.concurrentRequests.Add(-1)
+	atomic.AddInt32(&bu.concurrentRequests, -1)
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	pbus := up.bus.Load()
-	return len(*pbus)
+	return len(up.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func (up *URLPrefix) getBackendURL() *backendURL {
-	up.discoverBackendAddrsIfNeeded()
-
-	pbus := up.bus.Load()
-	bus := *pbus
 	if up.loadBalancingPolicy == "first_available" {
-		return getFirstAvailableBackendURL(bus)
+		return up.getFirstAvailableBackendURL()
 	}
-	return getLeastLoadedBackendURL(bus, &up.n)
-}
-
-func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
-	if !up.discoverBackendIPs {
-		// The discovery is disabled.
-		return
-	}
-
-	ct := fasttime.UnixTimestamp()
-	deadline := up.nextDiscoveryDeadline.Load()
-	if ct < deadline {
-		// There is no need in discovering backends.
-		return
-	}
-
-	intervalSec := math.Ceil(discoverBackendIPsInterval.Seconds())
-	if intervalSec <= 0 {
-		intervalSec = 1
-	}
-	nextDeadline := ct + uint64(intervalSec)
-	if !up.nextDiscoveryDeadline.CompareAndSwap(deadline, nextDeadline) {
-		// Concurrent goroutine already started the discovery.
-		return
-	}
-
-	// Discover ips for all the backendURLs
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(intervalSec))
-	hostToAddrs := make(map[string][]string)
-	for _, bu := range up.busOriginal {
-		host := bu.Hostname()
-		if hostToAddrs[host] != nil {
-			// ips for the given host have been already discovered
-			continue
-		}
-		var resolvedAddrs []string
-		if strings.HasPrefix(host, "srv+") {
-			// The host has the format 'srv+realhost'. Strip 'srv+' prefix before performing the lookup.
-			host = strings.TrimPrefix(host, "srv+")
-			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
-			if err != nil {
-				logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
-				resolvedAddrs = []string{host}
-			} else {
-				resolvedAddrs := make([]string, len(addrs))
-				for i, addr := range addrs {
-					resolvedAddrs[i] = fmt.Sprintf("%s:%d", addr.Target, addr.Port)
-				}
-			}
-		} else {
-			addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
-			if err != nil {
-				logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err)
-				resolvedAddrs = []string{host}
-			} else {
-				resolvedAddrs = make([]string, len(addrs))
-				for i, addr := range addrs {
-					resolvedAddrs[i] = addr.String()
-				}
-			}
-		}
-		// sort resolvedAddrs, so they could be compared below in areEqualBackendURLs()
-		sort.Strings(resolvedAddrs)
-		hostToAddrs[host] = resolvedAddrs
-	}
-	cancel()
-
-	// generate new backendURLs for the resolved IPs
-	var busNew []*backendURL
-	for _, bu := range up.busOriginal {
-		host := bu.Hostname()
-		port := bu.Port()
-		for _, addr := range hostToAddrs[host] {
-			buCopy := *bu
-			buCopy.Host = addr
-			if port != "" {
-				if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
-					// Drop the discovered port and substitute it the the port specified in bu.
-					buCopy.Host = buCopy.Host[:n]
-				}
-				buCopy.Host += ":" + port
-			}
-			busNew = append(busNew, &backendURL{
-				url: &buCopy,
-			})
-		}
-	}
-
-	pbus := up.bus.Load()
-	if areEqualBackendURLs(*pbus, busNew) {
-		return
-	}
-
-	// Store new backend urls
-	up.bus.Store(&busNew)
-}
-
-func areEqualBackendURLs(a, b []*backendURL) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i, aURL := range a {
-		bURL := b[i]
-		if aURL.url.String() != bURL.url.String() {
-			return false
-		}
-	}
-	return true
+	return up.getLeastLoadedBackendURL()
 }
 
 // getFirstAvailableBackendURL returns the first available backendURL, which isn't broken.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
-func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
+func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL {
+	bus := up.bus
+
 	bu := bus[0]
 	if !bu.isBroken() {
 		// Fast path - send the request to the first url.
@@ -444,7 +248,8 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 // getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
-func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
+func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
+	bus := up.bus
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
@@ -453,7 +258,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 
 	// Slow path - select other backend urls.
-	n := atomicCounter.Add(1)
+	n := atomic.AddUint32(&up.n, 1)
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -461,22 +266,20 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		if bu.isBroken() {
 			continue
 		}
-		if bu.concurrentRequests.Load() == 0 {
+		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
 			// Fast path - return the backend with zero concurrently executed requests.
-			// Do not use CompareAndSwap() instead of Load(), since it is much slower on systems with many CPU cores.
-			bu.concurrentRequests.Add(1)
 			return bu
 		}
 	}
 
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMin := bus[n%uint32(len(bus))]
-	minRequests := buMin.concurrentRequests.Load()
+	minRequests := atomic.LoadInt32(&buMin.concurrentRequests)
 	for _, bu := range bus {
 		if bu.isBroken() {
 			continue
 		}
-		if n := bu.concurrentRequests.Load(); n < minRequests {
+		if n := atomic.LoadInt32(&bu.concurrentRequests); n < minRequests {
 			buMin = bu
 			minRequests = n
 		}
@@ -491,7 +294,6 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 	if err := f(&v); err != nil {
 		return err
 	}
-	up.vOriginal = v
 
 	var urls []string
 	switch x := v.(type) {
@@ -514,28 +316,38 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
 	}
 
-	bus := make([]*url.URL, len(urls))
+	bus := make([]*backendURL, len(urls))
 	for i, u := range urls {
 		pu, err := url.Parse(u)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
 		}
-		bus[i] = pu
+		bus[i] = &backendURL{
+			url: pu,
+		}
 	}
-	up.busOriginal = bus
+	up.bus = bus
 	return nil
 }
 
 // MarshalYAML marshals up to yaml.
 func (up *URLPrefix) MarshalYAML() (interface{}, error) {
-	return up.vOriginal, nil
-}
-
-// Regex represents a regex
-type Regex struct {
-	re *regexp.Regexp
-
-	sOriginal string
+	var b []byte
+	if len(up.bus) == 1 {
+		u := up.bus[0].url.String()
+		b = strconv.AppendQuote(b, u)
+		return string(b), nil
+	}
+	b = append(b, '[')
+	for i, bu := range up.bus {
+		u := bu.url.String()
+		b = strconv.AppendQuote(b, u)
+		if i+1 < len(up.bus) {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, ']')
+	return string(b), nil
 }
 
 func (r *Regex) match(s string) bool {
@@ -556,13 +368,12 @@ func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
 	if err := f(&s); err != nil {
 		return err
 	}
-	r.sOriginal = s
-
 	sAnchored := "^(?:" + s + ")$"
 	re, err := regexp.Compile(sAnchored)
 	if err != nil {
 		return fmt.Errorf("cannot build regexp from %q: %w", s, err)
 	}
+	r.sOriginal = s
 	r.re = re
 	return nil
 }
@@ -651,19 +462,17 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var (
-	authConfig   atomic.Pointer[AuthConfig]
-	authUsers    atomic.Pointer[map[string]*UserInfo]
-	authConfigWG sync.WaitGroup
-	stopCh       chan struct{}
-)
+var authConfig atomic.Pointer[AuthConfig]
+var authUsers atomic.Pointer[map[string]*UserInfo]
+var authConfigWG sync.WaitGroup
+var stopCh chan struct{}
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
 func loadAuthConfig() (bool, error) {
-	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
+	data, err := fs.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
@@ -685,22 +494,9 @@ func loadAuthConfig() (bool, error) {
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
-	prevAc := authConfig.Load()
-	if prevAc != nil {
-		metrics.UnregisterSet(prevAc.ms)
-	}
-	metrics.RegisterSet(ac.ms)
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
-	if prevAc != nil {
-		// explicilty unregister metrics, since all summary type metrics
-		// are registered at global state of metrics package
-		// and must be removed from it to release memory.
-		// Metrics must be unregistered only after atomic.Value.Store calls above
-		// Otherwise it may lead to metric gaps, since UnregisterAllMetrics is slow operation
-		prevAc.ms.UnregisterAllMetrics()
-	}
 
 	return true, nil
 }
@@ -710,13 +506,10 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-	ac := &AuthConfig{
-		ms: metrics.NewSet(),
-	}
-	if err = yaml.UnmarshalStrict(data, ac); err != nil {
+	var ac AuthConfig
+	if err = yaml.UnmarshalStrict(data, &ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
-
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		if ui.Username != "" {
@@ -728,39 +521,29 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if ui.BearerToken != "" {
 			return nil, fmt.Errorf("field bearer_token can't be specified for unauthorized_user section")
 		}
-		if ui.AuthToken != "" {
-			return nil, fmt.Errorf("field auth_token can't be specified for unauthorized_user section")
-		}
 		if ui.Name != "" {
 			return nil, fmt.Errorf("field name can't be specified for unauthorized_user section")
 		}
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-
-		metricLabels, err := ui.getMetricLabels()
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
-		}
-		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
-		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
-		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = ac.ms.NewCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
-
-		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
+		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
-			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
+			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
-		ui.rt = rt
+		ui.httpTransport = tr
 	}
-	return ac, nil
+	return &ac, nil
 }
 
 func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
@@ -771,80 +554,64 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		ats, err := getAuthTokens(ui.AuthToken, ui.BearerToken, ui.Username, ui.Password)
-		if err != nil {
-			return nil, err
+		if ui.BearerToken == "" && ui.Username == "" {
+			return nil, fmt.Errorf("either bearer_token or username must be set")
 		}
-		for _, at := range ats {
-			if uiOld := byAuthToken[at]; uiOld != nil {
-				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
-					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
-			}
+		if ui.BearerToken != "" && ui.Username != "" {
+			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
 		}
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+		}
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
+		}
+
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
-		metricLabels, err := ui.getMetricLabels()
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
+		name := ui.name()
+		if ui.BearerToken != "" {
+			if ui.Password != "" {
+				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		}
+		if ui.Username != "" {
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
-		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
-		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
-		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
-		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
+		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
-			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
+			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
-		ui.rt = rt
+		ui.httpTransport = tr
 
-		for _, at := range ats {
-			byAuthToken[at] = ui
-		}
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
-var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
-
-func (ui *UserInfo) getMetricLabels() (string, error) {
-	name := ui.name()
-	if len(name) == 0 && len(ui.MetricLabels) == 0 {
-		// fast path
-		return "", nil
-	}
-	labels := make([]string, 0, len(ui.MetricLabels)+1)
-	if len(name) > 0 {
-		labels = append(labels, fmt.Sprintf(`username=%q`, name))
-	}
-	for k, v := range ui.MetricLabels {
-		if !labelNameRegexp.MatchString(k) {
-			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
-		}
-		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
-	}
-	sort.Strings(labels)
-	labelsStr := "{" + strings.Join(labels, ",") + "}"
-	return labelsStr, nil
-}
-
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
 	dropSrcPathPrefixParts := 0
-	discoverBackendIPs := *discoverBackendIPsGlobal
 	if ui.URLPrefix != nil {
-		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
+		if err := ui.URLPrefix.sanitize(); err != nil {
 			return err
 		}
 		if ui.RetryStatusCodes != nil {
@@ -856,35 +623,30 @@ func (ui *UserInfo) initURLs() error {
 		if ui.DropSrcPathPrefixParts != nil {
 			dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
 		}
-		if ui.DiscoverBackendIPs != nil {
-			discoverBackendIPs = *ui.DiscoverBackendIPs
-		}
 		ui.URLPrefix.retryStatusCodes = retryStatusCodes
 		ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
-		ui.URLPrefix.discoverBackendIPs = discoverBackendIPs
 		if err := ui.URLPrefix.setLoadBalancingPolicy(loadBalancingPolicy); err != nil {
 			return err
 		}
 	}
 	if ui.DefaultURL != nil {
-		if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil {
+		if err := ui.DefaultURL.sanitize(); err != nil {
 			return err
 		}
 	}
 	for _, e := range ui.URLMaps {
-		if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 && len(e.SrcQueryArgs) == 0 && len(e.SrcHeaders) == 0 {
-			return fmt.Errorf("missing `src_paths`, `src_hosts`, `src_query_args` and `src_headers` in `url_map`")
+		if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 {
+			return fmt.Errorf("missing `src_paths` and `src_hosts` in `url_map`")
 		}
 		if e.URLPrefix == nil {
 			return fmt.Errorf("missing `url_prefix` in `url_map`")
 		}
-		if err := e.URLPrefix.sanitizeAndInitialize(); err != nil {
+		if err := e.URLPrefix.sanitize(); err != nil {
 			return err
 		}
 		rscs := retryStatusCodes
 		lbp := loadBalancingPolicy
 		dsp := dropSrcPathPrefixParts
-		dbd := discoverBackendIPs
 		if e.RetryStatusCodes != nil {
 			rscs = e.RetryStatusCodes
 		}
@@ -894,18 +656,14 @@ func (ui *UserInfo) initURLs() error {
 		if e.DropSrcPathPrefixParts != nil {
 			dsp = *e.DropSrcPathPrefixParts
 		}
-		if e.DiscoverBackendIPs != nil {
-			dbd = *e.DiscoverBackendIPs
-		}
 		e.URLPrefix.retryStatusCodes = rscs
 		if err := e.URLPrefix.setLoadBalancingPolicy(lbp); err != nil {
 			return err
 		}
 		e.URLPrefix.dropSrcPathPrefixParts = dsp
-		e.URLPrefix.discoverBackendIPs = dbd
 	}
 	if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
-		return fmt.Errorf("missing `url_prefix` or `url_map`")
+		return fmt.Errorf("missing `url_prefix`")
 	}
 	return nil
 }
@@ -918,100 +676,39 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		h := xxhash.Sum64([]byte(ui.BearerToken))
-		return fmt.Sprintf("bearer_token:hash:%016X", h)
-	}
-	if ui.AuthToken != "" {
-		h := xxhash.Sum64([]byte(ui.AuthToken))
-		return fmt.Sprintf("auth_token:hash:%016X", h)
+		return "bearer_token"
 	}
 	return ""
 }
 
-func getAuthTokens(authToken, bearerToken, username, password string) ([]string, error) {
-	if authToken != "" {
-		if bearerToken != "" {
-			return nil, fmt.Errorf("bearer_token cannot be specified if auth_token is set")
-		}
-		if username != "" || password != "" {
-			return nil, fmt.Errorf("username and password cannot be specified if auth_token is set")
-		}
-		at := getHTTPAuthToken(authToken)
-		return []string{at}, nil
-	}
+func getAuthTokens(bearerToken, username, password string) (string, string) {
 	if bearerToken != "" {
-		if username != "" || password != "" {
-			return nil, fmt.Errorf("username and password cannot be specified if bearer_token is set")
-		}
 		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getHTTPAuthBearerToken(bearerToken)
-		at2 := getHTTPAuthBasicToken(bearerToken, "")
-		return []string{at1, at2}, nil
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
 	}
-	if username != "" {
-		at := getHTTPAuthBasicToken(username, password)
-		return []string{at}, nil
+	at := getAuthToken("", username, password)
+	return at, at
+}
+
+func getAuthToken(bearerToken, username, password string) string {
+	if bearerToken != "" {
+		return "Bearer " + bearerToken
 	}
-	return nil, fmt.Errorf("missing authorization options; bearer_token or username must be set")
-}
-
-func getHTTPAuthToken(authToken string) string {
-	return "http_auth:" + authToken
-}
-
-func getHTTPAuthBearerToken(bearerToken string) string {
-	return "http_auth:Bearer " + bearerToken
-}
-
-func getHTTPAuthBasicToken(username, password string) string {
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "http_auth:Basic " + token64
+	return "Basic " + token64
 }
 
-var defaultHeaderNames = []string{"Authorization"}
-
-func getAuthTokensFromRequest(r *http.Request) []string {
-	var ats []string
-
-	// Obtain possible auth tokens from one of the allowed auth headers
-	headerNames := *httpAuthHeader
-	if len(headerNames) == 0 {
-		headerNames = defaultHeaderNames
-	}
-	for _, headerName := range headerNames {
-		if ah := r.Header.Get(headerName); ah != "" {
-			if strings.HasPrefix(ah, "Token ") {
-				// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-				// See https://docs.influxdata.com/influxdb/v2.0/api/
-				ah = strings.Replace(ah, "Token", "Bearer", 1)
-			}
-			at := "http_auth:" + ah
-			ats = append(ats, at)
-		}
-	}
-
-	return ats
-}
-
-func (up *URLPrefix) sanitizeAndInitialize() error {
-	for i, bu := range up.busOriginal {
-		puNew, err := sanitizeURLPrefix(bu)
+func (up *URLPrefix) sanitize() error {
+	for _, bu := range up.bus {
+		puNew, err := sanitizeURLPrefix(bu.url)
 		if err != nil {
 			return err
 		}
-		up.busOriginal[i] = puNew
+		bu.url = puNew
 	}
-
-	// Initialize up.bus
-	bus := make([]*backendURL, len(up.busOriginal))
-	for i, bu := range up.busOriginal {
-		bus[i] = &backendURL{
-			url: bu,
-		}
-	}
-	up.bus.Store(&bus)
-
 	return nil
 }
 

app/vmauth/auth_config_test.go

@@ -4,11 +4,10 @@ import (
 	"bytes"
 	"fmt"
 	"net/url"
+	"regexp"
 	"testing"
 
 	"gopkg.in/yaml.v2"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 func TestParseAuthConfigFailure(t *testing.T) {
@@ -18,9 +17,9 @@ func TestParseAuthConfigFailure(t *testing.T) {
 		if err != nil {
 			return
 		}
-		users, err := parseAuthConfigUsers(ac)
+		_, err = parseAuthConfigUsers(ac)
 		if err == nil {
-			t.Fatalf("expecting non-nil error; got %v", users)
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
@@ -89,22 +88,6 @@ users:
   url_prefix: []
 `)
 
-	// auth_token and username in a single config
-	f(`
-users:
-- auth_token: foo
-  username: bbb
-  url_prefix: http://foo.bar
-`)
-
-	// auth_token and bearer_token in a single config
-	f(`
-users:
-- auth_token: foo
-  bearer_token: bbb
-  url_prefix: http://foo.bar
-`)
-
 	// Username and bearer_token in a single config
 	f(`
 users:
@@ -209,7 +192,7 @@ users:
   - url_prefix: http://foobar
 `)
 
-	// Invalid regexp in src_paths
+	// Invalid regexp in src_path.
 	f(`
 users:
 - username: a
@@ -227,24 +210,6 @@ users:
     url_prefix: http://foobar
 `)
 
-	// Invalid src_query_args
-	f(`
-users:
-- username: a
-  url_map:
-  - src_query_args: abc
-    url_prefix: http://foobar
-`)
-
-	// Invalid src_headers
-	f(`
-users:
-- username: a
-  url_map:
-  - src_headers: abc
-    url_prefix: http://foobar
-`)
-
 	// Invalid headers in url_map (missing ':')
 	f(`
 users:
@@ -264,14 +229,6 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
-`)
-	// Invalid metric label name
-	f(`
-users:
-- username: foo
-  url_prefix: http://foo.bar
-  metric_labels:
-    not-prometheus-compatible: value
 `)
 }
 
@@ -292,9 +249,8 @@ func TestParseAuthConfigSuccess(t *testing.T) {
 		}
 	}
 
-	insecureSkipVerifyTrue := true
-
 	// Single user
+	insecureSkipVerifyTrue := true
 	f(`
 users:
 - username: foo
@@ -303,7 +259,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", "bar"): {
+		getAuthToken("", "foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -312,58 +268,31 @@ users:
 		},
 	})
 
-	// Single user with auth_token
-	f(`
-users:
-- auth_token: foo
-  url_prefix: https://aaa:343/bbb
-  max_concurrent_requests: 5
-  tls_insecure_skip_verify: true
-  tls_server_name: "foo.bar"
-  tls_ca_file: "foo/bar"
-  tls_cert_file: "foo/baz"
-  tls_key_file: "foo/foo"
-`, map[string]*UserInfo{
-		getHTTPAuthToken("foo"): {
-			AuthToken:             "foo",
-			URLPrefix:             mustParseURL("https://aaa:343/bbb"),
-			MaxConcurrentRequests: 5,
-			TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
-			TLSServerName:         "foo.bar",
-			TLSCAFile:             "foo/bar",
-			TLSCertFile:           "foo/baz",
-			TLSKeyFile:            "foo/foo",
-		},
-	})
-
 	// Multiple url_prefix entries
 	insecureSkipVerifyFalse := false
-	discoverBackendIPsTrue := true
 	f(`
 users:
 - username: foo
   password: bar
   url_prefix:
   - http://node1:343/bbb
-  - http://srv+node2:343/bbb
+  - http://node2:343/bbb
   tls_insecure_skip_verify: false
   retry_status_codes: [500, 501]
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
-  discover_backend_ips: true
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", "bar"): {
+		getAuthToken("", "foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
 				"http://node1:343/bbb",
-				"http://srv+node2:343/bbb",
+				"http://node2:343/bbb",
 			}),
 			TLSInsecureSkipVerify:  &insecureSkipVerifyFalse,
 			RetryStatusCodes:       []int{500, 501},
 			LoadBalancingPolicy:    "first_available",
 			DropSrcPathPrefixParts: intp(1),
-			DiscoverBackendIPs:     &discoverBackendIPsTrue,
 		},
 	})
 
@@ -373,49 +302,19 @@ users:
 - username: foo
   url_prefix: http://foo
 - username: bar
-  url_prefix: https://bar/x/
+  url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo", ""): {
+		getAuthToken("", "foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getHTTPAuthBasicToken("bar", ""): {
+		getAuthToken("", "bar", ""): {
 			Username:  "bar",
-			URLPrefix: mustParseURL("https://bar/x/"),
+			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
 
 	// non-empty URLMap
-	sharedUserInfo := &UserInfo{
-		BearerToken: "foo",
-		URLMaps: []URLMap{
-			{
-				SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
-				URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
-			},
-			{
-				SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
-				SrcPaths: getRegexs([]string{"/api/v1/write"}),
-				SrcQueryArgs: []*QueryArg{
-					mustNewQueryArg("foo=b.+ar"),
-					mustNewQueryArg("baz=~.*x=y.+"),
-				},
-				SrcHeaders: []*Header{
-					mustNewHeader("'TenantID: 345'"),
-				},
-				URLPrefix: mustParseURLs([]string{
-					"http://vminsert1/insert/0/prometheus",
-					"http://vminsert2/insert/0/prometheus",
-				}),
-				HeadersConf: HeadersConf{
-					RequestHeaders: []*Header{
-						mustNewHeader("'foo: bar'"),
-						mustNewHeader("'xxx: y'"),
-					},
-				},
-			},
-		},
-	}
 	f(`
 users:
 - bearer_token: foo
@@ -424,18 +323,71 @@ users:
     url_prefix: http://vmselect/select/0/prometheus
   - src_paths: ["/api/v1/write"]
     src_hosts: ["foo\\.bar", "baz:1234"]
-    src_query_args: ['foo=b.+ar', 'baz=~.*x=y.+']
-    src_headers: ['TenantID: 345']
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
     headers:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
-		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
+		getAuthToken("foo", "", ""): {
+			BearerToken: "foo",
+			URLMaps: []URLMap{
+				{
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
+						},
+					},
+				},
+			},
+		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMaps: []URLMap{
+				{
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
+						},
+					},
+				},
+			},
+		},
 	})
-
-	// Multiple users with the same name - this should work, since these users have different passwords
+	// Multiple users with the same name
 	f(`
 users:
 - username: foo-same
@@ -443,20 +395,19 @@ users:
   url_prefix: http://foo
 - username: foo-same
   password: bar
-  url_prefix: https://bar/x
+  url_prefix: https://bar/x///
 `, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo-same", "baz"): {
+		getAuthToken("", "foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getHTTPAuthBasicToken("foo-same", "bar"): {
+		getAuthToken("", "foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
-
 	// with default url
 	f(`
 users:
@@ -473,7 +424,7 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getHTTPAuthBearerToken("foo"): {
+		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -487,9 +438,15 @@ users:
 						"http://vminsert2/insert/0/prometheus",
 					}),
 					HeadersConf: HeadersConf{
-						RequestHeaders: []*Header{
-							mustNewHeader("'foo: bar'"),
-							mustNewHeader("'xxx: y'"),
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -499,7 +456,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getHTTPAuthBasicToken("foo", ""): {
+		getAuthToken("", "foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
@@ -513,9 +470,15 @@ users:
 						"http://vminsert2/insert/0/prometheus",
 					}),
 					HeadersConf: HeadersConf{
-						RequestHeaders: []*Header{
-							mustNewHeader("'foo: bar'"),
-							mustNewHeader("'xxx: y'"),
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -527,41 +490,6 @@ users:
 		},
 	})
 
-	// With metric_labels
-	f(`
-users:
-- username: foo-same
-  password: baz
-  url_prefix: http://foo
-  metric_labels:
-    dc: eu
-    team: dev
-- username: foo-same
-  password: bar
-  url_prefix: https://bar/x
-  metric_labels:
-    backend_env: test
-    team: accounting
-`, map[string]*UserInfo{
-		getHTTPAuthBasicToken("foo-same", "baz"): {
-			Username:  "foo-same",
-			Password:  "baz",
-			URLPrefix: mustParseURL("http://foo"),
-			MetricLabels: map[string]string{
-				"dc":   "eu",
-				"team": "dev",
-			},
-		},
-		getHTTPAuthBasicToken("foo-same", "bar"): {
-			Username:  "foo-same",
-			Password:  "bar",
-			URLPrefix: mustParseURL("https://bar/x"),
-			MetricLabels: map[string]string{
-				"backend_env": "test",
-				"team":        "accounting",
-			},
-		},
-	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -588,96 +516,16 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getHTTPAuthBasicToken("foo", "bar")]
-	if !isSetBool(ui.TLSInsecureSkipVerify, true) {
+	ui := m[getAuthToken("", "foo", "bar")]
+	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
 
-	if !isSetBool(ac.UnauthorizedUser.TLSInsecureSkipVerify, false) {
+	if !isSetBool(ac.UnauthorizedUser.TLSInsecureSkipVerify, false) || ac.UnauthorizedUser.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for unauthorized_user")
 	}
 }
 
-func TestUserInfoGetMetricLabels(t *testing.T) {
-	t.Run("empty-labels", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "user1",
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-username", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "user1",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-name", func(t *testing.T) {
-		ui := &UserInfo{
-			Name:        "user1",
-			BearerToken: "abc",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("non-empty-bearer-token", func(t *testing.T) {
-		ui := &UserInfo{
-			BearerToken: "abc",
-			MetricLabels: map[string]string{
-				"env":        "prod",
-				"datacenter": "dc1",
-			},
-		}
-		labels, err := ui.getMetricLabels()
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
-		if labels != labelsExpected {
-			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
-		}
-	})
-	t.Run("invalid-label", func(t *testing.T) {
-		ui := &UserInfo{
-			Username: "foo",
-			MetricLabels: map[string]string{
-				",{": "aaaa",
-			},
-		}
-		_, err := ui.getMetricLabels()
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	})
-}
-
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false
@@ -685,74 +533,13 @@ func isSetBool(boolP *bool, expectedValue bool) bool {
 	return *boolP == expectedValue
 }
 
-func TestGetLeastLoadedBackendURL(t *testing.T) {
-	up := mustParseURLs([]string{
-		"http://node1:343",
-		"http://node2:343",
-		"http://node3:343",
-	})
-	up.loadBalancingPolicy = "least_loaded"
-
-	fn := func(ns ...int) {
-		t.Helper()
-		bus := up.bus.Load()
-		pbus := *bus
-		for i, b := range pbus {
-			got := int(b.concurrentRequests.Load())
-			exp := ns[i]
-			if got != exp {
-				t.Fatalf("expected %q to have %d concurrent requests; got %d instead", b.url, exp, got)
-			}
-		}
-	}
-
-	up.getBackendURL()
-	fn(0, 1, 0)
-	up.getBackendURL()
-	fn(0, 1, 1)
-	up.getBackendURL()
-	fn(1, 1, 1)
-
-	up.getBackendURL()
-	up.getBackendURL()
-	fn(1, 2, 2)
-
-	bus := up.bus.Load()
-	pbus := *bus
-	pbus[0].concurrentRequests.Add(2)
-	pbus[2].concurrentRequests.Add(5)
-	fn(3, 2, 7)
-
-	up.getBackendURL()
-	fn(3, 3, 7)
-
-	up.getBackendURL()
-	fn(3, 4, 7)
-
-	up.getBackendURL()
-	fn(4, 4, 7)
-
-	up.getBackendURL()
-	fn(5, 4, 7)
-
-	up.getBackendURL()
-	fn(5, 5, 7)
-
-	up.getBackendURL()
-	fn(6, 5, 7)
-
-	up.getBackendURL()
-	fn(6, 6, 7)
-
-	up.getBackendURL()
-	up.getBackendURL()
-	fn(7, 7, 7)
-}
-
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {
-		sps = append(sps, mustNewRegex(path))
+		sps = append(sps, &Regex{
+			sOriginal: path,
+			re:        regexp.MustCompile("^(?:" + path + ")$"),
+		})
 	}
 	return sps
 }
@@ -784,7 +571,6 @@ func mustParseURL(u string) *URLPrefix {
 
 func mustParseURLs(us []string) *URLPrefix {
 	bus := make([]*backendURL, len(us))
-	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
@@ -793,43 +579,12 @@ func mustParseURLs(us []string) *URLPrefix {
 		bus[i] = &backendURL{
 			url: pu,
 		}
-		urls[i] = pu
 	}
-	up := &URLPrefix{}
-	if len(us) == 1 {
-		up.vOriginal = us[0]
-	} else {
-		up.vOriginal = us
+	return &URLPrefix{
+		bus: bus,
 	}
-	up.bus.Store(&bus)
-	up.busOriginal = urls
-	return up
 }
 
 func intp(n int) *int {
 	return &n
 }
-
-func mustNewRegex(s string) *Regex {
-	var re Regex
-	if err := yaml.Unmarshal([]byte(s), &re); err != nil {
-		logger.Panicf("cannot unmarshal regex %q: %s", s, err)
-	}
-	return &re
-}
-
-func mustNewQueryArg(s string) *QueryArg {
-	var qa QueryArg
-	if err := yaml.Unmarshal([]byte(s), &qa); err != nil {
-		logger.Panicf("cannot unmarshal query arg filter %q: %s", s, err)
-	}
-	return &qa
-}
-
-func mustNewHeader(s string) *Header {
-	var h Header
-	if err := yaml.Unmarshal([]byte(s), &h); err != nil {
-		logger.Panicf("cannot unmarshal header filter %q: %s", s, err)
-	}
-	return &h
-}