app/vlselect/logsql: skip rows without _stream reference

_stream field can be empty for the recently ingested rows because respective entry in indexdb is not yet searchable as it haven't been flushed to storage yet.
This change just skips such items in the output response to make it more consistent.

See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6042
Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,35 @@
+### Describe Your Changes
+
+Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
+
+### Checklist
+
+The following checks are mandatory:
+
+- [ ] I have read the [Contributing Guidelines](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/CONTRIBUTING.md)
+- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
+- [ ] Tests are passing locally. Use `make test` to run all tests locally.
+- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.
+
+Further checks are optional for External Contributions:
+
+- [ ] Include a link to the GitHub issue in the commit message, if issue exists.
+- [ ] Mention the change in the [Changelog](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md). Explain what has changed and why. If there is a related issue or documentation change - link them as well.
+
+  Tips for writing a good changelog message::
+
+    * Write a human-readable changelog message that describes the problem and solution.
+    * Include a link to the issue or pull request in your changelog message.
+    * Use specific language identifying the fix, such as an error message, metric name, or flag name.
+    * Provide a link to the relevant documentation for any new features you add or modify.
+
+- [ ] After your pull request is merged, please add a message to the issue with instructions for how to test the fix or try the feature you added. Here is an [example](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048#issuecomment-1546453726)
+- [ ] Do not close the original issue before the change is released. Please note, in some cases Github can automatically close the issue once PR is merged. Re-open the issue in such case.
+- [ ] If the change somehow affects public interfaces (a new flag was added or updated, or some behavior has changed) - add the corresponding change to documentation.
+
+
+Examples of good changelog messages:
+
+1. FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent.html#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
+
+2. BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).

.github/workflows/check-licenses.yml

@@ -25,7 +25,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
         if: ${{ matrix.language == 'go' }}
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -75,7 +75,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +86,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +100,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -102,7 +102,7 @@ jobs:
           cache: false
 
       - name: Cache Go artifacts
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -115,6 +115,6 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.txt

.github/workflows/sync-docs.yml

@@ -6,9 +6,6 @@ on:
     paths:
       - 'docs/**'
   workflow_dispatch: {}
-env:
-  PAGEFIND_VERSION: "1.0.4"
-  HUGO_VERSION: "latest"
 permissions:
   contents: read  # This is required for actions/checkout and to commit back image update
   deployments: write
@@ -27,16 +24,6 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
-      - uses: peaceiris/actions-hugo@v2
-        with:
-          hugo-version: ${{env.HUGO_VERSION}}
-          extended: true
-      - name: Install PageFind #install the static search engine for index build
-        uses: supplypike/setup-bin@v3
-        with:
-          uri: "https://github.com/CloudCannon/pagefind/releases/download/v${{env.PAGEFIND_VERSION}}/pagefind-v${{env.PAGEFIND_VERSION}}-x86_64-unknown-linux-musl.tar.gz"
-          name: "pagefind"
-          version: ${{env.PAGEFIND_VERSION}}
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v5
         with:
@@ -51,13 +38,11 @@ jobs:
           calculatedSha=$(git rev-parse --short ${{ github.sha }})
           echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
         working-directory: main
-
       - name: update code and commit
         run: |
           rm -rf content
           cp -r ../main/docs content
           make clean-after-copy
-          make build-search-index
           git config --global user.name "${{ steps.import-gpg.outputs.email }}"
           git config --global user.email "${{ steps.import-gpg.outputs.email }}"
           git add .

.github/workflows/wiki.yml

@@ -1,33 +0,0 @@
-name: wiki
-on:
-  push:
-    paths:
-      - 'docs/*'
-    branches:
-      - master
-permissions:
-  contents: read
-
-jobs:
-  build:
-    permissions:
-      contents: write  # for Git to git push
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: publish
-        shell: bash
-        env:
-          TOKEN: ${{secrets.CI_TOKEN}}
-        run: |
-          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
-          cp -r docs/* wiki
-          cd wiki
-          git config --local user.email "info@victoriametrics.com"
-          git config --local user.name "Vika"
-          git add .
-          git commit -m "update wiki pages"
-          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
-          git push "${remote_repo}"
-          cd ..
-          rm -rf wiki

.gitignore

@@ -22,3 +22,4 @@ Gemfile.lock
 /_site
 _site
 *.tmp
+/docs/.jekyll-metadata

CONTRIBUTING.md

@@ -14,3 +14,8 @@ We are open to third-party pull requests provided they follow [KISS design princ
 - Avoid automated decisions, which may hurt cluster availability, consistency or performance.
 
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
+
+Before sending a pull request please check the following:
+- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
+- [ ] Tests are passing locally. Use `make test` to run all tests locally.
+- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2023 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,6 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
+MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
 MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
@@ -178,7 +178,8 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64
+	victoria-metrics-openbsd-amd64 \
+	victoria-metrics-windows-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
@@ -465,7 +466,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.20
+	go mod tidy -compat=1.21
 	go mod vendor
 
 app-local:
@@ -491,7 +492,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.55.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...

SECURITY.md

@@ -2,13 +2,17 @@
 
 ## Supported Versions
 
+The following versions of VictoriaMetrics receive regular security fixes:
+
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.93.x LTS release | :white_check_mark: |
-| v1.87.x LTS release | :white_check_mark: |
+| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
+See [this page](https://victoriametrics.com/security/) for more details.
+
 ## Reporting a Vulnerability
 
 Please report any security issues to security@victoriametrics.com

app/victoria-logs/deployment/Dockerfile

@@ -1,7 +1,7 @@
 ARG base_image
 FROM $base_image
 
-EXPOSE 8428
+EXPOSE 9428
 
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG src_binary

app/victoria-logs/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -22,11 +21,10 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -34,27 +32,31 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
-	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":9428"}
+	}
+	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-logs/multiarch/Dockerfile

@@ -6,7 +6,7 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8428
+EXPOSE 9428
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG TARGETARCH
 COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/main.go

@@ -26,12 +26,12 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
+		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/#deduplication")
 	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -48,7 +48,6 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -67,30 +66,37 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8428"}
+	}
+	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
+
 	startSelfScraper()
 
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
 	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
-	vminsert.Stop()
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	vminsert.Stop()
 
 	vmstorage.Stop()
 	vmselect.Stop()
@@ -122,6 +128,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,11 +7,13 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -37,11 +39,13 @@ const (
 )
 
 const (
-	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
-	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
-	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
+
+	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -54,15 +58,15 @@ var (
 )
 
 type test struct {
-	Name             string     `json:"name"`
-	Data             []string   `json:"data"`
-	InsertQuery      string     `json:"insert_query"`
-	Query            []string   `json:"query"`
-	ResultMetrics    []Metric   `json:"result_metrics"`
-	ResultSeries     Series     `json:"result_series"`
-	ResultQuery      Query      `json:"result_query"`
-	ResultQueryRange QueryRange `json:"result_query_range"`
-	Issue            string     `json:"issue"`
+	Name                     string   `json:"name"`
+	Data                     []string `json:"data"`
+	InsertQuery              string   `json:"insert_query"`
+	Query                    []string `json:"query"`
+	ResultMetrics            []Metric `json:"result_metrics"`
+	ResultSeries             Series   `json:"result_series"`
+	ResultQuery              Query    `json:"result_query"`
+	Issue                    string   `json:"issue"`
+	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
 }
 
 type Metric struct {
@@ -80,42 +84,90 @@ type Series struct {
 	Status string              `json:"status"`
 	Data   []map[string]string `json:"data"`
 }
+
 type Query struct {
-	Status string    `json:"status"`
-	Data   QueryData `json:"data"`
-}
-type QueryData struct {
-	ResultType string            `json:"resultType"`
-	Result     []QueryDataResult `json:"result"`
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string          `json:"resultType"`
+		Result     json.RawMessage `json:"result"`
+	} `json:"data"`
 }
 
-type QueryDataResult struct {
-	Metric map[string]string `json:"metric"`
-	Value  []interface{}     `json:"value"`
+const rtVector, rtMatrix = "vector", "matrix"
+
+func (q *Query) metrics() ([]Metric, error) {
+	switch q.Data.ResultType {
+	case rtVector:
+		var r QueryInstant
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	case rtMatrix:
+		var r QueryRange
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	default:
+		return nil, fmt.Errorf("unknown result type %q", q.Data.ResultType)
+	}
 }
 
-func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
-	type plain QueryDataResult
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+type QueryInstant struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TV     [2]interface{}    `json:"value"`
+	} `json:"result"`
+}
+
+func (q QueryInstant) metrics() ([]Metric, error) {
+	result := make([]Metric, len(q.Result))
+	for i, res := range q.Result {
+		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
+		}
+		var m Metric
+		m.Metric = res.Labels
+		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
+		m.Values = append(m.Values, f)
+		result[i] = m
+	}
+	return result, nil
 }
 
 type QueryRange struct {
-	Status string         `json:"status"`
-	Data   QueryRangeData `json:"data"`
-}
-type QueryRangeData struct {
-	ResultType string                 `json:"resultType"`
-	Result     []QueryRangeDataResult `json:"result"`
+	Result []struct {
+		Metric map[string]string `json:"metric"`
+		Values [][]interface{}   `json:"values"`
+	} `json:"result"`
 }
 
-type QueryRangeDataResult struct {
-	Metric map[string]string `json:"metric"`
-	Values [][]interface{}   `json:"values"`
+func (q QueryRange) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range q.Result {
+		var m Metric
+		for _, tv := range res.Values {
+			f, err := strconv.ParseFloat(tv[1].(string), 64)
+			if err != nil {
+				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
+			}
+			m.Values = append(m.Values, f)
+			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
+		}
+		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
+			return nil, fmt.Errorf("metric %v contains no values", res)
+		}
+		m.Metric = q.Result[i].Metric
+		result = append(result, m)
+	}
+	return result, nil
 }
 
-func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
-	type plain QueryRangeDataResult
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+func (q *Query) UnmarshalJSON(b []byte) error {
+	type plain Query
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(q))
 }
 
 func TestMain(m *testing.M) {
@@ -132,7 +184,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddr, false, requestHandler)
+	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -178,7 +230,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	if err := httpserver.Stop(*httpListenAddrs); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -189,14 +241,18 @@ func tearDown() {
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
+	time.Sleep(500 * time.Millisecond)
 	vmstorage.Storage.DebugFlush()
-	time.Sleep(1 * time.Second)
+	time.Sleep(1500 * time.Millisecond)
 	t.Run("read", testRead)
 }
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
 		for _, test := range readIn("prometheus", t, insertionTime) {
+			if test.Data == nil {
+				continue
+			}
 			s := newSuite(t)
 			r := testutil.WriteRequest{}
 			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
@@ -209,6 +265,14 @@ func testWrite(t *testing.T) {
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
+	t.Run("csv", func(t *testing.T) {
+		for _, test := range readIn("csv", t, insertionTime) {
+			if test.Data == nil {
+				continue
+			}
+			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
+		}
+	})
 
 	t.Run("influxdb", func(t *testing.T) {
 		for _, x := range readIn("influxdb", t, insertionTime) {
@@ -250,7 +314,7 @@ func testWrite(t *testing.T) {
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
 			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
@@ -261,7 +325,12 @@ func testRead(t *testing.T) {
 						if test.Issue != "" {
 							test.Issue = "\nRegression in " + test.Issue
 						}
-						switch true {
+						switch {
+						case strings.HasPrefix(q, "/api/v1/export/csv"):
+							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
+							if len(data) == test.ExpectedResultLinesCount {
+								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
+							}
 						case strings.HasPrefix(q, "/api/v1/export"):
 							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
 								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
@@ -272,17 +341,19 @@ func testRead(t *testing.T) {
 							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
 								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
 							}
-						case strings.HasPrefix(q, "/api/v1/query_range"):
-							queryResult := QueryRange{}
-							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
-								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
-							}
 						case strings.HasPrefix(q, "/api/v1/query"):
 							queryResult := Query{}
 							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
-								t.Fatalf("Query. %s fails with error: %s.%s", q, err, test.Issue)
+							gotMetrics, err := queryResult.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse query response: %s", err)
+							}
+							expMetrics, err := test.ResultQuery.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse expected response: %s", err)
+							}
+							if err := checkMetricsResult(gotMetrics, expMetrics); err != nil {
+								t.Fatalf("%q fails with error %s.%s", q, err, test.Issue)
 							}
 						default:
 							t.Fatalf("unsupported read query %s", q)
@@ -298,7 +369,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	t.Helper()
 	s := newSuite(t)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -360,6 +431,7 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
+
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -372,6 +444,20 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
+func httpReadData(t *testing.T, address, query string) []byte {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	s.equalInt(resp.StatusCode, 200)
+	data, err := io.ReadAll(resp.Body)
+	s.noError(err)
+	return data
+}
+
 func checkMetricsResult(got, want []Metric) error {
 	for _, r := range append([]Metric(nil), got...) {
 		want = removeIfFoundMetrics(r, want)
@@ -417,60 +503,6 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
-func checkQueryResult(got, want Query) error {
-	if got.Status != want.Status {
-		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
-	}
-	if got.Data.ResultType != want.Data.ResultType {
-		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
-	}
-	wantData := append([]QueryDataResult(nil), want.Data.Result...)
-	for _, r := range got.Data.Result {
-		wantData = removeIfFoundQueryData(r, wantData)
-	}
-	if len(wantData) > 0 {
-		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
-	}
-	return nil
-}
-
-func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
-	for i, item := range contains {
-		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
-			contains[i] = contains[len(contains)-1]
-			return contains[:len(contains)-1]
-		}
-	}
-	return contains
-}
-
-func checkQueryRangeResult(got, want QueryRange) error {
-	if got.Status != want.Status {
-		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
-	}
-	if got.Data.ResultType != want.Data.ResultType {
-		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
-	}
-	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
-	for _, r := range got.Data.Result {
-		wantData = removeIfFoundQueryRangeData(r, wantData)
-	}
-	if len(wantData) > 0 {
-		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
-	}
-	return nil
-}
-
-func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
-	for i, item := range contains {
-		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
-			contains[i] = contains[len(contains)-1]
-			return contains[:len(contains)-1]
-		}
-	}
-	return contains
-}
-
 type suite struct{ t *testing.T }
 
 func newSuite(t *testing.T) *suite { return &suite{t: t} }
@@ -498,3 +530,73 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
+
+func TestImportJSONLines(t *testing.T) {
+	f := func(labelsCount, labelLen int) {
+		t.Helper()
+
+		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
+		line := generateJSONLine(labelsCount, labelLen)
+		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
+		}
+		if resp.StatusCode != 204 {
+			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
+		}
+	}
+
+	// labels with various lengths
+	for i := 0; i < 500; i++ {
+		f(10, i*5)
+	}
+
+	// Too many labels
+	f(1000, 100)
+
+	// Too long labels
+	f(1, 100_000)
+	f(10, 100_000)
+	f(10, 10_000)
+}
+
+func generateJSONLine(labelsCount, labelLen int) string {
+	m := make(map[string]string, labelsCount)
+	m["__name__"] = generateSizedRandomString(labelLen)
+	for j := 1; j < labelsCount; j++ {
+		labelName := generateSizedRandomString(labelLen)
+		labelValue := generateSizedRandomString(labelLen)
+		m[labelName] = labelValue
+	}
+
+	type jsonLine struct {
+		Metric     map[string]string `json:"metric"`
+		Values     []float64         `json:"values"`
+		Timestamps []int64           `json:"timestamps"`
+	}
+	line := &jsonLine{
+		Metric:     m,
+		Values:     []float64{1.34},
+		Timestamps: []int64{time.Now().UnixNano() / 1e6},
+	}
+	data, err := json.Marshal(&line)
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal JSON: %w", err))
+	}
+	data = append(data, '\n')
+	return string(data)
+}
+
+const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
+
+func generateSizedRandomString(size int) string {
+	dst := make([]byte, size)
+	for i := range dst {
+		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
+	}
+	return string(dst)
+}

app/victoria-metrics/self_scraper.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -49,16 +50,8 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	var currentTimestamp int64
-	for {
-		select {
-		case <-selfScraperStopCh:
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			currentTimestamp = currentTime.UnixNano() / 1e6
-		}
+	f := func(currentTime time.Time, sendStaleMarkers bool) {
+		currentTimestamp := currentTime.UnixNano() / 1e6
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -83,12 +76,27 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			mr.Value = r.Value
+			if sendStaleMarkers {
+				mr.Value = decimal.StaleNaN
+			} else {
+				mr.Value = r.Value
+			}
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
+	for {
+		select {
+		case <-selfScraperStopCh:
+			f(time.Now(), true)
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			f(currentTime, false)
+		}
+	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
@@ -98,7 +106,7 @@ func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
-	lb.Name = bytesutil.ToUnsafeBytes(key)
-	lb.Value = bytesutil.ToUnsafeBytes(value)
+	lb.Name = key
+	lb.Value = value
 	return dst
 }

app/victoria-metrics/testdata/csv/basic.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export",
+  "data": [
+    "rfc3339,4,{TIME_MS}",
+    "rfc3339milli,6,{TIME_MS}",
+    "ts,8,{TIME_MS}",
+    "tsms,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
+  "query": [
+    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export with extra_labels",
+  "data": [
+    "location-1,4,{TIME_MS}",
+    "location-2,6,{TIME_MS}",
+    "location-3,8,{TIME_MS}",
+    "location-4,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
+  "query": [
+    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -7,7 +7,7 @@
     "not_nan_not_inf;item=y 3 {TIME_S-1m}",
     "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
   "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -6,7 +6,7 @@
     "empty_label_match;foo=bar 2 {TIME_S-1m}",
     "empty_label_match;foo=baz 3 {TIME_S-1m}"],
   "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -8,7 +8,7 @@
     "max_lookback_set 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_set"},"values":[

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -8,7 +8,7 @@
     "max_lookback_unset 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -8,7 +8,7 @@
     "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
   ],
   "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/prometheus/instant-matrix.json

@@ -0,0 +1,12 @@
+{
+  "name": "instant query with look-behind window",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo[5m]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/instant-scalar.json

@@ -0,0 +1,11 @@
+{
+  "name": "instant scalar query",
+  "query": ["/api/v1/query?query=42&time={TIME_S}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"vector",
+        "result":[{"metric":{},"value":["{TIME_S}", "42"]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/issue-5553-too-big-lookback.json

@@ -0,0 +1,13 @@
+{
+  "name": "too big look-behind window",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"},{\"name\":\"issue\",\"value\":\"5553\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo{issue=\"5553\"}[100y]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo", "issue": "5553"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/query-range.json

@@ -0,0 +1,18 @@
+{
+  "name": "query range",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"bar\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}, {\"value\":2,\"timestamp\":\"{TIME_MS-120s}\"}, {\"value\":1,\"timestamp\":\"{TIME_MS-180s}\"}]}]"],
+  "query": ["/api/v1/query_range?query=bar&step=30s&start={TIME_MS-180s}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[
+          {
+            "metric":{"__name__":"bar"},
+            "values":[["{TIME_S-180s}", "1"],["{TIME_S-150s}", "1"],["{TIME_S-120s}", "2"],["{TIME_S-90s}", "2"], ["{TIME_S-60s}", "1"], ["{TIME_S-30s}", "1"], ["{TIME_S}", "1"]]
+          }
+        ]
+      }
+    }
+}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	processLogMessage := func(timestmap int64, fields []logstorage.Field) {}
+	processLogMessage := func(_ int64, _ []logstorage.Field) {}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))

app/vlinsert/loki/loki_json_test.go

@@ -11,7 +11,7 @@ import (
 func TestParseJSONRequestFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
 			t.Fatalf("unexpected call to parseJSONRequest callback!")
 		})
 		if err == nil {

app/vlinsert/loki/loki_json_timing_test.go

@@ -27,7 +27,7 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
+			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -29,7 +29,7 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
+			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlselect/logsql/logsql.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
@@ -17,13 +18,18 @@ var (
 )
 
 // ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}, cancel func()) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
 
 	qStr := r.FormValue("query")
 	q, err := logstorage.ParseQuery(qStr)
@@ -34,7 +40,7 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
 
 	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN())
+	sw.Init(w, maxSortBufferSize.IntN(), limit)
 	tenantIDs := []logstorage.TenantID{tenantID}
 	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
 		if len(columns) == 0 {
@@ -42,11 +48,36 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 		}
 		rowsCount := len(columns[0].Values)
 
+		// skip entries with empty _stream column
+		// _stream is empty in case indexdb entry was not flushed to the storage yet
+		// skipping such entries makes the result more consistent
+		streamCol := 0
+
+		// fast path
+		// _stream column is a built-in column and it is always supposed to be at the same position
+		if len(columns) >= 2 && columns[1].Name == "_stream" {
+			streamCol = 1
+		} else {
+			for i := 1; i < len(columns); i++ {
+				if columns[i].Name == "_stream" {
+					streamCol = i
+					break
+				}
+			}
+		}
+
 		bb := blockResultPool.Get()
 		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
+			if columns[streamCol].Values[rowIdx] == "" {
+				continue
+			}
 			WriteJSONRow(bb, columns, rowIdx)
 		}
-		sw.MustWrite(bb.B)
+
+		if !sw.TryWrite(bb.B) {
+			cancel()
+		}
+
 		blockResultPool.Put(bb)
 	})
 	sw.FinalFlush()

app/vlselect/logsql/sort_writer.go

@@ -36,8 +36,12 @@ var sortWriterPool sync.Pool
 // If the buf isn't empty at FinalFlush() call, then the buffered data
 // is sorted by _time field.
 type sortWriter struct {
-	mu         sync.Mutex
-	w          io.Writer
+	mu sync.Mutex
+	w  io.Writer
+
+	maxLines     int
+	linesWritten int
+
 	maxBufLen  int
 	buf        []byte
 	bufFlushed bool
@@ -47,58 +51,119 @@ type sortWriter struct {
 
 func (sw *sortWriter) reset() {
 	sw.w = nil
+
+	sw.maxLines = 0
+	sw.linesWritten = 0
+
 	sw.maxBufLen = 0
 	sw.buf = sw.buf[:0]
 	sw.bufFlushed = false
 	sw.hasErr = false
 }
 
-func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
+// Init initializes sw.
+//
+// If maxLines is set to positive value, then sw accepts up to maxLines
+// and then rejects all the other lines by returning false from TryWrite.
+func (sw *sortWriter) Init(w io.Writer, maxBufLen, maxLines int) {
 	sw.reset()
 
 	sw.w = w
 	sw.maxBufLen = maxBufLen
+	sw.maxLines = maxLines
 }
 
-func (sw *sortWriter) MustWrite(p []byte) {
+// TryWrite writes p to sw.
+//
+// True is returned on successful write, false otherwise.
+//
+// Unsuccessful write may occur on underlying write error or when maxLines lines are already written to sw.
+func (sw *sortWriter) TryWrite(p []byte) bool {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
 
 	if sw.hasErr {
-		return
+		return false
 	}
 
 	if sw.bufFlushed {
-		if _, err := sw.w.Write(p); err != nil {
+		if !sw.writeToUnderlyingWriterLocked(p) {
 			sw.hasErr = true
+			return false
 		}
-		return
+		return true
 	}
+
 	if len(sw.buf)+len(p) < sw.maxBufLen {
 		sw.buf = append(sw.buf, p...)
-		return
+		return true
 	}
+
 	sw.bufFlushed = true
-	if len(sw.buf) > 0 {
-		if _, err := sw.w.Write(sw.buf); err != nil {
-			sw.hasErr = true
-			return
+	if !sw.writeToUnderlyingWriterLocked(sw.buf) {
+		sw.hasErr = true
+		return false
+	}
+	sw.buf = sw.buf[:0]
+
+	if !sw.writeToUnderlyingWriterLocked(p) {
+		sw.hasErr = true
+		return false
+	}
+	return true
+}
+
+func (sw *sortWriter) writeToUnderlyingWriterLocked(p []byte) bool {
+	if len(p) == 0 {
+		return true
+	}
+	if sw.maxLines > 0 {
+		if sw.linesWritten >= sw.maxLines {
+			return false
 		}
-		sw.buf = sw.buf[:0]
+		var linesLeft int
+		p, linesLeft = trimLines(p, sw.maxLines-sw.linesWritten)
+		sw.linesWritten += linesLeft
 	}
 	if _, err := sw.w.Write(p); err != nil {
-		sw.hasErr = true
+		return false
 	}
+	return true
 }
 
+func trimLines(p []byte, maxLines int) ([]byte, int) {
+	if maxLines <= 0 {
+		return nil, 0
+	}
+	n := bytes.Count(p, newline)
+	if n < maxLines {
+		return p, n
+	}
+	for n >= maxLines {
+		idx := bytes.LastIndexByte(p, '\n')
+		p = p[:idx]
+		n--
+	}
+	return p[:len(p)+1], maxLines
+}
+
+var newline = []byte("\n")
+
 func (sw *sortWriter) FinalFlush() {
 	if sw.hasErr || sw.bufFlushed {
 		return
 	}
+
 	rs := getRowsSorter()
 	rs.parseRows(sw.buf)
 	rs.sort()
-	WriteJSONRows(sw.w, rs.rows)
+
+	rows := rs.rows
+	if sw.maxLines > 0 && len(rows) > sw.maxLines {
+		rows = rows[:sw.maxLines]
+	}
+	WriteJSONRows(sw.w, rows)
+
 	putRowsSorter(rs)
 }
 

app/vlselect/logsql/sort_writer_test.go

@@ -7,15 +7,16 @@ import (
 )
 
 func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen int, data string, expectedResult string) {
+	f := func(maxBufLen, maxLines int, data string, expectedResult string) {
 		t.Helper()
 
 		var bb bytes.Buffer
 		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen)
-
+		sw.Init(&bb, maxBufLen, maxLines)
 		for _, s := range strings.Split(data, "\n") {
-			sw.MustWrite([]byte(s + "\n"))
+			if !sw.TryWrite([]byte(s + "\n")) {
+				break
+			}
 		}
 		sw.FinalFlush()
 		putSortWriter(sw)
@@ -26,14 +27,20 @@ func TestSortWriter(t *testing.T) {
 		}
 	}
 
-	f(100, "", "")
-	f(100, "{}", "{}\n")
+	f(100, 0, "", "")
+	f(100, 0, "{}", "{}\n")
 
 	data := `{"_time":"def","_msg":"xxx"}
 {"_time":"abc","_msg":"foo"}`
 	resultExpected := `{"_time":"abc","_msg":"foo"}
 {"_time":"def","_msg":"xxx"}
 `
-	f(100, data, resultExpected)
-	f(10, data, data+"\n")
+	f(100, 0, data, resultExpected)
+	f(10, 0, data, data+"\n")
+
+	// Test with the maxLines
+	f(100, 1, data, `{"_time":"abc","_msg":"foo"}`+"\n")
+	f(10, 1, data, `{"_time":"def","_msg":"xxx"}`+"\n")
+	f(10, 2, data, data+"\n")
+	f(100, 2, data, resultExpected)
 }

app/vlselect/main.go

@@ -1,6 +1,7 @@
 package vlselect
 
 import (
+	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -101,7 +102,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
-	stopCh := r.Context().Done()
+	ctx := r.Context()
+	stopCh := ctx.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
@@ -139,11 +141,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 	}
 
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+	stopCh = ctxWithCancel.Done()
+
 	switch {
 	case path == "/logsql/query":
 		logsqlQueryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh)
+		logsql.ProcessQueryRequest(w, r, stopCh, cancel)
 		return true
 	default:
 		return false

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.d1313636.css",
-    "main.js": "./static/js/main.1919fefe.js",
-    "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
+    "main.css": "./static/css/main.bc07cc78.css",
+    "main.js": "./static/js/main.034044a7.js",
+    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.10add6e7bdf0f1d98cf7.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d1313636.css",
-    "static/js/main.1919fefe.js"
+    "static/css/main.bc07cc78.css",
+    "static/js/main.034044a7.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.034044a7.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.034044a7.js.LICENSE.txt

@@ -4,10 +4,8 @@
 	http://jedwatson.github.io/classnames
 */
 
-/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
-
 /**
- * @remix-run/router v1.10.0
+ * @remix-run/router v1.15.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -18,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.17.0
+ * React Router DOM v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -29,7 +27,7 @@
  */
 
 /**
- * React Router v6.17.0
+ * React Router v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.10add6e7bdf0f1d98cf7.md

@@ -26,12 +26,18 @@ and introduction into [basic querying via MetricsQL](https://docs.victoriametric
 
 The following functionality is implemented differently in MetricsQL compared to PromQL. This improves user experience:
 
-* MetricsQL takes into account the previous point before the window in square brackets for range functions such as [rate](#rate) and [increase](#increase).
-  This allows returning the exact results users expect for `increase(metric[$__interval])` queries instead of incomplete results Prometheus returns for such queries.
-* MetricsQL doesn't extrapolate range function results. This addresses [this issue from Prometheus](https://github.com/prometheus/prometheus/issues/3746).
+* MetricsQL takes into account the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples) before the lookbehind window
+  in square brackets for [increase](#increase) and [rate](#rate) functions. This allows returning the exact results users expect for `increase(metric[$__interval])` queries
+  instead of incomplete results Prometheus returns for such queries. Prometheus misses the increase between the last sample before the lookbehind window
+  and the first sample inside the lookbehind window.
+* MetricsQL doesn't extrapolate [rate](#rate) and [increase](#increase) function results, so it always returns the expected results. For example, it returns
+  integer results from `increase()` over slow-changing integer counter. Prometheus in this case returns unexpected fractional results,
+  which may significantly differ from the expected results. This addresses [this issue from Prometheus](https://github.com/prometheus/prometheus/issues/3746).
   See technical details about VictoriaMetrics and Prometheus calculations for [rate](#rate)
   and [increase](#increase) [in this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1215#issuecomment-850305711).
-* MetricsQL returns the expected non-empty responses for [rate](#rate) with `step` values smaller than scrape interval.
+* MetricsQL returns the expected non-empty responses for [rate](#rate) function when Grafana or [vmui](https://docs.victoriametrics.com/#vmui)
+  passes `step` values smaller than the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+  to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query).
   This addresses [this issue from Grafana](https://github.com/grafana/grafana/issues/11451).
   See also [this blog post](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/).
 * MetricsQL treats `scalar` type the same as `instant vector` without labels, since subtle differences between these types usually confuse users.
@@ -61,16 +67,17 @@ The list of MetricsQL features on top of PromQL:
 
 * Graphite-compatible filters can be passed via `{__graphite__="foo.*.bar"}` syntax.
   See [these docs](https://docs.victoriametrics.com/#selecting-graphite-metrics).
-  VictoriaMetrics also can be used as Graphite datasource in Grafana.
-  See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details.
+  VictoriaMetrics can be used as Graphite datasource in Grafana. See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details.
   See also [label_graphite_group](#label_graphite_group) function, which can be used for extracting the given groups from Graphite metric name.
-* Lookbehind window in square brackets may be omitted. VictoriaMetrics automatically selects the lookbehind window
-  depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)).
+* Lookbehind window in square brackets for [rollup functions](#rollup-functions) may be omitted. VictoriaMetrics automatically selects the lookbehind window
+  depending on the `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query)
+  and the real interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) (aka `scrape_interval`).
   For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
-  It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
+  It is roughly equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
+  The difference is documented in [rate() docs](#rate).
 * Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
-  selects series with either `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
+  selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/keyConcepts.html#filtering-by-multiple-or-filters) for details.
 * Support for `group_left(*)` and `group_right(*)` for copying all the labels from time series on the `one` side
   of [many-to-one operations](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches).
@@ -98,7 +105,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `температура{город="Киев"}` is a value MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a value MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -117,7 +124,8 @@ The list of MetricsQL features on top of PromQL:
   Go to [WITH templates playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/expand-with-exprs) and try it.
 * String literals may be concatenated. This is useful with `WITH` templates:
   `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`.
-* `keep_metric_names` modifier can be applied to all the [rollup functions](#rollup-functions), [transform functions](#transform-functions) and [binary operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operators).
+* `keep_metric_names` modifier can be applied to all the [rollup functions](#rollup-functions), [transform functions](#transform-functions)
+  and [binary operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operators).
   This modifier prevents from dropping metric names in function results. See [these docs](#keep_metric_names).
 
 ## keep_metric_names
@@ -155,14 +163,15 @@ Additional details:
   The interval between points is set as `step` query arg passed by Grafana to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
 * If the given [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) returns multiple time series,
   then rollups are calculated individually per each returned series.
-* If lookbehind window in square brackets is missing, then MetricsQL automatically sets the lookbehind window
-  to the interval between points on the graph (aka `step` query arg at [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query),
-  `$__interval` value from Grafana or `1i` duration in MetricsQL).
-  For example, `rate(http_requests_total)` is equivalent to `rate(http_requests_total[$__interval])` in Grafana.
-  It is also equivalent to `rate(http_requests_total[1i])`.
+* If lookbehind window in square brackets is missing, then it is automatically set to the following value:
+  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query)
+    for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
+    For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
+  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+    for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * Every [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) in MetricsQL must be wrapped into a rollup function.
   Otherwise, it is automatically wrapped into [default_rollup](#default_rollup). For example, `foo{bar="baz"}`
-  is automatically converted to `default_rollup(foo{bar="baz"}[1i])` before performing the calculations.
+  is automatically converted to `default_rollup(foo{bar="baz"})` before performing the calculations.
 * If something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) is passed to rollup function,
   then the inner arg is automatically converted to a [subquery](#subqueries).
 * All the rollup functions accept optional `keep_metric_names` modifier. If it is set, then the function keeps metric names in results.
@@ -177,7 +186,9 @@ The list of supported rollup functions:
 `absent_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns 1
 if the given lookbehind window `d` doesn't contain raw samples. Otherwise, it returns an empty result.
 
-This function is supported by PromQL. See also [present_over_time](#present_over_time).
+This function is supported by PromQL.
+
+See also [present_over_time](#present_over_time).
 
 #### aggr_over_time
 
@@ -207,7 +218,9 @@ See also [descent_over_time](#descent_over_time).
 over raw samples on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [median_over_time](#median_over_time).
+This function is supported by PromQL.
+
+See also [median_over_time](#median_over_time).
 
 #### changes
 
@@ -220,7 +233,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [changes_prometheus](#changes_prometheus).
+This function is supported by PromQL.
+
+See also [changes_prometheus](#changes_prometheus).
 
 #### changes_prometheus
 
@@ -233,7 +248,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [changes](#changes).
+This function is supported by PromQL.
+
+See also [changes](#changes).
 
 #### count_eq_over_time
 
@@ -243,7 +260,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
 
@@ -253,7 +270,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
 
@@ -263,7 +280,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
 
@@ -282,8 +299,19 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time),
-[count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time).
+This function is supported by PromQL.
+
+See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time), [count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time).
+
+#### count_values_over_time
+
+`count_values_over_time("label", series_selector[d])` is a [rollup function](#rollup-functions), which counts the number of raw samples
+with the same value over the given lookbehind window and stores the counts in a time series with an additional `label`, which contains each initial value.
+The results are calculated independently per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
 
@@ -299,6 +327,11 @@ See also [increases_over_time](#increases_over_time).
 `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last raw sample value on the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
+If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
+passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
+while `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) for the selected time series.
+This allows avoiding unexpected gaps on the graph when `step` is smaller than the `scrape_interval`.
+
 #### delta
 
 `delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the difference between
@@ -310,7 +343,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [increase](#increase) and [delta_prometheus](#delta_prometheus).
+This function is supported by PromQL.
+
+See also [increase](#increase) and [delta_prometheus](#delta_prometheus).
 
 #### delta_prometheus
 
@@ -333,7 +368,9 @@ The derivative is calculated using linear regression.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv).
+This function is supported by PromQL.
+
+See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv).
 
 #### deriv_fast
 
@@ -364,6 +401,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_values_over_time](#count_values_over_time).
+
 #### duration_over_time
 
 `duration_over_time(series_selector[d], max_interval)` is a [rollup function](#rollup-functions), which returns the duration in seconds
@@ -423,7 +462,9 @@ over the given lookbehind window `d` using the given smoothing factor `sf` and t
 Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering)
 returns time series of [gauge type](https://docs.victoriametrics.com/keyConcepts.html#gauge).
 
-This function is supported by PromQL. See also [range_linear_regression](#range_linear_regression).
+This function is supported by PromQL.
+
+See also [range_linear_regression](#range_linear_regression).
 
 #### idelta
 
@@ -432,7 +473,9 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [delta](#delta).
+This function is supported by PromQL.
+
+See also [delta](#delta).
 
 #### ideriv
 
@@ -455,7 +498,9 @@ See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-co
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
+This function is supported by PromQL.
+
+See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
 
 #### increase_prometheus
 
@@ -499,7 +544,9 @@ It is expected that the `series_selector` returns time series of [counter type](
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [rate](#rate) and [rollup_rate](#rollup_rate).
+This function is supported by PromQL.
+
+See also [rate](#rate) and [rollup_rate](#rollup_rate).
 
 #### lag
 
@@ -516,7 +563,9 @@ See also [lifetime](#lifetime) and [duration_over_time](#duration_over_time).
 `last_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last raw sample value on the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [first_over_time](#first_over_time) and [tlast_over_time](#tlast_over_time).
+This function is supported by PromQL.
+
+See also [first_over_time](#first_over_time) and [tlast_over_time](#tlast_over_time).
 
 #### lifetime
 
@@ -539,7 +588,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over raw samples
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [tmax_over_time](#tmax_over_time).
+This function is supported by PromQL.
+
+See also [tmax_over_time](#tmax_over_time).
 
 #### median_over_time
 
@@ -554,7 +605,9 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over raw samples
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [tmin_over_time](#tmin_over_time).
+This function is supported by PromQL.
+
+See also [tmin_over_time](#tmin_over_time).
 
 #### mode_over_time
 
@@ -570,7 +623,7 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 - `q25` and `q75` are 25th and 75th [percentiles](https://en.wikipedia.org/wiki/Percentile) over raw samples on the lookbehind window `d`.
 
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
-For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last 24 hours.
+For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
 See also [outliers_iqr](#outliers_iqr).
 
@@ -580,7 +633,9 @@ See also [outliers_iqr](#outliers_iqr).
 linear interpolation over raw samples on the given lookbehind window `d`. The predicted value is calculated individually per each time series
 returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-This function is supported by PromQL. See also [range_linear_regression](#range_linear_regression).
+This function is supported by PromQL.
+
+See also [range_linear_regression](#range_linear_regression).
 
 #### present_over_time
 
@@ -597,7 +652,9 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 The `phi` value must be in the range `[0...1]`.
 
-This function is supported by PromQL. See also [quantiles_over_time](#quantiles_over_time).
+This function is supported by PromQL.
+
+See also [quantiles_over_time](#quantiles_over_time).
 
 #### quantiles_over_time
 
@@ -622,9 +679,16 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyConcepts.html#counter).
 
+If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
+passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
+while `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) for the selected time series.
+This allows avoiding unexpected gaps on the graph when `step` is smaller than the `scrape_interval`.
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [irate](#irate) and [rollup_rate](#rollup_rate).
+This function is supported by PromQL.
+
+See also [irate](#irate) and [rollup_rate](#rollup_rate).
 
 #### rate_over_sum
 
@@ -652,6 +716,7 @@ on the given lookbehind window `d` and returns them in time series with `rollup=
 These values are calculated individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 #### rollup_candlestick
 
@@ -660,7 +725,8 @@ over raw samples on the given lookbehind window `d` and returns them in time ser
 The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). This function is useful for financial applications.
 
-Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 #### rollup_delta
 
@@ -670,6 +736,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -683,6 +750,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -694,6 +762,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
@@ -707,10 +776,10 @@ See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-c
 when to use `rollup_rate()`.
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### rollup_scrape_interval
@@ -721,6 +790,7 @@ and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup=
 The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
+See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [scrape_interval](#scrape_interval).
 
@@ -743,7 +813,7 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_le_over_time](#share_le_over_time).
+See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
 
@@ -756,7 +826,7 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [share_gt_over_time](#share_gt_over_time).
+See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
 
@@ -766,6 +836,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.ht
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [count_eq_over_time](#count_eq_over_time).
+
 #### stale_samples_over_time
 
 `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
@@ -781,7 +853,9 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [stdvar_over_time](#stdvar_over_time).
+This function is supported by PromQL.
+
+See also [stdvar_over_time](#stdvar_over_time).
 
 #### stdvar_over_time
 
@@ -790,7 +864,36 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
+This function is supported by PromQL.
+
+See also [stddev_over_time](#stddev_over_time).
+
+#### sum_eq_over_time
+
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values equal to `eq`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
+
+#### sum_gt_over_time
+
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values bigger than `gt`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
+
+#### sum_le_over_time
+
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of raw sample values smaller or equal to `le`
+on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
 
@@ -810,25 +913,27 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 #### timestamp
 
-`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [timestamp_with_name](#timestamp_with_name).
+This function is supported by PromQL.
+
+See also [time](#time) and [now](#now).
 
 #### timestamp_with_name
 
-`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last raw sample
+`timestamp_with_name(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are preserved in the resulting rollups.
 
-See also [timestamp](#timestamp).
+See also [timestamp](#timestamp) and [keep_metric_names](#keep_metric_names) modifier.
 
 #### tfirst_over_time
 
-`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the first raw sample
+`tfirst_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the first raw sample
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -837,7 +942,7 @@ See also [first_over_time](#first_over_time).
 
 #### tlast_change_over_time
 
-`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the last change
+`tlast_change_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the last change
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) on the given lookbehind window `d`.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -852,7 +957,7 @@ See also [tlast_change_over_time](#tlast_change_over_time).
 
 #### tmax_over_time
 
-`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmax_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -862,7 +967,7 @@ See also [max_over_time](#max_over_time).
 
 #### tmin_over_time
 
-`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds for the raw sample
+`tmin_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision for the raw sample
 with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
 
@@ -891,7 +996,7 @@ Additional details:
 
 * If transform function is applied directly to a [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before calculating the transformations.
-  For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature[1i]))`.
+  For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature))`.
 * All the transform functions accept optional `keep_metric_names` modifier. If it is set,
   then the function doesn't drop metric names from the resulting time series. See [these docs](#keep_metric_names).
 
@@ -909,7 +1014,9 @@ This function is supported by PromQL.
 
 `absent(q)` is a [transform function](#transform-functions), which returns 1 if `q` has no points. Otherwise, returns an empty result.
 
-This function is supported by PromQL. See also [absent_over_time](#absent_over_time).
+This function is supported by PromQL.
+
+See also [absent_over_time](#absent_over_time).
 
 #### acos
 
@@ -918,7 +1025,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [asin](#asin) and [cos](#cos).
+This function is supported by PromQL.
+
+See also [asin](#asin) and [cos](#cos).
 
 #### acosh
 
@@ -927,7 +1036,9 @@ This function is supported by PromQL. See also [asin](#asin) and [cos](#cos).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sinh](#cosh).
+This function is supported by PromQL.
+
+See also [sinh](#cosh).
 
 #### asin
 
@@ -936,7 +1047,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [acos](#acos) and [sin](#sin).
+This function is supported by PromQL.
+
+See also [acos](#acos) and [sin](#sin).
 
 #### asinh
 
@@ -945,7 +1058,9 @@ This function is supported by PromQL. See also [acos](#acos) and [sin](#sin).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sinh](#sinh).
+This function is supported by PromQL.
+
+See also [sinh](#sinh).
 
 #### atan
 
@@ -954,7 +1069,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [tan](#tan).
+This function is supported by PromQL.
+
+See also [tan](#tan).
 
 #### atanh
 
@@ -963,7 +1080,9 @@ This function is supported by PromQL. See also [tan](#tan).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [tanh](#tanh).
+This function is supported by PromQL.
+
+See also [tanh](#tanh).
 
 #### bitmap_and
 
@@ -994,25 +1113,33 @@ See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#his
 
 `ceil(q)` is a [transform function](#transform-functions), which rounds every point for every time series returned by `q` to the upper nearest integer.
 
-This function is supported by PromQL. See also [floor](#floor) and [round](#round).
+This function is supported by PromQL.
+
+See also [floor](#floor) and [round](#round).
 
 #### clamp
 
 `clamp(q, min, max)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `min` and `max` values.
 
-This function is supported by PromQL. See also [clamp_min](#clamp_min) and [clamp_max](#clamp_max).
+This function is supported by PromQL.
+
+See also [clamp_min](#clamp_min) and [clamp_max](#clamp_max).
 
 #### clamp_max
 
 `clamp_max(q, max)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `max` value.
 
-This function is supported by PromQL. See also [clamp](#clamp) and [clamp_min](#clamp_min).
+This function is supported by PromQL.
+
+See also [clamp](#clamp) and [clamp_min](#clamp_min).
 
 #### clamp_min
 
 `clamp_min(q, min)` is a [transform function](#transform-functions), which clamps every point for every time series returned by `q` with the given `min` value.
 
-This function is supported by PromQL. See also [clamp](#clamp) and [clamp_max](#clamp_max).
+This function is supported by PromQL.
+
+See also [clamp](#clamp) and [clamp_max](#clamp_max).
 
 #### cos
 
@@ -1020,7 +1147,9 @@ This function is supported by PromQL. See also [clamp](#clamp) and [clamp_max](#
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [sin](#sin).
+This function is supported by PromQL.
+
+See also [sin](#sin).
 
 #### cosh
 
@@ -1029,7 +1158,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [acosh](#acosh).
+This function is supported by PromQL.
+
+See also [acosh](#acosh).
 
 #### day_of_month
 
@@ -1040,6 +1171,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_week](#day_of_week) and [day_of_year](#day_of_year).
+
 #### day_of_week
 
 `day_of_week(q)` is a [transform function](#transform-functions), which returns the day of week for every point of every time series returned by `q`.
@@ -1049,6 +1182,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_month](#day_of_month) and [day_of_year](#day_of_year).
+
 #### day_of_year
 
 `day_of_year(q)` is a [transform function](#transform-functions), which returns the day of year for every point of every time series returned by `q`.
@@ -1058,6 +1193,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 
 This function is supported by PromQL.
 
+See also [day_of_week](#day_of_week) and [day_of_month](#day_of_month).
+
 #### days_in_month
 
 `days_in_month(q)` is a [transform function](#transform-functions), which returns the number of days in the month identified
@@ -1075,7 +1212,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [rad](#rad).
+This function is supported by PromQL.
+
+See also [rad](#rad).
 
 #### drop_empty_series
 
@@ -1101,13 +1240,17 @@ See also [start](#start), [time](#time) and [now](#now).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [ln](#ln).
+This function is supported by PromQL.
+
+See also [ln](#ln).
 
 #### floor
 
 `floor(q)` is a [transform function](#transform-functions), which rounds every point for every time series returned by `q` to the lower nearest integer.
 
-This function is supported by PromQL. See also [ceil](#ceil) and [round](#round).
+This function is supported by PromQL.
+
+See also [ceil](#ceil) and [round](#round).
 
 #### histogram_avg
 
@@ -1130,8 +1273,9 @@ When the [percentile](https://en.wikipedia.org/wiki/Percentile) is calculated ov
 then all the input histograms **must** have buckets with identical boundaries, e.g. they must have the same set of `le` or `vmrange` labels.
 Otherwise, the returned result may be invalid. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3231) for details.
 
-This function is supported by PromQL (except of the `boundLabel` arg). See also [histogram_quantiles](#histogram_quantiles), [histogram_share](#histogram_share)
-and [quantile](#quantile).
+This function is supported by PromQL (except of the `boundLabel` arg).
+
+See also [histogram_quantiles](#histogram_quantiles), [histogram_share](#histogram_share) and [quantile](#quantile).
 
 #### histogram_quantiles
 
@@ -1203,7 +1347,9 @@ This allows implementing simple paging for `q` time series. See also [limitk](#l
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [exp](#exp) and [log2](#log2).
+This function is supported by PromQL.
+
+See also [exp](#exp) and [log2](#log2).
 
 #### log2
 
@@ -1211,7 +1357,9 @@ This function is supported by PromQL. See also [exp](#exp) and [log2](#log2).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [log10](#log10) and [ln](#ln).
+This function is supported by PromQL.
+
+See also [log10](#log10) and [ln](#ln).
 
 #### log10
 
@@ -1219,7 +1367,9 @@ This function is supported by PromQL. See also [log10](#log10) and [ln](#ln).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [log2](#log2) and [ln](#ln).
+This function is supported by PromQL.
+
+See also [log2](#log2) and [ln](#ln).
 
 #### minute
 
@@ -1258,7 +1408,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by PromQL. See also [deg](#deg).
+This function is supported by PromQL.
+
+See also [deg](#deg).
 
 #### prometheus_buckets
 
@@ -1386,7 +1538,9 @@ for points returned by `q`, e.g. it is equivalent to the following query: `(q -
 `round(q, nearest)` is a [transform function](#transform-functions), which rounds every point of every time series returned by `q` to the `nearest` multiple.
 If `nearest` is missing then the rounding is performed to the nearest integer.
 
-This function is supported by PromQL. See also [floor](#floor) and [ceil](#ceil).
+This function is supported by PromQL.
+
+See also [floor](#floor) and [ceil](#ceil).
 
 #### ru
 
@@ -1430,7 +1584,9 @@ This function is supported by PromQL.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [cos](#cos).
+This function is supported by MetricsQL.
+
+See also [cos](#cos).
 
 #### sinh
 
@@ -1439,7 +1595,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [cosh](#cosh).
+This function is supported by MetricsQL.
+
+See also [cosh](#cosh).
 
 #### tan
 
@@ -1447,7 +1605,9 @@ This function is supported by MetricsQL. See also [cosh](#cosh).
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [atan](#atan).
+This function is supported by MetricsQL.
+
+See also [atan](#atan).
 
 #### tanh
 
@@ -1456,7 +1616,9 @@ for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is supported by MetricsQL. See also [atanh](#atanh).
+This function is supported by MetricsQL.
+
+See also [atanh](#atanh).
 
 #### smooth_exponential
 
@@ -1467,13 +1629,17 @@ by `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_a
 
 `sort(q)` is a [transform function](#transform-functions), which sorts series in ascending order by the last point in every time series returned by `q`.
 
-This function is supported by PromQL. See also [sort_desc](#sort_desc) and [sort_by_label](#sort_by_label).
+This function is supported by PromQL.
+
+See also [sort_desc](#sort_desc) and [sort_by_label](#sort_by_label).
 
 #### sort_desc
 
 `sort_desc(q)` is a [transform function](#transform-functions), which sorts series in descending order by the last point in every time series returned by `q`.
 
-This function is supported by PromQL. See also [sort](#sort) and [sort_by_label](#sort_by_label_desc).
+This function is supported by PromQL.
+
+See also [sort](#sort) and [sort_by_label](#sort_by_label_desc).
 
 #### sqrt
 
@@ -1502,7 +1668,9 @@ See also [start](#start) and [end](#end).
 
 `time()` is a [transform function](#transform-functions), which returns unix timestamp for every returned point.
 
-This function is supported by PromQL. See also [now](#now), [start](#start) and [end](#end).
+This function is supported by PromQL.
+
+See also [timestamp](#timestamp), [now](#now), [start](#start) and [end](#end).
 
 #### timezone_offset
 
@@ -1551,7 +1719,7 @@ Additional details:
 
 * If label manipulation function is applied directly to a [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before performing the label transformation.
-  For example, `alias(temperature, "foo")` is implicitly transformed to `alias(default_rollup(temperature[1i]), "foo")`.
+  For example, `alias(temperature, "foo")` is implicitly transformed to `alias(default_rollup(temperature), "foo")`.
 
 See also [implicit query conversions](#implicit-query-conversions).
 
@@ -1728,7 +1896,7 @@ Additional details:
   Multiple labels can be put in `by` and `without` modifiers.
 * If the aggregate function is applied directly to a [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   then the [default_rollup()](#default_rollup) function is automatically applied before calculating the aggregate.
-  For example, `count(up)` is implicitly transformed to `count(default_rollup(up[1i]))`.
+  For example, `count(up)` is implicitly transformed to `count(default_rollup(up))`.
 * Aggregate functions accept arbitrary number of args. For example, `avg(q1, q2, q3)` would return the average values for every point
   across time series returned by `q1`, `q2` and `q3`.
 * Aggregate functions support optional `limit N` suffix, which can be used for limiting the number of output groups.
@@ -1756,7 +1924,9 @@ This function is supported by PromQL.
 `bottomk(k, q)` is [aggregate function](#aggregate-functions), which returns up to `k` points with the smallest values across all the time series returned by `q`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [topk](#topk).
+This function is supported by PromQL.
+
+See also [topk](#topk), [bottomk_min](#bottomk_min) and [#bottomk_last](#bottomk_last).
 
 #### bottomk_avg
 
@@ -1818,10 +1988,14 @@ The aggregate is calculated individually per each group of points with the same
 
 This function is supported by PromQL.
 
+See also [count_values_over_time](#count_values_over_time) and [label_match](#label_match).
+
 #### distinct
 
 `distinct(q)` is [aggregate function](#aggregate-functions), which calculates the number of unique values per each group of points with the same timestamp.
 
+See also [distinct_over_time](#distinct_over_time).
+
 #### geomean
 
 `geomean(q)` is [aggregate function](#aggregate-functions), which calculates geometric mean per each group of points with the same timestamp.
@@ -1913,7 +2087,9 @@ See also [outliers_iqr](#outliers_iqr) and [outliers_mad](#outliers_mad).
 for all the time series returned by `q`. `phi` must be in the range `[0...1]`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [quantiles](#quantiles) and [histogram_quantile](#histogram_quantile).
+This function is supported by PromQL.
+
+See also [quantiles](#quantiles) and [histogram_quantile](#histogram_quantile).
 
 #### quantiles
 
@@ -1972,7 +2148,9 @@ for all the time series returned by `q`. The aggregate is calculated individuall
 `topk(k, q)` is [aggregate function](#aggregate-functions), which returns up to `k` points with the biggest values across all the time series returned by `q`.
 The aggregate is calculated individually per each group of points with the same timestamp.
 
-This function is supported by PromQL. See also [bottomk](#bottomk).
+This function is supported by PromQL.
+
+See also [bottomk](#bottomk), [topk_max](#topk_max) and [topk_last](#topk_last).
 
 #### topk_avg
 
@@ -2032,7 +2210,7 @@ See also [zscore_over_time](#zscore_over_time), [range_trim_zscore](#range_trim_
 MetricsQL supports and extends PromQL subqueries. See [this article](https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3) for details.
 Any [rollup function](#rollup-functions) for something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering) form a subquery.
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
-For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m[1i]))[1i:1i])`, so it becomes a subquery,
+For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
 
 VictoriaMetrics performs subqueries in the following way:
@@ -2047,21 +2225,23 @@ VictoriaMetrics performs subqueries in the following way:
 
 VictoriaMetrics performs the following implicit conversions for incoming queries before starting the calculations:
 
-* If lookbehind window in square brackets is missing inside [rollup function](#rollup-functions),
-  then `[1i]` is automatically added there. The `[1i]` means one `step` value, which is passed
-  to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
-  It is also known as `$__interval` in Grafana. For example, `rate(http_requests_count)` is automatically transformed to `rate(http_requests_count[1i])`.
+* If lookbehind window in square brackets is missing inside [rollup function](#rollup-functions), then it is automatically set to the following value:
+  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query)
+    for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
+    For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
+  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+    for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * All the [series selectors](https://docs.victoriametrics.com/keyConcepts.html#filtering),
   which aren't wrapped into [rollup functions](#rollup-functions), are automatically wrapped into [default_rollup](#default_rollup) function.
   Examples:
-  * `foo` is transformed to `default_rollup(foo[1i])`
-  * `foo + bar` is transformed to `default_rollup(foo[1i]) + default_rollup(bar[1i])`
-  * `count(up)` is transformed to `count(default_rollup(up[1i]))`, because [count](#count) isn't a [rollup function](#rollup-functions) -
+  * `foo` is transformed to `default_rollup(foo)`
+  * `foo + bar` is transformed to `default_rollup(foo) + default_rollup(bar)`
+  * `count(up)` is transformed to `count(default_rollup(up))`, because [count](#count) isn't a [rollup function](#rollup-functions) -
     it is [aggregate function](#aggregate-functions)
-  * `abs(temperature)` is transformed to `abs(default_rollup(temperature[1i]))`, because [abs](#abs) isn't a [rollup function](#rollup-functions) -
+  * `abs(temperature)` is transformed to `abs(default_rollup(temperature))`, because [abs](#abs) isn't a [rollup function](#rollup-functions) -
     it is [transform function](#transform-functions)
 * If `step` in square brackets is missing inside [subquery](#subqueries), then `1i` step is automatically added there.
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up[1i])))[1i:1i])`.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.

app/vmagent/datadogsketches/request_handler.go

@@ -0,0 +1,95 @@
+package datadogsketches
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
+		return insertRows(at, sketches, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for _, sketch := range sketches {
+		ms := sketch.ToSummary()
+		for _, m := range ms {
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: m.Name,
+			})
+			for _, label := range m.Labels {
+				labels = append(labels, prompbmarshal.Label{
+					Name:  label.Name,
+					Value: label.Value,
+				})
+			}
+			for _, tag := range sketch.Tags {
+				name, value := datadogutils.SplitTag(tag)
+				if name == "host" {
+					name = "exported_host"
+				}
+				labels = append(labels, prompbmarshal.Label{
+					Name:  name,
+					Value: value,
+				})
+			}
+			labels = append(labels, extraLabels...)
+			samplesLen := len(samples)
+			for _, p := range m.Points {
+				samples = append(samples, prompbmarshal.Sample{
+					Timestamp: p.Timestamp,
+					Value:     p.Value,
+				})
+			}
+			rowsTotal += len(m.Points)
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[samplesLen:],
+			})
+		}
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/datadogv1/request_handler.go

@@ -1,4 +1,4 @@
-package datadog
+package datadogv1
 
 import (
 	"net/http"
@@ -8,33 +8,32 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
 )
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadog.Series) error {
+	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, series []datadog.Series, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -63,7 +62,7 @@ func insertRows(at *auth.Token, series []datadog.Series, extraLabels []prompbmar
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadog.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}

app/vmagent/datadogv2/request_handler.go

@@ -0,0 +1,102 @@
+package datadogv2
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ct := req.Header.Get("Content-Type")
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
+		return insertRows(at, series, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		for _, rs := range ss.Resources {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  rs.Type,
+				Value: rs.Name,
+			})
+		}
+		if ss.SourceTypeName != "" {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "source_type_name",
+				Value: ss.SourceTypeName,
+			})
+		}
+		for _, tag := range ss.Tags {
+			name, value := datadogutils.SplitTag(tag)
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp * 1000,
+				Value:     pt.Value,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/main.go

@@ -8,11 +8,12 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
@@ -39,15 +40,16 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -68,7 +70,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -95,7 +98,6 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -118,8 +120,13 @@ func main() {
 		return
 	}
 
-	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8429"}
+	}
+	logger.Infof("starting vmagent at %q...", listenAddrs)
 	startTime := time.Now()
+	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
@@ -141,22 +148,21 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	if len(*httpListenAddr) > 0 {
-		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
-	}
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	remotewrite.StopIngestionRateLimiter()
+	pushmetrics.Stop()
 
 	startTime = time.Now()
-	if len(*httpListenAddr) > 0 {
-		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-		if err := httpserver.Stop(*httpListenAddr); err != nil {
-			logger.Fatalf("cannot stop the webservice: %s", err)
-		}
-		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -258,7 +264,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write":
+	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
 		if common.HandleVMProtoServerHandshake(w, r) {
 			return true
 		}
@@ -310,14 +316,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push":
+	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusOK)
+		firehose.WriteSuccessResponse(w, r)
 		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
@@ -343,9 +349,20 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/api/v1/series":
-		datadogWriteRequests.Inc()
-		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogWriteErrors.Inc()
+		datadogv1WriteRequests.Inc()
+		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogv1WriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v2/series":
+		datadogv2WriteRequests.Inc()
+		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogv2WriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -354,6 +371,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "/datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -408,7 +434,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -417,7 +443,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -427,12 +453,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -484,7 +513,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
 	}
 	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write":
+	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -533,14 +562,14 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push":
+	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusOK)
+		firehose.WriteSuccessResponse(w, r)
 		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
@@ -566,9 +595,19 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "datadog/api/v1/series":
-		datadogWriteRequests.Inc()
-		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
-			datadogWriteErrors.Inc()
+		datadogv1WriteRequests.Inc()
+		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
+			datadogv1WriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v2/series":
+		datadogv2WriteRequests.Inc()
+		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
+			datadogv2WriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -576,6 +615,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
+	case "datadog/api/beta/sketches":
+		datadogsketchesWriteRequests.Inc()
+		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
+			datadogsketchesWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -626,16 +674,22 @@ var (
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
-	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
+	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
+
+	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)

app/vmagent/opentelemetry/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -27,10 +28,15 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
-		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
+			processBody = firehose.ProcessRequestBody
+		} else {
+			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+		}
 	}
-	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/promremotewrite/request_handler.go

@@ -6,7 +6,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -48,8 +47,8 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
 			labels = append(labels, prompbmarshal.Label{
-				Name:  bytesutil.ToUnsafeString(label.Name),
-				Value: bytesutil.ToUnsafeString(label.Value),
+				Name:  label.Name,
+				Value: label.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)

app/vmagent/remotewrite/client.go

@@ -17,7 +17,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -29,11 +31,12 @@ var (
 
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
-		"is sent after temporary unavailability of the remote storage")
+		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
 	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
+	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -58,8 +61,10 @@ var (
 	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
@@ -87,7 +92,7 @@ type client struct {
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
 
-	rl rateLimiter
+	rl *ratelimiter.RateLimiter
 
 	bytesSent       *metrics.Counter
 	blocksSent      *metrics.Counter
@@ -108,18 +113,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	if err != nil {
 		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
-	tlsCfg, err := authCfg.NewTLSConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize tls config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
-	}
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
 		DialContext:         statDial,
-		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 10 * time.Second,
+		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -137,7 +137,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		tr.Proxy = http.ProxyURL(pu)
 	}
 	hc := &http.Client{
-		Transport: tr,
+		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
@@ -173,12 +173,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
+	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl.perSecondLimit = int64(bytesPerSec)
+		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
 	}
-	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
-
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
@@ -234,10 +233,16 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
 	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
 	if clientSecretFile != "" || clientSecret != "" {
+		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
+		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
+		}
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
 			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
+			EndpointParams:   endpointParams,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
 		}
@@ -386,8 +391,9 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // The function returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.register(len(block), c.stopCh)
-	retryDuration := time.Second
+	c.rl.Register(len(block))
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:
@@ -397,8 +403,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > time.Minute {
-			retryDuration = time.Minute
+		if retryDuration > maxRetryDuration {
+			retryDuration = maxRetryDuration
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -444,8 +450,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > time.Minute {
-		retryDuration = time.Minute
+	if retryDuration > maxRetryDuration {
+		retryDuration = maxRetryDuration
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
@@ -468,45 +474,3 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-
-type rateLimiter struct {
-	perSecondLimit int64
-
-	// mu protects budget and deadline from concurrent access.
-	mu sync.Mutex
-
-	// The current budget. It is increased by perSecondLimit every second.
-	budget int64
-
-	// The next deadline for increasing the budget by perSecondLimit
-	deadline time.Time
-
-	limitReached *metrics.Counter
-}
-
-func (rl *rateLimiter) register(dataLen int, stopCh <-chan struct{}) {
-	limit := rl.perSecondLimit
-	if limit <= 0 {
-		return
-	}
-
-	rl.mu.Lock()
-	defer rl.mu.Unlock()
-
-	for rl.budget <= 0 {
-		if d := time.Until(rl.deadline); d > 0 {
-			rl.limitReached.Inc()
-			t := timerpool.Get(d)
-			select {
-			case <-stopCh:
-				timerpool.Put(t)
-				return
-			case <-t.C:
-				timerpool.Put(t)
-			}
-		}
-		rl.budget += limit
-		rl.deadline = time.Now().Add(time.Second)
-	}
-	rl.budget -= int64(dataLen)
-}

app/vmagent/remotewrite/pendingseries.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -15,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -69,7 +71,8 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	ticker := time.NewTicker(*flushInterval)
+	d := timeutil.AddJitterToDuration(*flushInterval)
+	ticker := time.NewTicker(d)
 	defer ticker.Stop()
 	for {
 		select {
@@ -79,7 +82,7 @@ func (ps *pendingSeries) periodicFlusher() {
 			ps.mu.Unlock()
 			return
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
 				continue
 			}
 		}
@@ -90,8 +93,7 @@ func (ps *pendingSeries) periodicFlusher() {
 }
 
 type writeRequest struct {
-	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
-	lastFlushTime uint64
+	lastFlushTime atomic.Uint64
 
 	// The queue to send blocks to.
 	fq *persistentqueue.FastQueue
@@ -107,11 +109,12 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss []prompbmarshal.TimeSeries
-
+	tss     []prompbmarshal.TimeSeries
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-	buf     []byte
+
+	// buf holds labels data
+	buf []byte
 }
 
 func (wr *writeRequest) reset() {
@@ -151,7 +154,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
-	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
 	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
 		return false
 	}
@@ -222,33 +225,45 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
+// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
+var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
+
 func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
+
+	marshalConcurrencyCh <- struct{}{}
+
 	bb := writeRequestBufPool.Get()
-	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := snappyBufPool.Get()
+		zb := compressBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
+
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			if !tryPushBlock(zb.B) {
-				return false
+			zbLen := len(zb.B)
+			ok := tryPushBlock(zb.B)
+			compressBufPool.Put(zb)
+			if ok {
+				blockSizeRows.Update(float64(len(wr.Timeseries)))
+				blockSizeBytes.Update(float64(zbLen))
 			}
-			blockSizeRows.Update(float64(len(wr.Timeseries)))
-			blockSizeBytes.Update(float64(len(zb.B)))
-			snappyBufPool.Put(zb)
-			return true
+			return ok
 		}
-		snappyBufPool.Put(zb)
+		compressBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
+
+		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -294,5 +309,7 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var writeRequestBufPool bytesutil.ByteBufferPool
-var snappyBufPool bytesutil.ByteBufferPool
+var (
+	writeRequestBufPool bytesutil.ByteBufferPool
+	compressBufPool     bytesutil.ByteBufferPool
+)

app/vmagent/remotewrite/pendingseries_test.go

@@ -43,7 +43,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 	}
 
 	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 0)
+	f(false, expectedBlockLenProm, 3)
 
 	// Check VictoriaMetrics remote write
 	f(true, expectedBlockLenVM, 15)

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/golang/snappy"
 	"github.com/klauspost/compress/s2"
 )
@@ -22,7 +21,7 @@ func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []by
 	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
 		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
 			wr := newTestWriteRequest(rowsCount, 10)
-			data := prompbmarshal.MarshalWriteRequest(nil, wr)
+			data := wr.MarshalProtobuf(nil)
 			b.ReportAllocs()
 			b.SetBytes(int64(rowsCount))
 			b.RunParallel(func(pb *testing.PB) {

app/vmagent/remotewrite/remotewrite.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -50,13 +51,17 @@ var (
 		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
 	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems")
+		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
+	shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
+		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
+		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
+		"isn't enough for sending high volume of collected data to remote storage. "+
+		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -79,6 +84,8 @@ var (
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
+	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
+		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
@@ -89,8 +96,13 @@ var (
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
-		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
+		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation.html#deduplication")
+	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+
+		"for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation.html#ignoring-old-samples")
+	streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
+		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation.html#dropping-unneeded-labels")
+
 	disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
 		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence ."+
 		"See also -remoteWrite.dropSamplesOnOverload")
@@ -140,7 +152,10 @@ func InitSecretFlags() {
 	}
 }
 
-var shardByURLLabelsMap map[string]struct{}
+var (
+	shardByURLLabelsMap       map[string]struct{}
+	shardByURLIgnoreLabelsMap map[string]struct{}
+)
 
 // Init initializes remotewrite.
 //
@@ -172,19 +187,21 @@ func Init() {
 			return float64(dailySeriesLimiter.CurrentItems())
 		})
 	}
+
 	if *queues > maxQueues {
 		*queues = maxQueues
 	}
 	if *queues <= 0 {
 		*queues = 1
 	}
-	if len(*shardByURLLabels) > 0 {
-		m := make(map[string]struct{}, len(*shardByURLLabels))
-		for _, label := range *shardByURLLabels {
-			m[label] = struct{}{}
-		}
-		shardByURLLabelsMap = m
+
+	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
+		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
+			"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
 	}
+	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
+	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
+
 	initLabelsGlobal()
 
 	// Register SIGHUP handler for config reload before loadRelabelConfigs.
@@ -276,7 +293,7 @@ func reloadRelabelConfigs() {
 var (
 	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
 	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
+	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -336,6 +353,35 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 var configReloaderStopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
+// StartIngestionRateLimiter starts ingestion rate limiter.
+//
+// Ingestion rate limiter must be started before Init() call.
+//
+// StopIngestionRateLimiter must be called before Stop() call in order to unblock all the callers
+// to ingestion rate limiter. Otherwise deadlock may occur at Stop() call.
+func StartIngestionRateLimiter() {
+	if *maxIngestionRate <= 0 {
+		return
+	}
+	ingestionRateLimitReached := metrics.NewCounter(`vmagent_max_ingestion_rate_limit_reached_total`)
+	ingestionRateLimiterStopCh = make(chan struct{})
+	ingestionRateLimiter = ratelimiter.New(int64(*maxIngestionRate), ingestionRateLimitReached, ingestionRateLimiterStopCh)
+}
+
+// StopIngestionRateLimiter stops ingestion rate limiter.
+func StopIngestionRateLimiter() {
+	if ingestionRateLimiterStopCh == nil {
+		return
+	}
+	close(ingestionRateLimiterStopCh)
+	ingestionRateLimiterStopCh = nil
+}
+
+var (
+	ingestionRateLimiter       *ratelimiter.RateLimiter
+	ingestionRateLimiterStopCh chan struct{}
+)
+
 // Stop stops remotewrite.
 //
 // It is expected that nobody calls TryPush during and after the call to this func.
@@ -462,6 +508,9 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 				break
 			}
 		}
+
+		ingestionRateLimiter.Register(samplesCount)
+
 		tssBlock := tss
 		if i < len(tss) {
 			tssBlock = tss[:i]
@@ -526,6 +575,15 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 						hashLabels = append(hashLabels, label)
 					}
 				}
+				tmpLabels.Labels = hashLabels
+			} else if len(shardByURLIgnoreLabelsMap) > 0 {
+				hashLabels = tmpLabels.Labels[:0]
+				for _, label := range ts.Labels {
+					if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
+						hashLabels = append(hashLabels, label)
+					}
+				}
+				tmpLabels.Labels = hashLabels
 			}
 			h := getLabelsHash(hashLabels)
 			idx := h % uint64(len(tssByURL))
@@ -536,22 +594,22 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 		// Push sharded data to remote storages in parallel in order to reduce
 		// the time needed for sending the data to multiple remote storage systems.
 		var wg sync.WaitGroup
-		wg.Add(len(rwctxs))
-		var anyPushFailed uint64
+		var anyPushFailed atomic.Bool
 		for i, rwctx := range rwctxs {
 			tssShard := tssByURL[i]
 			if len(tssShard) == 0 {
 				continue
 			}
+			wg.Add(1)
 			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
 				defer wg.Done()
 				if !rwctx.TryPush(tss) {
-					atomic.StoreUint64(&anyPushFailed, 1)
+					anyPushFailed.Store(true)
 				}
 			}(rwctx, tssShard)
 		}
 		wg.Wait()
-		return atomic.LoadUint64(&anyPushFailed) == 0
+		return !anyPushFailed.Load()
 	}
 
 	// Replicate data among rwctxs.
@@ -559,17 +617,17 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
-	var anyPushFailed uint64
+	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
 			if !rwctx.TryPush(tssBlock) {
-				atomic.StoreUint64(&anyPushFailed, 1)
+				anyPushFailed.Store(true)
 			}
 		}(rwctx)
 	}
 	wg.Wait()
-	return atomic.LoadUint64(&anyPushFailed) == 0
+	return !anyPushFailed.Load()
 }
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
@@ -665,12 +723,14 @@ type remoteWriteCtx struct {
 	fq  *persistentqueue.FastQueue
 	c   *client
 
-	sas                 atomic.Pointer[streamaggr.Aggregators]
+	sas          atomic.Pointer[streamaggr.Aggregators]
+	deduplicator *streamaggr.Deduplicator
+
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
 
 	pss        []*pendingSeries
-	pssNextIdx uint64
+	pssNextIdx atomic.Uint64
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
@@ -738,9 +798,15 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 
 	// Initialize sas
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
+	dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
+	ignoreOldSamples := streamAggrIgnoreOldSamples.GetOptionalArg(argIdx)
 	if sasFile != "" {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
+		opts := &streamaggr.Options{
+			DedupInterval:    dedupInterval,
+			DropInputLabels:  *streamAggrDropInputLabels,
+			IgnoreOldSamples: ignoreOldSamples,
+		}
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
@@ -749,17 +815,24 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+	} else if dedupInterval > 0 {
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels)
 	}
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
-	// sas must be stopped before rwctx is closed
+	// sas and deduplicator must be stopped before rwctx is closed
 	// because sas can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
+	if rwctx.deduplicator != nil {
+		rwctx.deduplicator.MustStop()
+		rwctx.deduplicator = nil
+	}
+
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -798,7 +871,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	rowsCount := getRowsCount(tss)
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 
-	// Apply stream aggregation if any
+	// Apply stream aggregation or deduplication if they are configured
 	sas := rwctx.sas.Load()
 	if sas != nil {
 		matchIdxs := matchIdxsPool.Get()
@@ -813,6 +886,10 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
+	} else if rwctx.deduplicator != nil {
+		rwctx.deduplicator.Push(tss)
+		clear(tss)
+		tss = tss[:0]
 	}
 
 	// Try pushing the data to remote storage
@@ -841,7 +918,7 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 		}
 	}
 	tail := src[len(dst):]
-	_ = prompbmarshal.ResetTimeSeries(tail)
+	clear(tail)
 	return dst
 }
 
@@ -872,7 +949,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	}
 
 	pss := rwctx.pss
-	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
 	ok := pss[idx].TryPush(tss)
 
@@ -894,8 +971,12 @@ func (rwctx *remoteWriteCtx) reinitStreamAggr() {
 
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
+	opts := &streamaggr.Options{
+		DedupInterval:    streamAggrDedupInterval.GetOptionalArg(rwctx.idx),
+		DropInputLabels:  *streamAggrDropInputLabels,
+		IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(rwctx.idx),
+	}
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
@@ -932,13 +1013,17 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 
 // CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
 func CheckStreamAggrConfigs() error {
-	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
+	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
 	for idx, sasFile := range *streamAggrConfig {
 		if sasFile == "" {
 			continue
 		}
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
+		opts := &streamaggr.Options{
+			DedupInterval:    streamAggrDedupInterval.GetOptionalArg(idx),
+			DropInputLabels:  *streamAggrDropInputLabels,
+			IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
+		}
+		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, opts)
 		if err != nil {
 			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
 		}
@@ -946,3 +1031,11 @@ func CheckStreamAggrConfigs() error {
 	}
 	return nil
 }
+
+func newMapFromStrings(a []string) map[string]struct{} {
+	m := make(map[string]struct{}, len(a))
+	for _, s := range a {
+		m[s] = struct{}{}
+	}
+	return m
+}

app/vmagent/remotewrite/statconn.go

@@ -50,7 +50,7 @@ var (
 )
 
 type statConn struct {
-	closed uint64
+	closed atomic.Int32
 	net.Conn
 }
 
@@ -76,7 +76,7 @@ func (sc *statConn) Write(p []byte) (int, error) {
 
 func (sc *statConn) Close() error {
 	err := sc.Conn.Close()
-	if atomic.AddUint64(&sc.closed, 1) == 1 {
+	if sc.closed.Add(1) == 1 {
 		conns.Dec()
 	}
 	return err

app/vmalert-tool/unittest/unittest.go

@@ -25,6 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -184,7 +185,8 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
+	var ab flagutil.ArrayBool
+	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
 		switch r.URL.Path {
 		case "/prometheus/api/v1/query":
 			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
@@ -225,7 +227,7 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop(httpListenAddr); err != nil {
+	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
 		logger.Errorf("cannot stop the webservice: %s", err)
 	}
 	vmstorage.Stop()

app/vmalert/Makefile

@@ -68,6 +68,7 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier

app/vmalert/config/testdata/rules/kube-good.rules

@@ -1158,9 +1158,9 @@
           $labels.pod }}.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
       expr: |
-        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
+        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace)
           /
-        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
+        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace)
           > ( 25 / 100 )
       for: 15m
       labels:

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -22,6 +22,7 @@ groups:
               {{ . | first | value }}
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
+          link: http://localhost:3000/d/wNf0q_kZk?viewPanel=51&from={{($activeAt.Add (parseDurationTime "1h")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime "-1h")).UnixMilli}}
       - alert: ExampleAlertAlwaysFiring
         update_entries_limit: -1
         expr: sum by(job)

app/vmalert/datasource/init.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -37,13 +38,15 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
-	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
-	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
-	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
-	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
-	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url")
+	oauth2EndpointParams   = flag.String("datasource.oauth2.endpointParams", "", "Optional OAuth2 endpoint parameters to use for -datasource.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url")
+	oauth2Scopes   = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
-	lookBack = flag.Duration("datasource.lookback", 0, `Will be deprecated soon, please adjust "-search.latencyOffset"  at datasource side `+
+	lookBack = flag.Duration("datasource.lookback", 0, `Deprecated: please adjust "-search.latencyOffset" at datasource side `+
 		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
 		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
@@ -88,10 +91,10 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		logger.Warnf("flag `-datasource.queryTimeAlignment` is deprecated and will be removed in next releases. Please use `eval_alignment` in rule group instead.")
 	}
 	if *lookBack != 0 {
-		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
+		logger.Warnf("flag `-datasource.lookback` is deprecated and will be removed in next releases. Please adjust `-search.latencyOffset` at datasource side or specify `latency_offset` in rule group's params. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
 	}
 
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
@@ -108,10 +111,14 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}
 
+	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
+	}
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
@@ -126,7 +133,6 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
-		lookBack:         *lookBack,
 		queryStep:        *queryStep,
 		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,

app/vmalert/datasource/vm.go

@@ -35,7 +35,6 @@ type VMStorage struct {
 	authCfg          *promauth.Config
 	datasourceURL    string
 	appendTypePrefix bool
-	lookBack         time.Duration
 	queryStep        time.Duration
 	dataSourceType   datasourceType
 
@@ -63,7 +62,6 @@ func (s *VMStorage) Clone() *VMStorage {
 		authCfg:          s.authCfg,
 		datasourceURL:    s.datasourceURL,
 		appendTypePrefix: s.appendTypePrefix,
-		lookBack:         s.lookBack,
 		queryStep:        s.queryStep,
 
 		dataSourceType:     s.dataSourceType,
@@ -122,13 +120,12 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
 	return &VMStorage{
 		c:                c,
 		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
 		appendTypePrefix: appendTypePrefix,
-		lookBack:         lookBack,
 		queryStep:        queryStep,
 		dataSourceType:   datasourcePrometheus,
 		extraParams:      url.Values{},
@@ -137,11 +134,11 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := s.newQueryRequest(query, ts)
+	req, err := s.newQueryRequest(ctx, query, ts)
 	if err != nil {
 		return Result{}, nil, err
 	}
-	resp, err := s.do(ctx, req)
+	resp, err := s.do(req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -149,11 +146,11 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRequest(query, ts)
+		req, err = s.newQueryRequest(ctx, query, ts)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(ctx, req)
+		resp, err = s.do(req)
 		if err != nil {
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
@@ -182,11 +179,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	req, err := s.newQueryRangeRequest(query, start, end)
+	req, err := s.newQueryRangeRequest(ctx, query, start, end)
 	if err != nil {
 		return res, err
 	}
-	resp, err := s.do(ctx, req)
+	resp, err := s.do(req)
 	if err != nil {
 		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// Return unexpected error to the caller.
@@ -194,11 +191,11 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 		}
 		// Something in the middle between client and datasource might be closing
 		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = s.newQueryRangeRequest(query, start, end)
+		req, err = s.newQueryRangeRequest(ctx, query, start, end)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
-		resp, err = s.do(ctx, req)
+		resp, err = s.do(req)
 		if err != nil {
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
@@ -210,7 +207,7 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	return res, err
 }
 
-func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
 	ru := req.URL.Redacted()
 	if *showDatasourceURL {
 		ru = req.URL.String()
@@ -218,7 +215,7 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	if s.debug {
 		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru)
 	}
-	resp, err := s.c.Do(req.WithContext(ctx))
+	resp, err := s.c.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", ru, err)
 	}
@@ -230,8 +227,8 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	return resp, nil
 }
 
-func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*http.Request, error) {
-	req, err := s.newRequest()
+func (s *VMStorage) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) {
+	req, err := s.newRequest(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -239,8 +236,8 @@ func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*h
 	return req, nil
 }
 
-func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request, error) {
-	req, err := s.newRequest()
+func (s *VMStorage) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) {
+	req, err := s.newRequest(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create query request to datasource %q: %w", s.datasourceURL, err)
 	}
@@ -248,15 +245,15 @@ func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request,
 	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
 	case datasourceGraphite:
-		s.setGraphiteReqParams(req, query, ts)
+		s.setGraphiteReqParams(req, query)
 	default:
 		logger.Panicf("BUG: engine not found: %q", s.dataSourceType)
 	}
 	return req, nil
 }
 
-func (s *VMStorage) newRequest() (*http.Request, error) {
-	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
+func (s *VMStorage) newRequest(ctx context.Context) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}

app/vmalert/datasource/vm_graphite_api.go

@@ -4,8 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strconv"
-	"time"
 )
 
 type graphiteResponse []graphiteResponseTarget
@@ -48,17 +46,13 @@ const (
 	graphitePrefix = "/graphite"
 )
 
-func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestamp time.Time) {
+func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string) {
 	if s.appendTypePrefix {
 		r.URL.Path += graphitePrefix
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
 	from := "-5min"
-	if s.lookBack > 0 {
-		lookBack := timestamp.Add(-s.lookBack)
-		from = strconv.FormatInt(lookBack.Unix(), 10)
-	}
 	q.Set("from", from)
 	q.Set("format", "json")
 	q.Set("target", query)

app/vmalert/datasource/vm_prom_api.go

@@ -161,9 +161,6 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
-	if s.lookBack > 0 {
-		timestamp = timestamp.Add(-s.lookBack)
-	}
 	q.Set("time", timestamp.Format(time.RFC3339))
 	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older

app/vmalert/datasource/vm_test.go

@@ -71,7 +71,7 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
 		}
 	})
-	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
+	mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
 		c++
 		switch c {
 		case 8:
@@ -86,7 +86,7 @@ func TestVMInstantQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
+	s := NewVMStorage(srv.URL, authCfg, 0, false, srv.Client())
 
 	p := datasourcePrometheus
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
@@ -225,7 +225,7 @@ func TestVMInstantQueryWithRetry(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, nil, time.Minute, 0, false, srv.Client())
+	s := NewVMStorage(srv.URL, nil, 0, false, srv.Client())
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus)})
 
 	expErr := func(err string) {
@@ -334,7 +334,7 @@ func TestVMRangeQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, *queryStep, false, srv.Client())
+	s := NewVMStorage(srv.URL, authCfg, *queryStep, false, srv.Client())
 
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})
 
@@ -487,17 +487,6 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, "bar", p)
 			},
 		},
-		{
-			"lookback",
-			false,
-			&VMStorage{
-				lookBack: time.Minute,
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "time": {timestamp.Add(-time.Minute).Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
 		{
 			"evaluation interval",
 			false,
@@ -510,20 +499,6 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
-		{
-			"lookback + evaluation interval",
-			false,
-			&VMStorage{
-				lookBack:           time.Minute,
-				evaluationInterval: 15 * time.Second,
-			},
-			func(t *testing.T, r *http.Request) {
-				evalInterval := 15 * time.Second
-				tt := timestamp.Add(-time.Minute)
-				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
 		{
 			"step override",
 			false,
@@ -637,7 +612,7 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequest()
+			req, err := tc.vm.newRequest(ctx)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -649,7 +624,7 @@ func TestRequestParams(t *testing.T) {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
 			case datasourceGraphite:
-				tc.vm.setGraphiteReqParams(req, query, timestamp)
+				tc.vm.setGraphiteReqParams(req, query)
 			}
 			tc.checkFn(t, req)
 		})
@@ -735,7 +710,7 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req, err := vm.newQueryRequest("foo", time.Now())
+			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
 			if err != nil {
 				t.Fatal(err)
 			}

app/vmalert/main.go

@@ -59,8 +59,8 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -96,7 +96,6 @@ func main() {
 	notifier.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if !*remoteReadIgnoreRestoreErrors {
 		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
@@ -118,9 +117,9 @@ func main() {
 		return
 	}
 
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	eu, err := getExternalURL(*externalURL)
 	if err != nil {
-		logger.Fatalf("failed to init `external.url`: %s", err)
+		logger.Fatalf("failed to init `-external.url`: %s", err)
 	}
 
 	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
@@ -179,12 +178,19 @@ func main() {
 
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8880"}
+	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	pushmetrics.Stop()
+
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
@@ -194,7 +200,7 @@ func main() {
 var (
 	configReloads      = metrics.NewCounter(`vmalert_config_last_reload_total`)
 	configReloadErrors = metrics.NewCounter(`vmalert_config_last_reload_errors_total`)
-	configSuccess      = metrics.NewCounter(`vmalert_config_last_reload_successful`)
+	configSuccess      = metrics.NewGauge(`vmalert_config_last_reload_successful`, nil)
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -243,16 +249,34 @@ func newManager(ctx context.Context) (*manager, error) {
 	return manager, nil
 }
 
-func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
-	if externalURL != "" {
-		return url.Parse(externalURL)
+func getExternalURL(customURL string) (*url.URL, error) {
+	if customURL == "" {
+		// use local hostname as external URL
+		listenAddr := ":8880"
+		if len(*httpListenAddrs) > 0 {
+			listenAddr = (*httpListenAddrs)[0]
+		}
+		isTLS := httpserver.IsTLS(0)
+
+		return getHostnameAsExternalURL(listenAddr, isTLS)
 	}
-	hname, err := os.Hostname()
+	u, err := url.Parse(customURL)
 	if err != nil {
 		return nil, err
 	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, fmt.Errorf("invalid scheme %q in url %q, only 'http' and 'https' are supported", u.Scheme, u.String())
+	}
+	return u, nil
+}
+
+func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get hostname: %w", err)
+	}
 	port := ""
-	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+	if ipport := strings.Split(addr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]
 	}
 	schema := "http://"
@@ -280,7 +304,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		qFn := func(query string) ([]datasource.Metric, error) {
+		qFn := func(_ string) ([]datasource.Metric, error) {
 			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
 		}
 		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)

app/vmalert/main_test.go

@@ -22,22 +22,29 @@ func init() {
 }
 
 func TestGetExternalURL(t *testing.T) {
-	expURL := "https://vicotriametrics.com/path"
-	u, err := getExternalURL(expURL, "", false)
+	invalidURL := "victoriametrics.com/path"
+	_, err := getExternalURL(invalidURL)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+
+	expURL := "https://victoriametrics.com/path"
+	u, err := getExternalURL(expURL)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
 	}
+
 	h, _ := os.Hostname()
-	expURL = fmt.Sprintf("https://%s:4242", h)
-	u, err = getExternalURL("", "0.0.0.0:4242", true)
+	expURL = fmt.Sprintf("http://%s:8880", h)
+	u, err = getExternalURL("")
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -134,7 +141,7 @@ groups:
 				t.Fatalf("expected to have config error %s; got nil instead", cErr)
 			}
 			if cfgSuc != 0 {
-				t.Fatalf("expected to have metric configSuccess to be set to 0; got %d instead", cfgSuc)
+				t.Fatalf("expected to have metric configSuccess to be set to 0; got %v instead", cfgSuc)
 			}
 			return
 		}
@@ -143,7 +150,7 @@ groups:
 			t.Fatalf("unexpected config error: %s", cErr)
 		}
 		if cfgSuc != 1 {
-			t.Fatalf("expected to have metric configSuccess to be set to 1; got %d instead", cfgSuc)
+			t.Fatalf("expected to have metric configSuccess to be set to 1; got %v instead", cfgSuc)
 		}
 	}
 

app/vmalert/manager.go

@@ -156,11 +156,14 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
+			// cancel evaluation so the Update will be applied as fast as possible.
+			// it is important to call InterruptEval before the update, because cancel fn
+			// can be re-assigned during the update.
+			item.old.InterruptEval()
 			go func(old *rule.Group, new *rule.Group) {
 				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
-			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alert_test.go

@@ -178,7 +178,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		},
 	}
 
-	qFn := func(q string) ([]datasource.Metric, error) {
+	qFn := func(_ string) ([]datasource.Metric, error) {
 		return []datasource.Metric{
 			{
 				Labels: []datasource.Label{

app/vmalert/notifier/alertmanager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -104,7 +105,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 	if *showNotifierURL {
 		amURL = am.addr.String()
 	}
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode/100 != 2 {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return fmt.Errorf("failed to read response from %q: %w", amURL, err)
@@ -127,7 +128,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
@@ -144,7 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/init.go

@@ -46,6 +46,8 @@ var (
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 	oauth2ClientSecretFile = flagutil.NewArrayString("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2EndpointParams = flagutil.NewArrayString("notifier.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -notifier.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
 	oauth2TokenURL = flagutil.NewArrayString("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 	oauth2Scopes = flagutil.NewArrayString("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
@@ -141,6 +143,11 @@ func InitSecretFlags() {
 func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 	var notifiers []Notifier
 	for i, addr := range *addrs {
+		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(i)
+		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse JSON for -notifier.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
+		}
 		authCfg := promauth.HTTPClientConfig{
 			TLSConfig: &promauth.TLSConfig{
 				CAFile:             tlsCAFile.GetOptionalArg(i),
@@ -160,6 +167,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				ClientID:         oauth2ClientID.GetOptionalArg(i),
 				ClientSecret:     promauth.NewSecret(oauth2ClientSecret.GetOptionalArg(i)),
 				ClientSecretFile: oauth2ClientSecretFile.GetOptionalArg(i),
+				EndpointParams:   endpointParams,
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},

app/vmalert/remoteread/init.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -41,8 +42,10 @@ var (
 	oauth2ClientID         = flag.String("remoteRead.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteRead.url.")
 	oauth2ClientSecret     = flag.String("remoteRead.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteRead.url.")
 	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
-	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
-	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
+	oauth2EndpointParams   = flag.String("remoteRead.oauth2.endpointParams", "", "Optional OAuth2 endpoint parameters to use for -remoteRead.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
+	oauth2Scopes   = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -58,19 +61,23 @@ func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
-	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
+	}
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -123,14 +123,12 @@ func (c *Client) Push(s prompbmarshal.TimeSeries) error {
 	case <-c.doneCh:
 		rwErrors.Inc()
 		droppedRows.Add(len(s.Samples))
-		droppedBytes.Add(s.Size())
 		return fmt.Errorf("client is closed")
 	case c.input <- s:
 		return nil
 	default:
 		rwErrors.Inc()
 		droppedRows.Add(len(s.Samples))
-		droppedBytes.Add(s.Size())
 		return fmt.Errorf("failed to push timeseries - queue is full (%d entries). "+
 			"Queue size is controlled by -remoteWrite.maxQueueSize flag",
 			c.maxQueueSize)
@@ -153,12 +151,22 @@ func (c *Client) run(ctx context.Context) {
 	ticker := time.NewTicker(c.flushInterval)
 	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
+		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
+		logger.Infof("shutting down remote write client and flushing remained series")
+
+		shutdownFlushCnt := 0
 		for ts := range c.input {
 			wr.Timeseries = append(wr.Timeseries, ts)
+			if len(wr.Timeseries) >= c.maxBatchSize {
+				shutdownFlushCnt += len(wr.Timeseries)
+				c.flush(lastCtx, wr)
+			}
 		}
-		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
-		logger.Infof("shutting down remote write client and flushing remained %d series", len(wr.Timeseries))
+		// flush the last batch. `flush` will re-check and avoid flushing empty batch.
+		shutdownFlushCnt += len(wr.Timeseries)
 		c.flush(lastCtx, wr)
+
+		logger.Infof("shutting down remote write client flushed %d series", shutdownFlushCnt)
 		cancel()
 	}
 	c.wg.Add(1)
@@ -195,7 +203,6 @@ var (
 	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
 	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
 	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
 	sendDuration        = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
 	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 
@@ -211,15 +218,10 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 	if len(wr.Timeseries) < 1 {
 		return
 	}
-	defer prompbmarshal.ResetWriteRequest(wr)
+	defer wr.Reset()
 	defer bufferFlushDuration.UpdateDuration(time.Now())
 
-	data, err := wr.Marshal()
-	if err != nil {
-		logger.Errorf("failed to marshal WriteRequest: %s", err)
-		return
-	}
-
+	data := wr.MarshalProtobuf(nil)
 	b := snappy.Encode(nil, data)
 
 	retryInterval, maxRetryInterval := *retryMinInterval, *retryMaxTime
@@ -276,15 +278,18 @@ L:
 	}
 
 	rwErrors.Inc()
-	droppedRows.Add(len(wr.Timeseries))
-	droppedBytes.Add(len(b))
+	rows := 0
+	for _, ts := range wr.Timeseries {
+		rows += len(ts.Samples)
+	}
+	droppedRows.Add(rows)
 	logger.Errorf("attempts to send remote-write request failed - dropping %d time series",
 		len(wr.Timeseries))
 }
 
 func (c *Client) send(ctx context.Context, data []byte) error {
 	r := bytes.NewReader(data)
-	req, err := http.NewRequest(http.MethodPost, c.addr, r)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
@@ -307,7 +312,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
-	resp, err := c.c.Do(req.WithContext(ctx))
+	resp, err := c.c.Do(req)
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
 			req.URL.Redacted(), err, len(data), r.Size())

app/vmalert/remotewrite/client_test.go

@@ -84,6 +84,70 @@ func TestClient_Push(t *testing.T) {
 	}
 }
 
+func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
+	batchSize := 20
+
+	testTable := []struct {
+		name     string // name of the test case
+		pushCnt  int    // how many time series is pushed to the client
+		batchCnt int    // the expected batch count sent by the client
+	}{
+		{
+			name:     "pushCnt % batchSize == 0",
+			pushCnt:  batchSize * 40,
+			batchCnt: 40,
+		},
+		{
+			name:     "pushCnt % batchSize != 0",
+			pushCnt:  batchSize*40 + 1,
+			batchCnt: 40 + 1,
+		},
+	}
+
+	for _, tt := range testTable {
+		t.Run(tt.name, func(t *testing.T) {
+			// run new server
+			bcServer := newBatchCntRWServer()
+
+			// run new client
+			rwClient, err := NewClient(context.Background(), Config{
+				MaxBatchSize: batchSize,
+
+				// Set everything to 1 to simplify the calculation.
+				Concurrency:   1,
+				MaxQueueSize:  1000,
+				FlushInterval: time.Minute,
+
+				// batch count server
+				Addr: bcServer.URL,
+			})
+			if err != nil {
+				t.Fatalf("new remote write client failed, err: %v", err)
+			}
+
+			// push time series to the client.
+			for i := 0; i < tt.pushCnt; i++ {
+				if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
+					t.Fatalf("push time series to the client failed, err: %v", err)
+				}
+			}
+
+			// close the client so the rest ts will be flushed in `shutdown`
+			if err = rwClient.Close(); err != nil {
+				t.Fatalf("shutdown client failed, err: %v", err)
+			}
+
+			// finally check how many batches is sent.
+			if tt.batchCnt != bcServer.acceptedBatches() {
+				t.Errorf("client sent batch count incorrect, want: %d, get: %d", tt.batchCnt, bcServer.acceptedBatches())
+			}
+			if tt.pushCnt != bcServer.accepted() {
+				t.Errorf("client sent time series count incorrect, want: %d, get: %d", tt.pushCnt, bcServer.accepted())
+			}
+		})
+	}
+}
+
 func newRWServer() *rwServer {
 	rw := &rwServer{}
 	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
@@ -91,14 +155,12 @@ func newRWServer() *rwServer {
 }
 
 type rwServer struct {
-	// WARN: ordering of fields is important for alignment!
-	// see https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	acceptedRows uint64
+	acceptedRows atomic.Uint64
 	*httptest.Server
 }
 
 func (rw *rwServer) accepted() int {
-	return int(atomic.LoadUint64(&rw.acceptedRows))
+	return int(rw.acceptedRows.Load())
 }
 
 func (rw *rwServer) err(w http.ResponseWriter, err error) {
@@ -140,11 +202,11 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	wr := &prompb.WriteRequest{}
-	if err := wr.Unmarshal(b); err != nil {
+	if err := wr.UnmarshalProtobuf(b); err != nil {
 		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))
 		return
 	}
-	atomic.AddUint64(&rw.acceptedRows, uint64(len(wr.Timeseries)))
+	rw.acceptedRows.Add(uint64(len(wr.Timeseries)))
 	w.WriteHeader(http.StatusNoContent)
 }
 
@@ -186,3 +248,27 @@ func (frw *faultyRWServer) handler(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("server overloaded"))
 	}
 }
+
+type batchCntRWServer struct {
+	*rwServer
+
+	batchCnt atomic.Int64 // accepted batch count, which also equals to request count
+}
+
+func newBatchCntRWServer() *batchCntRWServer {
+	bc := &batchCntRWServer{
+		rwServer: &rwServer{},
+	}
+
+	bc.Server = httptest.NewServer(http.HandlerFunc(bc.handler))
+	return bc
+}
+
+func (bc *batchCntRWServer) handler(w http.ResponseWriter, r *http.Request) {
+	bc.batchCnt.Add(1)
+	bc.rwServer.handler(w, r)
+}
+
+func (bc *batchCntRWServer) acceptedBatches() int {
+	return int(bc.batchCnt.Load())
+}

app/vmalert/remotewrite/debug_client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -30,7 +30,7 @@ func NewDebugClient() (*DebugClient, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
@@ -49,10 +49,7 @@ func (c *DebugClient) Push(s prompbmarshal.TimeSeries) error {
 	c.wg.Add(1)
 	defer c.wg.Done()
 	wr := &prompbmarshal.WriteRequest{Timeseries: []prompbmarshal.TimeSeries{s}}
-	data, err := wr.Marshal()
-	if err != nil {
-		return fmt.Errorf("failed to marshal the given time series: %w", err)
-	}
+	data := wr.MarshalProtobuf(nil)
 
 	return c.send(data)
 }

app/vmalert/remotewrite/init.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
 
 var (
@@ -41,11 +42,13 @@ var (
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default, the server name from -remoteWrite.url is used")
 
-	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url.")
-	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url.")
-	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
-	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
-	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
+	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url")
+	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url")
+	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url")
+	oauth2EndpointParams   = flag.String("remoteWrite.oauth2.endpointParams", "", "Optional OAuth2 endpoint parameters to use for -remoteWrite.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
+	oauth2Scopes   = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -62,15 +65,19 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, nil
 	}
 
-	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
+	}
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)

app/vmalert/rule/alerting.go

@@ -237,11 +237,30 @@ type labelSet struct {
 	origin map[string]string
 	// processed labels includes origin labels
 	// plus extra labels (group labels, service labels like alertNameLabel).
-	// in case of conflicts, extra labels are preferred.
+	// in case of key conflicts, origin labels are renamed with prefix `exported_` and extra labels are preferred.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5161
 	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
 	processed map[string]string
 }
 
+// add adds a value v with key k to origin and processed label sets.
+// On k conflicts in processed set, the passed v is preferred.
+// On k conflicts in origin set, the original value is preferred and copied
+// to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
+func (ls *labelSet) add(k, v string) {
+	ls.processed[k] = v
+	ov, ok := ls.origin[k]
+	if !ok {
+		ls.origin[k] = v
+		return
+	}
+	if ov != v {
+		// copy value only if v and ov are different
+		key := fmt.Sprintf("exported_%s", k)
+		ls.processed[key] = ov
+	}
+}
+
 // toLabels converts labels from given Metric
 // to labelSet which contains original and processed labels.
 func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
@@ -267,24 +286,14 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		return nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
 	for k, v := range extraLabels {
-		ls.processed[k] = v
-		if _, ok := ls.origin[k]; !ok {
-			ls.origin[k] = v
-		}
+		ls.add(k, v)
 	}
-
 	// set additional labels to identify group and rule name
 	if ar.Name != "" {
-		ls.processed[alertNameLabel] = ar.Name
-		if _, ok := ls.origin[alertNameLabel]; !ok {
-			ls.origin[alertNameLabel] = ar.Name
-		}
+		ls.add(alertNameLabel, ar.Name)
 	}
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
-		ls.processed[alertGroupNameLabel] = ar.GroupName
-		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
-			ls.origin[alertGroupNameLabel] = ar.GroupName
-		}
+		ls.add(alertGroupNameLabel, ar.GroupName)
 	}
 	return ls, nil
 }
@@ -301,7 +310,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 	}
 	var result []prompbmarshal.TimeSeries
 	holdAlertState := make(map[uint64]*notifier.Alert)
-	qFn := func(query string) ([]datasource.Metric, error) {
+	qFn := func(_ string) ([]datasource.Metric, error) {
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range res.Data {
@@ -315,16 +324,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 
-		// if alert is instant, For: 0
-		if ar.For == 0 {
-			a.State = notifier.StateFiring
-			for i := range s.Values {
-				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
-			}
-			continue
-		}
-
-		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
@@ -345,6 +344,10 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.Start = at
 			}
 			prevT = at
+			if ar.For == 0 {
+				// rules with `for: 0` are always firing when they have Value
+				a.State = notifier.StateFiring
+			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
 			// save alert's state on last iteration, so it can be used on the next execRange call
@@ -414,8 +417,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		}
 		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
-			// duplicate may be caused by extra labels
-			// conflicting with the metric labels
+			// duplicate may be caused the removal of `__name__` label
 			curState.Err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
 			return nil, curState.Err
 		}
@@ -438,14 +440,13 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, start, qFn)
+		a, err := ar.newAlert(m, ls, ts, qFn)
 		if err != nil {
 			curState.Err = fmt.Errorf("failed to create alert: %w", err)
 			return nil, curState.Err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
-		a.ActiveAt = ts
 		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
@@ -471,7 +472,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -551,9 +552,9 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 }
 
 const (
-	// alertMetricName is the metric name for synthetic alert timeseries.
+	// alertMetricName is the metric name for time series reflecting the alert state.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for 'for' state of alert.
+	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -568,12 +569,10 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(a, timestamp))
-	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(a, timestamp))
+	return []prompbmarshal.TimeSeries{
+		alertToTimeSeries(a, timestamp),
+		alertForToTimeSeries(a, timestamp),
 	}
-	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
@@ -656,15 +655,19 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 // alertsToSend walks through the current alerts of AlertingRule
 // and returns only those which should be sent to notifier.
 // Isn't concurrent safe.
-func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+func (ar *AlertingRule) alertsToSend(resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	currentTime := time.Now()
 	needsSending := func(a *notifier.Alert) bool {
 		if a.State == notifier.StatePending {
 			return false
 		}
-		if a.ResolvedAt.After(a.LastSent) {
+		if a.State == notifier.StateFiring && a.End.Before(a.LastSent) {
 			return true
 		}
-		return a.LastSent.Add(resendDelay).Before(ts)
+		if a.State == notifier.StateInactive && a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(currentTime)
 	}
 
 	var alerts []notifier.Alert
@@ -672,11 +675,11 @@ func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay
 		if !needsSending(a) {
 			continue
 		}
-		a.End = ts.Add(resolveDuration)
+		a.End = currentTime.Add(resolveDuration)
 		if a.State == notifier.StateInactive {
 			a.End = a.ResolvedAt
 		}
-		a.LastSent = ts
+		a.LastSent = currentTime
 		alerts = append(alerts, *a)
 	}
 	return alerts

app/vmalert/rule/alerting_test.go

@@ -28,20 +28,28 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				"job":      "foo",
-				"instance": "bar",
-			}},
+			&notifier.Alert{
+				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					"job":      "foo",
+					"instance": "bar",
+				},
+			},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -49,19 +57,35 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+						"job":      "foo",
+						"instance": "bar",
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
-				alertStateLabel: "foo",
-				"__name__":      "bar",
-			}},
+			&notifier.Alert{
+				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
+				Labels: map[string]string{
+					alertStateLabel: "foo",
+					"__name__":      "bar",
+				},
+			},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__":      alertForStateMetricName,
+						alertStateLabel: "foo",
+					}),
 			},
 		},
 		{
@@ -308,14 +332,17 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &datasource.FakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
+			ts := time.Now()
 			for i, step := range tc.steps {
 				fq.Reset()
 				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
+				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-				// artificial delay between applying steps
-				time.Sleep(defaultStep)
+
+				// shift the execution timestamp before the next iteration
+				ts = ts.Add(defaultStep)
+
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -367,7 +394,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 			nil,
 		},
@@ -378,8 +405,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels: map[string]string{"name": "foo"},
-					State:  notifier.StateFiring,
+					Labels:   map[string]string{"name": "foo"},
+					State:    notifier.StateFiring,
+					ActiveAt: time.Unix(1, 0),
 				},
 			},
 			nil,
@@ -390,9 +418,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
-				{State: notifier.StateFiring},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
 			},
 			nil,
 		},
@@ -460,6 +488,20 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				For:         time.Second,
 			}},
 		},
+		{
+			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
+			},
+			nil,
+		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
 			[]datasource.Metric{
@@ -534,21 +576,33 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"source": "vm",
-				}},
+				{
+					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					},
+				},
+				{
+					State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
+					Labels: map[string]string{
+						"source": "vm",
+					},
+				},
 				//
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
-				{State: notifier.StateFiring, Labels: map[string]string{
-					"foo":    "bar",
-					"source": "vm",
-				}},
+				{
+					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					},
+				},
+				{
+					State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo":    "bar",
+						"source": "vm",
+					},
+				},
 			},
 			nil,
 		},
@@ -768,14 +822,16 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	ar.q = fq
 
 	// successful attempt
+	// label `job` will be overridden by rule extra label, the original value will be reserved by "exported_job"
 	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
 	_, err := ar.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// label `job` will collide with rule extra label and will make both time series equal
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	// label `__name__` will be omitted and get duplicated results here
+	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo_1", "job", "bar"))
 	_, err = ar.exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
@@ -899,20 +955,22 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
+				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 					Labels: map[string]string{
-						alertNameLabel: "override label",
-						"instance":     "foo",
+						alertNameLabel:       "override label",
+						"exported_alertname": "override",
+						"instance":           "foo",
 					},
 					Annotations: map[string]string{
 						"summary":     `first: Too high connection number for "foo"`,
 						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
+				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "bar"}): {
 					Labels: map[string]string{
-						alertNameLabel: "override label",
-						"instance":     "bar",
+						alertNameLabel:       "override label",
+						"exported_alertname": "override",
+						"instance":           "bar",
 					},
 					Annotations: map[string]string{
 						"summary":     `second: Too high connection number for "bar"`,
@@ -941,14 +999,18 @@ func TestAlertingRule_Template(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{
-					alertNameLabel:      "OriginLabels",
-					alertGroupNameLabel: "Testing",
-					"instance":          "foo",
+					alertNameLabel:        "OriginLabels",
+					"exported_alertname":  "originAlertname",
+					alertGroupNameLabel:   "Testing",
+					"exported_alertgroup": "originGroupname",
+					"instance":            "foo",
 				}): {
 					Labels: map[string]string{
-						alertNameLabel:      "OriginLabels",
-						alertGroupNameLabel: "Testing",
-						"instance":          "foo",
+						alertNameLabel:        "OriginLabels",
+						"exported_alertname":  "originAlertname",
+						alertGroupNameLabel:   "Testing",
+						"exported_alertgroup": "originGroupname",
+						"instance":            "foo",
 					},
 					Annotations: map[string]string{
 						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
@@ -992,7 +1054,7 @@ func TestAlertsToSend(t *testing.T) {
 		for i, a := range alerts {
 			ar.alerts[uint64(i)] = a
 		}
-		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		gotAlerts := ar.alertsToSend(resolveDuration, resendDelay)
 		if gotAlerts == nil && expAlerts == nil {
 			return
 		}
@@ -1008,60 +1070,36 @@ func TestAlertsToSend(t *testing.T) {
 		})
 		for i, exp := range expAlerts {
 			got := gotAlerts[i]
-			if got.LastSent != exp.LastSent {
-				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
-			}
-			if got.End != exp.End {
-				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			if got.Name != exp.Name {
+				t.Fatalf("expected Name to be %v; got %v", exp.Name, got.Name)
 			}
 		}
 	}
 
-	f( // send firing alert with custom resolve time
-		[]*notifier.Alert{{State: notifier.StateFiring}},
-		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+	f( // check if firing alerts need to be sent with non-zero resendDelay
+		[]*notifier.Alert{
+			{Name: "a", State: notifier.StateFiring, Start: ts},
+			// no need to resend firing
+			{Name: "b", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(5 * time.Minute)},
+			// last message is for resolved, send firing message this time
+			{Name: "c", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(-1 * time.Minute)},
+			// resend firing
+			{Name: "d", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-1 * time.Minute)},
+		},
+		[]*notifier.Alert{{Name: "a"}, {Name: "c"}, {Name: "d"}},
 		5*time.Minute, time.Minute,
 	)
-	f( // resolve inactive alert at the current timestamp
-		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
-		[]*notifier.Alert{{LastSent: ts, End: ts}},
-		time.Minute, time.Minute,
-	)
-	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
-		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
-		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+	f( // check if resolved alerts need to be sent with non-zero resendDelay
+		[]*notifier.Alert{
+			{Name: "a", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts.Add(-30 * time.Second)},
+			// no need to resend resolved
+			{Name: "b", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts},
+			// resend resolved
+			{Name: "c", State: notifier.StateInactive, ResolvedAt: ts.Add(-1 * time.Minute), LastSent: ts.Add(-1 * time.Minute)},
+		},
+		[]*notifier.Alert{{Name: "a"}, {Name: "c"}},
 		5*time.Minute, time.Minute,
 	)
-	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
-		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
-		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
-		5*time.Minute, time.Minute,
-	)
-	f( // attempt to send alert that was already sent in the resendDelay interval
-		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
-		nil,
-		time.Minute, time.Minute,
-	)
-	f( // attempt to send alert that was sent out of the resendDelay interval
-		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
-		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
-		time.Minute, time.Minute,
-	)
-	f( // alert must be sent even if resendDelay interval is 0
-		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
-		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
-		time.Minute, 0,
-	)
-	f( // inactive alert which has been sent already
-		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
-		nil,
-		time.Minute, time.Minute,
-	)
-	f( // inactive alert which has been resolved after last send
-		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
-		[]*notifier.Alert{{LastSent: ts, End: ts}},
-		time.Minute, time.Minute,
-	)
 }
 
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
@@ -1087,8 +1125,65 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 	return &rule
 }
 
+func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
+	rule := newTestAlertingRule(name, waitFor)
+	rule.EvalInterval = evalInterval
+	return rule
+}
+
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor
 	return rule
 }
+
+func TestAlertingRule_ToLabels(t *testing.T) {
+	metric := datasource.Metric{
+		Labels: []datasource.Label{
+			{Name: "instance", Value: "0.0.0.0:8800"},
+			{Name: "group", Value: "vmalert"},
+			{Name: "alertname", Value: "ConfigurationReloadFailure"},
+		},
+		Values:     []float64{1},
+		Timestamps: []int64{time.Now().UnixNano()},
+	}
+
+	ar := &AlertingRule{
+		Labels: map[string]string{
+			"instance": "override", // this should override instance with new value
+			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
+		},
+		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
+		Name:      "AlertingRulesError",
+		GroupName: "vmalert",
+	}
+
+	expectedOriginLabels := map[string]string{
+		"instance":   "0.0.0.0:8800",
+		"group":      "vmalert",
+		"alertname":  "ConfigurationReloadFailure",
+		"alertgroup": "vmalert",
+	}
+
+	expectedProcessedLabels := map[string]string{
+		"instance":           "override",
+		"exported_instance":  "0.0.0.0:8800",
+		"alertname":          "AlertingRulesError",
+		"exported_alertname": "ConfigurationReloadFailure",
+		"group":              "vmalert",
+		"alertgroup":         "vmalert",
+	}
+
+	ls, err := ar.toLabels(metric, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {
+		t.Errorf("origin labels mismatch, got: %v, want: %v", ls.origin, expectedOriginLabels)
+	}
+
+	if !reflect.DeepEqual(ls.processed, expectedProcessedLabels) {
+		t.Errorf("processed labels mismatch, got: %v, want: %v", ls.processed, expectedProcessedLabels)
+	}
+}

app/vmalert/rule/group.go

@@ -704,7 +704,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return nil
 	}
 
-	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
+	alerts := ar.alertsToSend(resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
@@ -724,7 +724,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	return errGr.Err()
 }
 
-// getStaledSeries checks whether there are stale series from previously sent ones.
+// getStaleSeries checks whether there are stale series from previously sent ones.
 func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
 	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
 	for _, ts := range tss {

app/vmalert/rule/recording.go

@@ -194,6 +194,9 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSer
 	labels["__name__"] = rr.Name
 	// override existing labels with configured ones
 	for k, v := range rr.Labels {
+		if _, ok := labels[k]; ok && labels[k] != v {
+			labels[fmt.Sprintf("exported_%s", k)] = labels[k]
+		}
 		labels[k] = v
 	}
 	return newTimeSeries(m.Values, m.Timestamps, labels)
@@ -203,7 +206,7 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSer
 func (rr *RecordingRule) updateWith(r Rule) error {
 	nr, ok := r.(*RecordingRule)
 	if !ok {
-		return fmt.Errorf("BUG: attempt to update recroding rule with wrong type %#v", r)
+		return fmt.Errorf("BUG: attempt to update recording rule with wrong type %#v", r)
 	}
 	rr.Expr = nr.Expr
 	rr.Labels = nr.Labels

app/vmalert/rule/recording_test.go

@@ -61,7 +61,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
 			},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
@@ -70,9 +70,10 @@ func TestRecordingRule_Exec(t *testing.T) {
 					"source":   "test",
 				}),
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "job:foo",
-					"job":      "bar",
-					"source":   "test",
+					"__name__":        "job:foo",
+					"job":             "bar",
+					"source":          "test",
+					"exported_source": "origin",
 				}),
 			},
 		},
@@ -254,10 +255,7 @@ func TestRecordingRule_ExecNegative(t *testing.T) {
 	fq.Add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
-	if err == nil {
-		t.Fatalf("expected to get err; got nil")
-	}
-	if !strings.Contains(err.Error(), errDuplicate.Error()) {
-		t.Fatalf("expected to get err %q; got %q insterad", errDuplicate, err)
+	if err != nil {
+		t.Fatal(err)
 	}
 }

app/vmalert/static/js/custom.js

@@ -1,5 +1,12 @@
 function expandAll() {
-    $('.collapse').addClass('show');
+    $('.group-heading').each(function () {
+        let style = $(this).attr("style")
+        // display only elements that are currently visible
+        if (style === "display: none;") {
+            return
+        }
+        $(this).next().addClass('show')
+    });
 }
 
 function collapseAll() {
@@ -15,6 +22,100 @@ function toggleByID(id) {
     }
 }
 
+function debounce(func, delay) {
+    let timer;
+    return function (...args) {
+        clearTimeout(timer);
+        timer = setTimeout(() => {
+            func.apply(this, args);
+        }, delay);
+    };
+}
+
+$('#search').on("keyup", debounce(search, 500));
+
+// search shows or hides groups&rules that satisfy the search phrase.
+// case-insensitive, respects GET param `search`.
+function search() {
+    $(".rule").show();
+
+    let groupHeader = $(".group-heading")
+    let searchPhrase = $("#search").val().toLowerCase()
+    if (searchPhrase.length === 0) {
+        groupHeader.show()
+        setParamURL('search', '')
+        return
+    }
+
+    $(".rule-table").removeClass('show');
+    groupHeader.hide()
+
+    searchPhrase = searchPhrase.toLowerCase()
+    filterRuleByName(searchPhrase);
+    filterRuleByLabels(searchPhrase);
+    filterGroupsByName(searchPhrase);
+
+    setParamURL('search', searchPhrase)
+}
+
+function setParamURL(key, value) {
+    let url = new URL(location.href)
+    url.searchParams.set(key, value);
+    window.history.replaceState(null, null, `?${url.searchParams.toString()}`);
+}
+
+function getParamURL(key) {
+    let url = new URL(location.href)
+    return url.searchParams.get(key)
+}
+
+function filterGroupsByName(searchPhrase) {
+    $(".group-heading").each(function () {
+        const groupName = $(this).attr('data-group-name').toLowerCase();
+        const hasValue = groupName.indexOf(searchPhrase) >= 0
+
+        if (!hasValue) {
+            return
+        }
+
+        const target = $(this).attr("data-bs-target");
+        $(`div[id="${target}"] .rule`).show();
+        $(this).show();
+    });
+}
+
+function filterRuleByName(searchPhrase) {
+    $(".rule").each(function () {
+        const ruleName = $(this).attr("data-rule-name").toLowerCase();
+        const hasValue = ruleName.indexOf(searchPhrase) >= 0
+        if (!hasValue) {
+            $(this).hide();
+            return
+        }
+
+        const target = $(this).attr('data-bs-target')
+        $(`#rules-${target}`).addClass('show');
+        $(`div[data-bs-target='rules-${target}']`).show();
+        $(this).show();
+    });
+}
+
+function filterRuleByLabels(searchPhrase) {
+    $(".rule").each(function () {
+        const matches = $(".label", this).filter(function () {
+            const label = $(this).text().toLowerCase();
+            return label.indexOf(searchPhrase) >= 0;
+        }).length;
+
+        if (matches > 0) {
+            const target = $(this).attr('data-bs-target')
+            $(`#rules-${target}`).addClass('show');
+            $(`div[data-bs-target='rules-${target}']`).show();
+            $(this).show();
+        }
+    });
+}
+
 $(document).ready(function () {
     $(".group-heading a").click(function (e) {
         e.stopPropagation(); // prevent collapse logic on link click
@@ -32,6 +133,13 @@ $(document).ready(function () {
         });
     });
 
+    // update search element with value from URL, if any
+    let searchPhrase = getParamURL('search')
+    $("#search").val(searchPhrase)
+
+    // apply filtering by search phrase
+    search()
+
     let hash = window.location.hash.substr(1);
     toggleByID(hash);
 });

app/vmalert/templates/template.go

@@ -476,7 +476,7 @@ func templateFuncs() textTpl.FuncMap {
 		// For example, {{ query "foo" | first | value }} will
 		// execute "/api/v1/query?query=foo" request and will return
 		// the first value in response.
-		"query": func(q string) ([]metric, error) {
+		"query": func(_ string) ([]metric, error) {
 			// query function supposed to be substituted at FuncsWithQuery().
 			// it is present here only for validation purposes, when there is no
 			// provided datasource.

app/vmalert/utils/auth.go

@@ -45,13 +45,14 @@ func WithBearer(token, tokenFile string) AuthConfigOptions {
 }
 
 // WithOAuth returns AuthConfigOptions and set OAuth params based on given params
-func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string) AuthConfigOptions {
+func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string, endpointParams map[string]string) AuthConfigOptions {
 	return func(config *promauth.HTTPClientConfig) {
 		if clientSecretFile != "" || clientSecret != "" {
 			config.OAuth2 = &promauth.OAuth2Config{
 				ClientID:         clientID,
 				ClientSecret:     promauth.NewSecret(clientSecret),
 				ClientSecretFile: clientSecretFile,
+				EndpointParams:   endpointParams,
 				TokenURL:         tokenURL,
 				Scopes:           strings.Split(scopes, ";"),
 			}

app/vmalert/web.go

@@ -12,11 +12,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+
 var (
 	apiLinks = [][2]string{
 		// api links are relative since they can be used by external clients,
@@ -84,7 +88,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -95,12 +102,20 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		WriteListGroups(w, r, rh.groups())
+		var data []apiGroup
+		rf := extractRulesFilter(r)
+		data = rh.groups(rf)
+		WriteListGroups(w, r, data)
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		data, err := rh.listGroups()
+		var data []byte
+		var err error
+
+		rf := extractRulesFilter(r)
+		data, err = rh.listGroups(rf)
+
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -108,6 +123,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
+
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
@@ -151,6 +167,9 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+			return true
+		}
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -201,26 +220,94 @@ type listGroupsResponse struct {
 	} `json:"data"`
 }
 
-func (rh *requestHandler) groups() []apiGroup {
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
+type rulesFilter struct {
+	files         []string
+	groupNames    []string
+	ruleNames     []string
+	ruleType      string
+	excludeAlerts bool
+}
+
+func extractRulesFilter(r *http.Request) rulesFilter {
+	rf := rulesFilter{}
+
+	var ruleType string
+	ruleTypeParam := r.URL.Query().Get("type")
+	// for some reason, `type` in filter doesn't match `type` in response,
+	// so we use this matching here
+	if ruleTypeParam == "alert" {
+		ruleType = ruleTypeAlerting
+	} else if ruleTypeParam == "record" {
+		ruleType = ruleTypeRecording
+	}
+	rf.ruleType = ruleType
+
+	rf.excludeAlerts = httputils.GetBool(r, "exclude_alerts")
+	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
+	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
+	rf.files = append([]string{}, r.Form["file[]"]...)
+	return rf
+}
+
+func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]apiGroup, 0)
-	for _, g := range rh.m.groups {
-		groups = append(groups, groupToAPI(g))
+	isInList := func(list []string, needle string) bool {
+		if len(list) < 1 {
+			return true
+		}
+		for _, i := range list {
+			if i == needle {
+				return true
+			}
+		}
+		return false
 	}
 
-	// sort list of alerts for deterministic output
-	sort.Slice(groups, func(i, j int) bool {
-		return groups[i].Name < groups[j].Name
-	})
+	groups := make([]apiGroup, 0)
+	for _, group := range rh.m.groups {
+		if !isInList(rf.groupNames, group.Name) {
+			continue
+		}
+		if !isInList(rf.files, group.File) {
+			continue
+		}
 
+		g := groupToAPI(group)
+		// the returned list should always be non-nil
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
+		filteredRules := make([]apiRule, 0)
+		for _, r := range g.Rules {
+			if rf.ruleType != "" && rf.ruleType != r.Type {
+				continue
+			}
+			if !isInList(rf.ruleNames, r.Name) {
+				continue
+			}
+			if rf.excludeAlerts {
+				r.Alerts = nil
+			}
+			filteredRules = append(filteredRules, r)
+		}
+		g.Rules = filteredRules
+		groups = append(groups, g)
+	}
+	// sort list of groups for deterministic output
+	sort.Slice(groups, func(i, j int) bool {
+		a, b := groups[i], groups[j]
+		if a.Name != b.Name {
+			return a.Name < b.Name
+		}
+		return a.File < b.File
+	})
 	return groups
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups()
+	lr.Data.Groups = rh.groups(rf)
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{

app/vmalert/web.qtpl

@@ -70,15 +70,29 @@ btn-primary
                 }
             }
         %}
-         <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
-         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-         <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
-         <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
+        <div class="btn-toolbar mb-3" role="toolbar">
+          <div>
+            <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
+            <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+            <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+            <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
+            <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
+          </div>
+          <div class="col-md-4 col-lg-5">
+            <div class="px-3 input-group">
+              <div class="input-group-prepend">
+                <span class="input-group-text">
+                  <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
+                </span>
+              </div>
+              <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
+            </div>
+          </div>
+        </div>
         {%  if len(groups) > 0 %}
             {% for _, g := range groups  %}
                   <div
-                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}">
+                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
                     <span class="anchor" id="group-{%s g.ID %}"></span>
                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                      {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
@@ -100,7 +114,7 @@ btn-primary
                         </div>
                     {% endif %}
                 </div>
-                <div class="collapse" id="rules-{%s g.ID %}">
+                <div class="collapse rule-table" id="rules-{%s g.ID %}">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
                             <tr>
@@ -111,7 +125,7 @@ btn-primary
                         </thead>
                         <tbody>
                         {% for _, r := range g.Rules %}
-                            <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
+                            <tr class="rule{% if r.LastError != "" %} alert-danger{% endif %}" data-rule-name="{%s r.Name %}" data-bs-target="{%s g.ID %}">
                                 <td>
                                     <div class="row">
                                         <div class="col-12 mb-2">
@@ -134,7 +148,7 @@ btn-primary
                                         <div class="col-12 mb-2">
                                             {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
                                             {% for k, v := range r.Labels %}
-                                                    <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                                    <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
                                             {% endfor %}
                                         </div>
                                         {% if r.LastError != "" %}
@@ -170,11 +184,25 @@ btn-primary
     {%code prefix := utils.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {% if len(groupAlerts) > 0 %}
-         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         <div class="btn-toolbar mb-3" role="toolbar">
+              <div>
+                <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+                 <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+              </div>
+              <div class="col-md-4 col-lg-5">
+                <div class="px-3 input-group">
+                  <div class="input-group-prepend">
+                    <span class="input-group-text">
+                      <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
+                    </span>
+                  </div>
+                  <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
+                </div>
+              </div>
+          </div>
          {% for _, ga := range groupAlerts %}
             {%code g := ga.Group %}
-            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
                 <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
@@ -192,7 +220,7 @@ btn-primary
                 }
                 sort.Strings(keys)
             %}
-            <div class="collapse" id="rules-{%s g.ID %}">
+            <div class="collapse rule-table" id="rules-{%s g.ID %}">
                 {% for _, ruleID := range keys %}
                     {%code
                         defaultAR := alertsByRule[ruleID][0]
@@ -203,45 +231,46 @@ btn-primary
                         sort.Strings(labelKeys)
                     %}
                     <br>
-                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
-                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
-                    <br>
-                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
-                    <table class="table table-striped table-hover table-sm">
-                        <thead>
-                            <tr>
-                                <th scope="col">Labels</th>
-                                <th scope="col">State</th>
-                                <th scope="col">Active at</th>
-                                <th scope="col">Value</th>
-                                <th scope="col">Link</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                        {% for _, ar := range alertsByRule[ruleID] %}
-                            <tr>
-                                <td>
-                                    {% for _, k := range labelKeys %}
-                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
-                                    {% endfor %}
-                                </td>
-                                <td>{%= badgeState(ar.State) %}</td>
-                                <td>
-                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
-                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
-                                    {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
-                                </td>
-                                <td>{%s ar.Value %}</td>
-                                <td>
-                                    <a href="{%s prefix+ar.WebLink() %}">Details</a>
-                                </td>
-                            </tr>
-                        {% endfor %}
-                     </tbody>
-                    </table>
+                    <div class="rule" data-rule-name="{%s defaultAR.Name %}" data-bs-target="{%s g.ID %}">
+                      <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                       | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                      <br>
+                      <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                      <table class="table table-striped table-hover table-sm">
+                          <thead>
+                              <tr>
+                                  <th scope="col">Labels</th>
+                                  <th scope="col">State</th>
+                                  <th scope="col">Active at</th>
+                                  <th scope="col">Value</th>
+                                  <th scope="col">Link</th>
+                              </tr>
+                          </thead>
+                          <tbody>
+                          {% for _, ar := range alertsByRule[ruleID] %}
+                              <tr>
+                                  <td>
+                                      {% for _, k := range labelKeys %}
+                                          <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
+                                      {% endfor %}
+                                  </td>
+                                  <td>{%= badgeState(ar.State) %}</td>
+                                  <td>
+                                      {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                      {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                      {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
+                                  </td>
+                                  <td>{%s ar.Value %}</td>
+                                  <td>
+                                      <a href="{%s prefix+ar.WebLink() %}">Details</a>
+                                  </td>
+                              </tr>
+                          {% endfor %}
+                       </tbody>
+                      </table>
+                    </div>
                 {% endfor %}
             </div>
-            <br>
         {% endfor %}
 
     {% else %}

app/vmalert/web_test.go

@@ -23,6 +23,7 @@ func TestHandler(t *testing.T) {
 	})
 	g := &rule.Group{
 		Name:        "group",
+		File:        "rules.yaml",
 		Concurrency: 1,
 	}
 	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
@@ -35,7 +36,7 @@ func TestHandler(t *testing.T) {
 	}}
 	rh := &requestHandler{m: m}
 
-	getResp := func(url string, to interface{}, code int) {
+	getResp := func(t *testing.T, url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
@@ -59,43 +60,43 @@ func TestHandler(t *testing.T) {
 	defer ts.Close()
 
 	t.Run("/", func(t *testing.T) {
-		getResp(ts.URL, nil, 200)
-		getResp(ts.URL+"/vmalert", nil, 200)
-		getResp(ts.URL+"/vmalert/alerts", nil, 200)
-		getResp(ts.URL+"/vmalert/groups", nil, 200)
-		getResp(ts.URL+"/vmalert/notifiers", nil, 200)
-		getResp(ts.URL+"/rules", nil, 200)
+		getResp(t, ts.URL, nil, 200)
+		getResp(t, ts.URL+"/vmalert", nil, 200)
+		getResp(t, ts.URL+"/vmalert/alerts", nil, 200)
+		getResp(t, ts.URL+"/vmalert/groups", nil, 200)
+		getResp(t, ts.URL+"/vmalert/notifiers", nil, 200)
+		getResp(t, ts.URL+"/rules", nil, 200)
 	})
 
 	t.Run("/vmalert/rule", func(t *testing.T) {
 		a := ruleToAPI(ar)
-		getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
 		r := ruleToAPI(rr)
-		getResp(ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
+		getResp(t, ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
 	})
 	t.Run("/vmalert/alert", func(t *testing.T) {
 		alerts := ruleToAPIAlert(ar)
 		for _, a := range alerts {
-			getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+			getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
 		}
 	})
 	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)
-		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
+		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
 
 		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramRuleID)
-		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
+		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
 	})
 
 	t.Run("/api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
-		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 
 		lr = listAlertsResponse{}
-		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
 			t.Errorf("expected 1 alert got %d", length)
 		}
@@ -103,13 +104,13 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
 		expAlert := newAlertAPI(ar, ar.GetAlerts()[0])
 		alert := &apiAlert{}
-		getResp(ts.URL+"/"+expAlert.APILink(), alert, 200)
+		getResp(t, ts.URL+"/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
 
 		alert = &apiAlert{}
-		getResp(ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
+		getResp(t, ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
@@ -117,28 +118,28 @@ func TestHandler(t *testing.T) {
 
 	t.Run("/api/v1/alert?badParams", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramAlertID)
-		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
-		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
 		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramAlertID)
-		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
-		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
 		// bad request, alertID is missing
 		params = fmt.Sprintf("?%s=1", paramGroupID)
-		getResp(ts.URL+"/api/v1/alert"+params, nil, 400)
-		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
+		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 400)
+		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
 	})
 
 	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}
 
 		lr = listGroupsResponse{}
-		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}
@@ -146,25 +147,93 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
 		expRule := ruleToAPI(ar)
 		gotRule := apiRule{}
-		getResp(ts.URL+"/"+expRule.APILink(), &gotRule, 200)
+		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRule = apiRule{}
-		getResp(ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
+		getResp(t, ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
-		getResp(ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
+		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
 		if gotRuleWithUpdates.StateUpdates == nil || len(gotRuleWithUpdates.StateUpdates) < 1 {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
+
+	t.Run("/api/v1/rules&filters", func(t *testing.T) {
+		check := func(url string, expGroups, expRules int) {
+			t.Helper()
+			lr := listGroupsResponse{}
+			getResp(t, ts.URL+url, &lr, 200)
+			if length := len(lr.Data.Groups); length != expGroups {
+				t.Errorf("expected %d groups got %d", expGroups, length)
+			}
+			if len(lr.Data.Groups) < 1 {
+				return
+			}
+			var rulesN int
+			for _, gr := range lr.Data.Groups {
+				rulesN += len(gr.Rules)
+			}
+			if rulesN != expRules {
+				t.Errorf("expected %d rules got %d", expRules, rulesN)
+			}
+		}
+
+		check("/api/v1/rules?type=alert", 1, 1)
+		check("/api/v1/rules?type=record", 1, 1)
+
+		check("/vmalert/api/v1/rules?type=alert", 1, 1)
+		check("/vmalert/api/v1/rules?type=record", 1, 1)
+
+		// no filtering expected due to bad params
+		check("/api/v1/rules?type=badParam", 1, 2)
+		check("/api/v1/rules?foo=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
+
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
+	})
+	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
+		// check if response returns active alerts by default
+		lr := listGroupsResponse{}
+		getResp(t, ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml", &lr, 200)
+		activeAlerts := 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts == 0 {
+			t.Fatalf("expected at least 1 active alert in response; got 0")
+		}
+
+		// disable returning alerts via param
+		lr = listGroupsResponse{}
+		getResp(t, ts.URL+"/api/v1/rules?rule_group[]=group&file[]=rules.yaml&exclude_alerts=true", &lr, 200)
+		activeAlerts = 0
+		for _, gr := range lr.Data.Groups {
+			for _, r := range gr.Rules {
+				activeAlerts += len(r.Alerts)
+			}
+		}
+		if activeAlerts != 0 {
+			t.Fatalf("expected to get 0 active alert in response; got %d", activeAlerts)
+		}
+	})
 }
 
 func TestEmptyResponse(t *testing.T) {
@@ -172,7 +241,7 @@ func TestEmptyResponse(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithNoGroups.handler(w, r) }))
 	defer ts.Close()
 
-	getResp := func(url string, to interface{}, code int) {
+	getResp := func(t *testing.T, url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
@@ -195,13 +264,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("no groups /api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
-		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
 			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
 
 		lr = listAlertsResponse{}
-		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
 			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
@@ -209,13 +278,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("no groups /api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
-		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
@@ -226,13 +295,13 @@ func TestEmptyResponse(t *testing.T) {
 
 	t.Run("empty group /api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
-		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
 			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}

app/vmalert/web_types.go

@@ -193,10 +193,15 @@ func ruleToAPI(r interface{}) apiRule {
 	return apiRule{}
 }
 
+const (
+	ruleTypeRecording = "recording"
+	ruleTypeAlerting  = "alerting"
+)
+
 func recordingToAPI(rr *rule.RecordingRule) apiRule {
 	lastState := rule.GetLastEntry(rr)
 	r := apiRule{
-		Type:              "recording",
+		Type:              ruleTypeRecording,
 		DatasourceType:    rr.Type.String(),
 		Name:              rr.Name,
 		Query:             rr.Expr,
@@ -224,7 +229,7 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 func alertingToAPI(ar *rule.AlertingRule) apiRule {
 	lastState := rule.GetLastEntry(ar)
 	r := apiRule{
-		Type:              "alerting",
+		Type:              ruleTypeAlerting,
 		DatasourceType:    ar.Type.String(),
 		Name:              ar.Name,
 		Query:             ar.Expr,

app/vmauth/auth_config.go

@@ -2,26 +2,30 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"flag"
 	"fmt"
+	"math"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"regexp"
-	"strconv"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
@@ -34,38 +38,53 @@ var (
 	defaultRetryStatusCodes = flagutil.NewArrayInt("retryStatusCodes", 0, "Comma-separated list of default HTTP response status codes when vmauth re-tries the request on other backends. "+
 		"See https://docs.victoriametrics.com/vmauth.html#load-balancing for details")
 	defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+
-		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing for more details")
+		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing")
+	discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+
+		"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/vmauth.html#discovering-backend-ips")
+	discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+
+		"Too low value may lead to DNS errors")
+	httpAuthHeader = flagutil.NewArrayString("httpAuthHeader", "HTTP request header to use for obtaining authorization tokens. By default auth tokens are read from Authorization request header")
 )
 
 // AuthConfig represents auth config.
 type AuthConfig struct {
 	Users            []UserInfo `yaml:"users,omitempty"`
 	UnauthorizedUser *UserInfo  `yaml:"unauthorized_user,omitempty"`
+
+	// ms holds all the metrics for the given AuthConfig
+	ms *metrics.Set
 }
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name                   string      `yaml:"name,omitempty"`
-	BearerToken            string      `yaml:"bearer_token,omitempty"`
-	Username               string      `yaml:"username,omitempty"`
-	Password               string      `yaml:"password,omitempty"`
+	Name string `yaml:"name,omitempty"`
+
+	BearerToken string `yaml:"bearer_token,omitempty"`
+	AuthToken   string `yaml:"auth_token,omitempty"`
+	Username    string `yaml:"username,omitempty"`
+	Password    string `yaml:"password,omitempty"`
+
 	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
+	DiscoverBackendIPs     *bool       `yaml:"discover_backend_ips,omitempty"`
 	URLMaps                []URLMap    `yaml:"url_map,omitempty"`
 	HeadersConf            HeadersConf `yaml:",inline"`
 	MaxConcurrentRequests  int         `yaml:"max_concurrent_requests,omitempty"`
 	DefaultURL             *URLPrefix  `yaml:"default_url,omitempty"`
 	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
 	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
-	DropSrcPathPrefixParts int         `yaml:"drop_src_path_prefix_parts,omitempty"`
+	DropSrcPathPrefixParts *int        `yaml:"drop_src_path_prefix_parts,omitempty"`
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
+	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
+	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -101,6 +120,8 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 type Header struct {
 	Name  string
 	Value string
+
+	sOriginal string
 }
 
 // UnmarshalYAML unmarshals h from f.
@@ -109,6 +130,8 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 	if err := f(&s); err != nil {
 		return err
 	}
+	h.sOriginal = s
+
 	n := strings.IndexByte(s, ':')
 	if n < 0 {
 		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
@@ -120,38 +143,108 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 
 // MarshalYAML marshals h to yaml.
 func (h *Header) MarshalYAML() (interface{}, error) {
-	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
-	return s, nil
+	return h.sOriginal, nil
 }
 
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths               []*SrcPath  `yaml:"src_paths,omitempty"`
-	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
-	HeadersConf            HeadersConf `yaml:",inline"`
-	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
-	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
-	DropSrcPathPrefixParts int         `yaml:"drop_src_path_prefix_parts,omitempty"`
+	// SrcPaths is an optional list of regular expressions, which must match the request path.
+	SrcPaths []*Regex `yaml:"src_paths,omitempty"`
+
+	// SrcHosts is an optional list of regular expressions, which must match the request hostname.
+	SrcHosts []*Regex `yaml:"src_hosts,omitempty"`
+
+	// SrcQueryArgs is an optional list of query args, which must match request URL query args.
+	SrcQueryArgs []QueryArg `yaml:"src_query_args,omitempty"`
+
+	// SrcHeaders is an optional list of headers, which must match request headers.
+	SrcHeaders []Header `yaml:"src_headers,omitempty"`
+
+	// UrlPrefix contains backend url prefixes for the proxied request url.
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+
+	// DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS.
+	DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
+
+	// HeadersConf is the config for augumenting request and response headers.
+	HeadersConf HeadersConf `yaml:",inline"`
+
+	// RetryStatusCodes is the list of response status codes used for retrying requests.
+	RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"`
+
+	// LoadBalancingPolicy is load balancing policy among UrlPrefix backends.
+	LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"`
+
+	// DropSrcPathPrefixParts is the number of `/`-delimited request path prefix parts to drop before proxying the request to backend.
+	DropSrcPathPrefixParts *int `yaml:"drop_src_path_prefix_parts,omitempty"`
 }
 
-// SrcPath represents an src path
-type SrcPath struct {
+// Regex represents a regex
+type Regex struct {
+	re *regexp.Regexp
+
 	sOriginal string
-	re        *regexp.Regexp
+}
+
+// QueryArg represents HTTP query arg
+type QueryArg struct {
+	Name  string
+	Value string
+
+	sOriginal string
+}
+
+// UnmarshalYAML unmarshals up from yaml.
+func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
+	var s string
+	if err := f(&s); err != nil {
+		return err
+	}
+	qa.sOriginal = s
+
+	n := strings.IndexByte(s, '=')
+	if n >= 0 {
+		qa.Name = s[:n]
+		qa.Value = s[n+1:]
+	}
+	return nil
+}
+
+// MarshalYAML marshals up to yaml.
+func (qa *QueryArg) MarshalYAML() (interface{}, error) {
+	return qa.sOriginal, nil
 }
 
 // URLPrefix represents passed `url_prefix`
 type URLPrefix struct {
-	n uint32
-
-	// the list of backend urls
-	bus []*backendURL
-
 	// requests are re-tried on other backend urls for these http response status codes
 	retryStatusCodes []int
 
 	// load balancing policy used
 	loadBalancingPolicy string
+
+	// how many request path prefix parts to drop before routing the request to backendURL
+	dropSrcPathPrefixParts int
+
+	// busOriginal contains the original list of backends specified in yaml config.
+	busOriginal []*url.URL
+
+	// n is an atomic counter, which is used for balancing load among available backends.
+	n atomic.Uint32
+
+	// the list of backend urls
+	//
+	// the list can be dynamically updated if `discover_backend_ips` option is set.
+	bus atomic.Pointer[[]*backendURL]
+
+	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
+	discoverBackendIPs bool
+
+	// The next deadline for DNS-based discovery of backend IPs
+	nextDiscoveryDeadline atomic.Uint64
+
+	// vOriginal contains the original yaml value for URLPrefix.
+	vOriginal interface{}
 }
 
 func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
@@ -167,49 +260,146 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 }
 
 type backendURL struct {
-	brokenDeadline     uint64
-	concurrentRequests int32
-	url                *url.URL
+	brokenDeadline     atomic.Uint64
+	concurrentRequests atomic.Int32
+
+	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
 	ct := fasttime.UnixTimestamp()
-	return ct < atomic.LoadUint64(&bu.brokenDeadline)
+	return ct < bu.brokenDeadline.Load()
 }
 
 func (bu *backendURL) setBroken() {
 	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	atomic.StoreUint64(&bu.brokenDeadline, deadline)
+	bu.brokenDeadline.Store(deadline)
 }
 
 func (bu *backendURL) get() {
-	atomic.AddInt32(&bu.concurrentRequests, 1)
+	bu.concurrentRequests.Add(1)
 }
 
 func (bu *backendURL) put() {
-	atomic.AddInt32(&bu.concurrentRequests, -1)
+	bu.concurrentRequests.Add(-1)
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	return len(up.bus)
+	pbus := up.bus.Load()
+	return len(*pbus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func (up *URLPrefix) getBackendURL() *backendURL {
+	up.discoverBackendIPsIfNeeded()
+
+	pbus := up.bus.Load()
+	bus := *pbus
 	if up.loadBalancingPolicy == "first_available" {
-		return up.getFirstAvailableBackendURL()
+		return getFirstAvailableBackendURL(bus)
 	}
-	return up.getLeastLoadedBackendURL()
+	return getLeastLoadedBackendURL(bus, &up.n)
+}
+
+func (up *URLPrefix) discoverBackendIPsIfNeeded() {
+	if !up.discoverBackendIPs {
+		// The discovery is disabled.
+		return
+	}
+
+	ct := fasttime.UnixTimestamp()
+	deadline := up.nextDiscoveryDeadline.Load()
+	if ct < deadline {
+		// There is no need in discovering backends.
+		return
+	}
+
+	intervalSec := math.Ceil(discoverBackendIPsInterval.Seconds())
+	if intervalSec <= 0 {
+		intervalSec = 1
+	}
+	nextDeadline := ct + uint64(intervalSec)
+	if !up.nextDiscoveryDeadline.CompareAndSwap(deadline, nextDeadline) {
+		// Concurrent goroutine already started the discovery.
+		return
+	}
+
+	// Discover ips for all the backendURLs
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(intervalSec))
+	hostToIPs := make(map[string][]string)
+	for _, bu := range up.busOriginal {
+		host := bu.Hostname()
+		if hostToIPs[host] != nil {
+			// ips for the given host have been already discovered
+			continue
+		}
+		addrs, err := resolver.LookupIPAddr(ctx, host)
+		var ips []string
+		if err != nil {
+			logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err)
+			ips = []string{host}
+		} else {
+			ips = make([]string, len(addrs))
+			for i, addr := range addrs {
+				ips[i] = addr.String()
+			}
+			// sort ips, so they could be compared below in areEqualBackendURLs()
+			sort.Strings(ips)
+		}
+		hostToIPs[host] = ips
+	}
+	cancel()
+
+	// generate new backendURLs for the resolved IPs
+	var busNew []*backendURL
+	for _, bu := range up.busOriginal {
+		host := bu.Hostname()
+		port := bu.Port()
+		for _, ip := range hostToIPs[host] {
+			buCopy := *bu
+			buCopy.Host = ip
+			if port != "" {
+				buCopy.Host += ":" + port
+			}
+			busNew = append(busNew, &backendURL{
+				url: &buCopy,
+			})
+		}
+	}
+
+	pbus := up.bus.Load()
+	if areEqualBackendURLs(*pbus, busNew) {
+		return
+	}
+
+	// Store new backend urls
+	up.bus.Store(&busNew)
+}
+
+func areEqualBackendURLs(a, b []*backendURL) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, aURL := range a {
+		bURL := b[i]
+		if aURL.url.String() != bURL.url.String() {
+			return false
+		}
+	}
+	return true
+}
+
+var resolver = &net.Resolver{
+	PreferGo:     true,
+	StrictErrors: true,
 }
 
 // getFirstAvailableBackendURL returns the first available backendURL, which isn't broken.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
-func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL {
-	bus := up.bus
-
+func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	bu := bus[0]
 	if !bu.isBroken() {
 		// Fast path - send the request to the first url.
@@ -231,8 +421,7 @@ func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL {
 // getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
-func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
-	bus := up.bus
+func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
@@ -241,7 +430,7 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 	}
 
 	// Slow path - select other backend urls.
-	n := atomic.AddUint32(&up.n, 1)
+	n := atomicCounter.Add(1)
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -249,20 +438,22 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
 		if bu.isBroken() {
 			continue
 		}
-		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
+		if bu.concurrentRequests.Load() == 0 {
 			// Fast path - return the backend with zero concurrently executed requests.
+			// Do not use CompareAndSwap() instead of Load(), since it is much slower on systems with many CPU cores.
+			bu.concurrentRequests.Add(1)
 			return bu
 		}
 	}
 
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMin := bus[n%uint32(len(bus))]
-	minRequests := atomic.LoadInt32(&buMin.concurrentRequests)
+	minRequests := buMin.concurrentRequests.Load()
 	for _, bu := range bus {
 		if bu.isBroken() {
 			continue
 		}
-		if n := atomic.LoadInt32(&bu.concurrentRequests); n < minRequests {
+		if n := bu.concurrentRequests.Load(); n < minRequests {
 			buMin = bu
 			minRequests = n
 		}
@@ -277,6 +468,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 	if err := f(&v); err != nil {
 		return err
 	}
+	up.vOriginal = v
 
 	var urls []string
 	switch x := v.(type) {
@@ -299,42 +491,25 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
 	}
 
-	bus := make([]*backendURL, len(urls))
+	bus := make([]*url.URL, len(urls))
 	for i, u := range urls {
 		pu, err := url.Parse(u)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
 		}
-		bus[i] = &backendURL{
-			url: pu,
-		}
+		bus[i] = pu
 	}
-	up.bus = bus
+	up.busOriginal = bus
 	return nil
 }
 
 // MarshalYAML marshals up to yaml.
 func (up *URLPrefix) MarshalYAML() (interface{}, error) {
-	var b []byte
-	if len(up.bus) == 1 {
-		u := up.bus[0].url.String()
-		b = strconv.AppendQuote(b, u)
-		return string(b), nil
-	}
-	b = append(b, '[')
-	for i, bu := range up.bus {
-		u := bu.url.String()
-		b = strconv.AppendQuote(b, u)
-		if i+1 < len(up.bus) {
-			b = append(b, ',')
-		}
-	}
-	b = append(b, ']')
-	return string(b), nil
+	return up.vOriginal, nil
 }
 
-func (sp *SrcPath) match(s string) bool {
-	prefix, ok := sp.re.LiteralPrefix()
+func (r *Regex) match(s string) bool {
+	prefix, ok := r.re.LiteralPrefix()
 	if ok {
 		// Fast path - literal match
 		return s == prefix
@@ -342,34 +517,35 @@ func (sp *SrcPath) match(s string) bool {
 	if !strings.HasPrefix(s, prefix) {
 		return false
 	}
-	return sp.re.MatchString(s)
+	return r.re.MatchString(s)
 }
 
 // UnmarshalYAML implements yaml.Unmarshaler
-func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error {
+func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
 	}
+	r.sOriginal = s
+
 	sAnchored := "^(?:" + s + ")$"
 	re, err := regexp.Compile(sAnchored)
 	if err != nil {
 		return fmt.Errorf("cannot build regexp from %q: %w", s, err)
 	}
-	sp.sOriginal = s
-	sp.re = re
+	r.re = re
 	return nil
 }
 
 // MarshalYAML implements yaml.Marshaler.
-func (sp *SrcPath) MarshalYAML() (interface{}, error) {
-	return sp.sOriginal, nil
+func (r *Regex) MarshalYAML() (interface{}, error) {
+	return r.sOriginal, nil
 }
 
 var (
 	configReloads      = metrics.NewCounter(`vmauth_config_last_reload_total`)
 	configReloadErrors = metrics.NewCounter(`vmauth_config_last_reload_errors_total`)
-	configSuccess      = metrics.NewCounter(`vmauth_config_last_reload_successful`)
+	configSuccess      = metrics.NewGauge(`vmauth_config_last_reload_successful`, nil)
 	configTimestamp    = metrics.NewCounter(`vmauth_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -445,17 +621,19 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var authConfig atomic.Pointer[AuthConfig]
-var authUsers atomic.Pointer[map[string]*UserInfo]
-var authConfigWG sync.WaitGroup
-var stopCh chan struct{}
+var (
+	authConfig   atomic.Pointer[AuthConfig]
+	authUsers    atomic.Pointer[map[string]*UserInfo]
+	authConfigWG sync.WaitGroup
+	stopCh       chan struct{}
+)
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
 func loadAuthConfig() (bool, error) {
-	data, err := fs.ReadFileOrHTTP(*authConfigPath)
+	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
@@ -477,6 +655,11 @@ func loadAuthConfig() (bool, error) {
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
+	prevAc := authConfig.Load()
+	if prevAc != nil {
+		metrics.UnregisterSet(prevAc.ms)
+	}
+	metrics.RegisterSet(ac.ms)
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
@@ -489,10 +672,13 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-	var ac AuthConfig
-	if err = yaml.UnmarshalStrict(data, &ac); err != nil {
+	ac := &AuthConfig{
+		ms: metrics.NewSet(),
+	}
+	if err = yaml.UnmarshalStrict(data, ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
+
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		if ui.Username != "" {
@@ -504,29 +690,39 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if ui.BearerToken != "" {
 			return nil, fmt.Errorf("field bearer_token can't be specified for unauthorized_user section")
 		}
+		if ui.AuthToken != "" {
+			return nil, fmt.Errorf("field auth_token can't be specified for unauthorized_user section")
+		}
 		if ui.Name != "" {
 			return nil, fmt.Errorf("field name can't be specified for unauthorized_user section")
 		}
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
-		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
+		}
+		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
+		ui.concurrencyLimitReached = ac.ms.NewCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
+		_ = ac.ms.NewGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
 		}
 		ui.httpTransport = tr
 	}
-	return &ac, nil
+	return ac, nil
 }
 
 func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
@@ -537,43 +733,34 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	byAuthToken := make(map[string]*UserInfo, len(uis))
 	for i := range uis {
 		ui := &uis[i]
-		if ui.BearerToken == "" && ui.Username == "" {
-			return nil, fmt.Errorf("either bearer_token or username must be set")
+		ats, err := getAuthTokens(ui.AuthToken, ui.BearerToken, ui.Username, ui.Password)
+		if err != nil {
+			return nil, err
 		}
-		if ui.BearerToken != "" && ui.Username != "" {
-			return nil, fmt.Errorf("bearer_token=%q and username=%q cannot be set simultaneously", ui.BearerToken, ui.Username)
+		for _, at := range ats {
+			if uiOld := byAuthToken[at]; uiOld != nil {
+				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
+					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
+			}
 		}
-		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[at1] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
-		}
-		if byAuthToken[at2] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
-		}
-
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
-		name := ui.name()
-		if ui.BearerToken != "" {
-			if ui.Password != "" {
-				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			}
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
-		}
-		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
+		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -583,60 +770,104 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		}
 		ui.httpTransport = tr
 
-		byAuthToken[at1] = ui
-		byAuthToken[at2] = ui
+		for _, at := range ats {
+			byAuthToken[at] = ui
+		}
 	}
 	return byAuthToken, nil
 }
 
+var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
+
+func (ui *UserInfo) getMetricLabels() (string, error) {
+	name := ui.name()
+	if len(name) == 0 && len(ui.MetricLabels) == 0 {
+		// fast path
+		return "", nil
+	}
+	labels := make([]string, 0, len(ui.MetricLabels)+1)
+	if len(name) > 0 {
+		labels = append(labels, fmt.Sprintf(`username=%q`, name))
+	}
+	for k, v := range ui.MetricLabels {
+		if !labelNameRegexp.MatchString(k) {
+			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
+		}
+		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
+	}
+	sort.Strings(labels)
+	labelsStr := "{" + strings.Join(labels, ",") + "}"
+	return labelsStr, nil
+}
+
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
+	dropSrcPathPrefixParts := 0
+	discoverBackendIPs := *discoverBackendIPsGlobal
 	if ui.URLPrefix != nil {
-		if err := ui.URLPrefix.sanitize(); err != nil {
+		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
 			return err
 		}
-		if len(ui.RetryStatusCodes) > 0 {
+		if ui.RetryStatusCodes != nil {
 			retryStatusCodes = ui.RetryStatusCodes
 		}
 		if ui.LoadBalancingPolicy != "" {
 			loadBalancingPolicy = ui.LoadBalancingPolicy
 		}
+		if ui.DropSrcPathPrefixParts != nil {
+			dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
+		}
+		if ui.DiscoverBackendIPs != nil {
+			discoverBackendIPs = *ui.DiscoverBackendIPs
+		}
 		ui.URLPrefix.retryStatusCodes = retryStatusCodes
+		ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
+		ui.URLPrefix.discoverBackendIPs = discoverBackendIPs
 		if err := ui.URLPrefix.setLoadBalancingPolicy(loadBalancingPolicy); err != nil {
 			return err
 		}
 	}
 	if ui.DefaultURL != nil {
-		if err := ui.DefaultURL.sanitize(); err != nil {
+		if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil {
 			return err
 		}
 	}
 	for _, e := range ui.URLMaps {
-		if len(e.SrcPaths) == 0 {
-			return fmt.Errorf("missing `src_paths` in `url_map`")
+		if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 && len(e.SrcQueryArgs) == 0 && len(e.SrcHeaders) == 0 {
+			return fmt.Errorf("missing `src_paths`, `src_hosts`, `src_query_args` and `src_headers` in `url_map`")
 		}
 		if e.URLPrefix == nil {
 			return fmt.Errorf("missing `url_prefix` in `url_map`")
 		}
-		if err := e.URLPrefix.sanitize(); err != nil {
+		if err := e.URLPrefix.sanitizeAndInitialize(); err != nil {
 			return err
 		}
 		rscs := retryStatusCodes
 		lbp := loadBalancingPolicy
-		if len(e.RetryStatusCodes) > 0 {
+		dsp := dropSrcPathPrefixParts
+		dbd := discoverBackendIPs
+		if e.RetryStatusCodes != nil {
 			rscs = e.RetryStatusCodes
 		}
 		if e.LoadBalancingPolicy != "" {
 			lbp = e.LoadBalancingPolicy
 		}
+		if e.DropSrcPathPrefixParts != nil {
+			dsp = *e.DropSrcPathPrefixParts
+		}
+		if e.DiscoverBackendIPs != nil {
+			dbd = *e.DiscoverBackendIPs
+		}
 		e.URLPrefix.retryStatusCodes = rscs
 		if err := e.URLPrefix.setLoadBalancingPolicy(lbp); err != nil {
 			return err
 		}
+		e.URLPrefix.dropSrcPathPrefixParts = dsp
+		e.URLPrefix.discoverBackendIPs = dbd
 	}
 	if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
-		return fmt.Errorf("missing `url_prefix`")
+		return fmt.Errorf("missing `url_prefix` or `url_map`")
 	}
 	return nil
 }
@@ -649,39 +880,100 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		return "bearer_token"
+		h := xxhash.Sum64([]byte(ui.BearerToken))
+		return fmt.Sprintf("bearer_token:hash:%016X", h)
+	}
+	if ui.AuthToken != "" {
+		h := xxhash.Sum64([]byte(ui.AuthToken))
+		return fmt.Sprintf("auth_token:hash:%016X", h)
 	}
 	return ""
 }
 
-func getAuthTokens(bearerToken, username, password string) (string, string) {
-	if bearerToken != "" {
-		// Accept the bearerToken as Basic Auth username with empty password
-		at1 := getAuthToken(bearerToken, "", "")
-		at2 := getAuthToken("", bearerToken, "")
-		return at1, at2
+func getAuthTokens(authToken, bearerToken, username, password string) ([]string, error) {
+	if authToken != "" {
+		if bearerToken != "" {
+			return nil, fmt.Errorf("bearer_token cannot be specified if auth_token is set")
+		}
+		if username != "" || password != "" {
+			return nil, fmt.Errorf("username and password cannot be specified if auth_token is set")
+		}
+		at := getHTTPAuthToken(authToken)
+		return []string{at}, nil
 	}
-	at := getAuthToken("", username, password)
-	return at, at
+	if bearerToken != "" {
+		if username != "" || password != "" {
+			return nil, fmt.Errorf("username and password cannot be specified if bearer_token is set")
+		}
+		// Accept the bearerToken as Basic Auth username with empty password
+		at1 := getHTTPAuthBearerToken(bearerToken)
+		at2 := getHTTPAuthBasicToken(bearerToken, "")
+		return []string{at1, at2}, nil
+	}
+	if username != "" {
+		at := getHTTPAuthBasicToken(username, password)
+		return []string{at}, nil
+	}
+	return nil, fmt.Errorf("missing authorization options; bearer_token or username must be set")
 }
 
-func getAuthToken(bearerToken, username, password string) string {
-	if bearerToken != "" {
-		return "Bearer " + bearerToken
-	}
+func getHTTPAuthToken(authToken string) string {
+	return "http_auth:" + authToken
+}
+
+func getHTTPAuthBearerToken(bearerToken string) string {
+	return "http_auth:Bearer " + bearerToken
+}
+
+func getHTTPAuthBasicToken(username, password string) string {
 	token := username + ":" + password
 	token64 := base64.StdEncoding.EncodeToString([]byte(token))
-	return "Basic " + token64
+	return "http_auth:Basic " + token64
 }
 
-func (up *URLPrefix) sanitize() error {
-	for _, bu := range up.bus {
-		puNew, err := sanitizeURLPrefix(bu.url)
+var defaultHeaderNames = []string{"Authorization"}
+
+func getAuthTokensFromRequest(r *http.Request) []string {
+	var ats []string
+
+	// Obtain possible auth tokens from one of the allowed auth headers
+	headerNames := *httpAuthHeader
+	if len(headerNames) == 0 {
+		headerNames = defaultHeaderNames
+	}
+	for _, headerName := range headerNames {
+		if ah := r.Header.Get(headerName); ah != "" {
+			if strings.HasPrefix(ah, "Token ") {
+				// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+				// See https://docs.influxdata.com/influxdb/v2.0/api/
+				ah = strings.Replace(ah, "Token", "Bearer", 1)
+			}
+			at := "http_auth:" + ah
+			ats = append(ats, at)
+		}
+	}
+
+	return ats
+}
+
+func (up *URLPrefix) sanitizeAndInitialize() error {
+	for i, bu := range up.busOriginal {
+		puNew, err := sanitizeURLPrefix(bu)
 		if err != nil {
 			return err
 		}
-		bu.url = puNew
+		up.busOriginal[i] = puNew
 	}
+
+	// Initialize up.bus
+	bus := make([]*backendURL, len(up.busOriginal))
+	for i, bu := range up.busOriginal {
+		bus[i] = &backendURL{
+			url: bu,
+		}
+	}
+	up.bus.Store(&bus)
+
 	return nil
 }
 

app/vmauth/auth_config_test.go

@@ -17,9 +17,9 @@ func TestParseAuthConfigFailure(t *testing.T) {
 		if err != nil {
 			return
 		}
-		_, err = parseAuthConfigUsers(ac)
+		users, err := parseAuthConfigUsers(ac)
 		if err == nil {
-			t.Fatalf("expecting non-nil error")
+			t.Fatalf("expecting non-nil error; got %v", users)
 		}
 	}
 
@@ -88,6 +88,22 @@ users:
   url_prefix: []
 `)
 
+	// auth_token and username in a single config
+	f(`
+users:
+- auth_token: foo
+  username: bbb
+  url_prefix: http://foo.bar
+`)
+
+	// auth_token and bearer_token in a single config
+	f(`
+users:
+- auth_token: foo
+  bearer_token: bbb
+  url_prefix: http://foo.bar
+`)
+
 	// Username and bearer_token in a single config
 	f(`
 users:
@@ -145,6 +161,12 @@ users:
   url_map:
   - src_paths: ["/foo/bar"]
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ["foobar"]
+`)
 
 	// Invalid url_prefix in url_map
 	f(`
@@ -154,6 +176,13 @@ users:
   - src_paths: ["/foo/bar"]
     url_prefix: foo.bar
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ["foobar"]
+    url_prefix: foo.bar
+`)
 
 	// empty url_prefix in url_map
 	f(`
@@ -163,8 +192,15 @@ users:
   - src_paths: ['/foo/bar']
     url_prefix: []
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_phosts: ['foobar']
+    url_prefix: []
+`)
 
-	// Missing src_paths in url_map
+	// Missing src_paths and src_hosts in url_map
 	f(`
 users:
 - username: a
@@ -172,7 +208,7 @@ users:
   - url_prefix: http://foobar
 `)
 
-	// Invalid regexp in src_path.
+	// Invalid regexp in src_paths
 	f(`
 users:
 - username: a
@@ -181,6 +217,33 @@ users:
     url_prefix: http://foobar
 `)
 
+	// Invalid regexp in src_hosts
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ['fo[obar']
+    url_prefix: http://foobar
+`)
+
+	// Invalid src_query_args
+	f(`
+users:
+- username: a
+  url_map:
+  - src_query_args: abc
+    url_prefix: http://foobar
+`)
+
+	// Invalid src_headers
+	f(`
+users:
+- username: a
+  url_map:
+  - src_headers: abc
+    url_prefix: http://foobar
+`)
+
 	// Invalid headers in url_map (missing ':')
 	f(`
 users:
@@ -200,6 +263,14 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
+`)
+	// Invalid metric label name
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  metric_labels:
+    not-prometheus-compatible: value
 `)
 }
 
@@ -220,8 +291,9 @@ func TestParseAuthConfigSuccess(t *testing.T) {
 		}
 	}
 
-	// Single user
 	insecureSkipVerifyTrue := true
+
+	// Single user
 	f(`
 users:
 - username: foo
@@ -230,7 +302,7 @@ users:
   max_concurrent_requests: 5
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
@@ -239,6 +311,22 @@ users:
 		},
 	})
 
+	// Single user with auth_token
+	f(`
+users:
+- auth_token: foo
+  url_prefix: http://aaa:343/bbb
+  max_concurrent_requests: 5
+  tls_insecure_skip_verify: true
+`, map[string]*UserInfo{
+		getHTTPAuthToken("foo"): {
+			AuthToken:             "foo",
+			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
+			MaxConcurrentRequests: 5,
+			TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
+		},
+	})
+
 	// Multiple url_prefix entries
 	insecureSkipVerifyFalse := false
 	f(`
@@ -253,7 +341,7 @@ users:
   load_balancing_policy: first_available
   drop_src_path_prefix_parts: 1
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", "bar"): {
+		getHTTPAuthBasicToken("foo", "bar"): {
 			Username: "foo",
 			Password: "bar",
 			URLPrefix: mustParseURLs([]string{
@@ -263,7 +351,7 @@ users:
 			TLSInsecureSkipVerify:  &insecureSkipVerifyFalse,
 			RetryStatusCodes:       []int{500, 501},
 			LoadBalancingPolicy:    "first_available",
-			DropSrcPathPrefixParts: 1,
+			DropSrcPathPrefixParts: intp(1),
 		},
 	})
 
@@ -273,19 +361,60 @@ users:
 - username: foo
   url_prefix: http://foo
 - username: bar
-  url_prefix: https://bar/x///
+  url_prefix: https://bar/x/
 `, map[string]*UserInfo{
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			Username:  "foo",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "bar", ""): {
+		getHTTPAuthBasicToken("bar", ""): {
 			Username:  "bar",
-			URLPrefix: mustParseURL("https://bar/x"),
+			URLPrefix: mustParseURL("https://bar/x/"),
 		},
 	})
 
 	// non-empty URLMap
+	sharedUserInfo := &UserInfo{
+		BearerToken: "foo",
+		URLMaps: []URLMap{
+			{
+				SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+				URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+			},
+			{
+				SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
+				SrcPaths: getRegexs([]string{"/api/v1/write"}),
+				SrcQueryArgs: []QueryArg{
+					{
+						Name:  "foo",
+						Value: "bar",
+					},
+				},
+				SrcHeaders: []Header{
+					{
+						Name:  "TenantID",
+						Value: "345",
+					},
+				},
+				URLPrefix: mustParseURLs([]string{
+					"http://vminsert1/insert/0/prometheus",
+					"http://vminsert2/insert/0/prometheus",
+				}),
+				HeadersConf: HeadersConf{
+					RequestHeaders: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+		},
+	}
 	f(`
 users:
 - bearer_token: foo
@@ -293,69 +422,19 @@ users:
   - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"]
     url_prefix: http://vmselect/select/0/prometheus
   - src_paths: ["/api/v1/write"]
+    src_hosts: ["foo\\.bar", "baz:1234"]
+    src_query_args: ['foo=bar']
+    src_headers: ['TenantID: 345']
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
     headers:
     - "foo: bar"
     - "xxx: y"
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
-			BearerToken: "foo",
-			URLMaps: []URLMap{
-				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
-					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
-				},
-				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
-					URLPrefix: mustParseURLs([]string{
-						"http://vminsert1/insert/0/prometheus",
-						"http://vminsert2/insert/0/prometheus",
-					}),
-					HeadersConf: HeadersConf{
-						RequestHeaders: []Header{
-							{
-								Name:  "foo",
-								Value: "bar",
-							},
-							{
-								Name:  "xxx",
-								Value: "y",
-							},
-						},
-					},
-				},
-			},
-		},
-		getAuthToken("", "foo", ""): {
-			BearerToken: "foo",
-			URLMaps: []URLMap{
-				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
-					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
-				},
-				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
-					URLPrefix: mustParseURLs([]string{
-						"http://vminsert1/insert/0/prometheus",
-						"http://vminsert2/insert/0/prometheus",
-					}),
-					HeadersConf: HeadersConf{
-						RequestHeaders: []Header{
-							{
-								Name:  "foo",
-								Value: "bar",
-							},
-							{
-								Name:  "xxx",
-								Value: "y",
-							},
-						},
-					},
-				},
-			},
-		},
+		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
+		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
 	})
-	// Multiple users with the same name
+
+	// Multiple users with the same name - this should work, since these users have different passwords
 	f(`
 users:
 - username: foo-same
@@ -363,19 +442,20 @@ users:
   url_prefix: http://foo
 - username: foo-same
   password: bar
-  url_prefix: https://bar/x///
+  url_prefix: https://bar/x
 `, map[string]*UserInfo{
-		getAuthToken("", "foo-same", "baz"): {
+		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
 			Password:  "baz",
 			URLPrefix: mustParseURL("http://foo"),
 		},
-		getAuthToken("", "foo-same", "bar"): {
+		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
+
 	// with default url
 	f(`
 users:
@@ -392,15 +472,15 @@ users:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
 `, map[string]*UserInfo{
-		getAuthToken("foo", "", ""): {
+		getHTTPAuthBearerToken("foo"): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@@ -424,15 +504,15 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-		getAuthToken("", "foo", ""): {
+		getHTTPAuthBasicToken("foo", ""): {
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@@ -458,6 +538,41 @@ users:
 		},
 	})
 
+	// With metric_labels
+	f(`
+users:
+- username: foo-same
+  password: baz
+  url_prefix: http://foo
+  metric_labels:
+    dc: eu
+    team: dev
+- username: foo-same
+  password: bar
+  url_prefix: https://bar/x
+  metric_labels:
+    backend_env: test
+    team: accounting
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo-same", "baz"): {
+			Username:  "foo-same",
+			Password:  "baz",
+			URLPrefix: mustParseURL("http://foo"),
+			MetricLabels: map[string]string{
+				"dc":   "eu",
+				"team": "dev",
+			},
+		},
+		getHTTPAuthBasicToken("foo-same", "bar"): {
+			Username:  "foo-same",
+			Password:  "bar",
+			URLPrefix: mustParseURL("https://bar/x"),
+			MetricLabels: map[string]string{
+				"backend_env": "test",
+				"team":        "accounting",
+			},
+		},
+	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -484,7 +599,7 @@ unauthorized_user:
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	ui := m[getAuthToken("", "foo", "bar")]
+	ui := m[getHTTPAuthBasicToken("foo", "bar")]
 	if !isSetBool(ui.TLSInsecureSkipVerify, true) || !ui.httpTransport.TLSClientConfig.InsecureSkipVerify {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
@@ -494,6 +609,86 @@ unauthorized_user:
 	}
 }
 
+func TestUserInfoGetMetricLabels(t *testing.T) {
+	t.Run("empty-labels", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-username", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-name", func(t *testing.T) {
+		ui := &UserInfo{
+			Name:        "user1",
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-bearer-token", func(t *testing.T) {
+		ui := &UserInfo{
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("invalid-label", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "foo",
+			MetricLabels: map[string]string{
+				",{": "aaaa",
+			},
+		}
+		_, err := ui.getMetricLabels()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	})
+}
+
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false
@@ -501,10 +696,10 @@ func isSetBool(boolP *bool, expectedValue bool) bool {
 	return *boolP == expectedValue
 }
 
-func getSrcPaths(paths []string) []*SrcPath {
-	var sps []*SrcPath
+func getRegexs(paths []string) []*Regex {
+	var sps []*Regex
 	for _, path := range paths {
-		sps = append(sps, &SrcPath{
+		sps = append(sps, &Regex{
 			sOriginal: path,
 			re:        regexp.MustCompile("^(?:" + path + ")$"),
 		})
@@ -539,6 +734,7 @@ func mustParseURL(u string) *URLPrefix {
 
 func mustParseURLs(us []string) *URLPrefix {
 	bus := make([]*backendURL, len(us))
+	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
@@ -547,8 +743,19 @@ func mustParseURLs(us []string) *URLPrefix {
 		bus[i] = &backendURL{
 			url: pu,
 		}
+		urls[i] = pu
 	}
-	return &URLPrefix{
-		bus: bus,
+	up := &URLPrefix{}
+	if len(us) == 1 {
+		up.vOriginal = us[0]
+	} else {
+		up.vOriginal = us
 	}
+	up.bus.Store(&bus)
+	up.busOriginal = urls
+	return up
+}
+
+func intp(n int) *int {
+	return &n
 }

app/vmauth/example_config.yml

@@ -10,6 +10,11 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Adds labels to the exported metrics for given user section
+  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
+  metric_labels:
+    backend_dc: eu
+    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.
@@ -92,7 +97,7 @@ users:
   #  - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
   #  - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
   #
-  # Regular expressions are allowed in `src_paths` entries.
+  # Regular expressions are allowed in `src_paths` and `src_hosts` entries.
 - username: "foobar"
   url_map:
   - src_paths:

app/vmauth/main.go

@@ -13,6 +13,7 @@ import (
 	"net/textproto"
 	"net/url"
 	"os"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -24,7 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -33,8 +34,8 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
@@ -45,7 +46,7 @@ var (
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
@@ -64,20 +65,25 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
-	logger.Infof("starting vmauth at %q...", *httpListenAddr)
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":8427"}
+	}
+	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	if err := httpserver.Stop(listenAddrs); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
@@ -88,7 +94,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, *reloadAuthKey, "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -96,8 +102,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
-	authToken := r.Header.Get("Authorization")
-	if authToken == "" {
+
+	ats := getAuthTokensFromRequest(r)
+	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
@@ -109,18 +116,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
-	if strings.HasPrefix(authToken, "Token ") {
-		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
-		// See https://docs.influxdata.com/influxdb/v2.0/api/
-		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
-	}
 
-	ac := *authUsers.Load()
-	ui := ac[authToken]
+	ui := getUserInfoByAuthTokens(ats)
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
+			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
@@ -136,6 +137,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+func getUserInfoByAuthTokens(ats []string) *UserInfo {
+	ac := *authUsers.Load()
+	for _, at := range ats {
+		ui := ac[at]
+		if ui != nil {
+			return ui
+		}
+	}
+	return nil
+}
+
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
@@ -164,7 +176,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
-	up, hc, dropSrcPathPrefixParts := ui.getURLPrefixAndHeaders(u)
+	up, hc := ui.getURLPrefixAndHeaders(u, r.Header)
 	isDefault := false
 	if up == nil {
 		if ui.DefaultURL == nil {
@@ -198,9 +210,9 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
 		} else { // Update path for regular routes.
-			targetURL = mergeURLs(targetURL, u, dropSrcPathPrefixParts)
+			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -212,15 +224,23 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
+	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
-	req.Host = targetURL.Host
+
+	if req.URL.Scheme == "https" {
+		// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
+		// so it expects the targetURL.Host in the request.
+		// There is no need in overriding the req.Host for http requests, since it is expected that backend server
+		// may properly process queries with the original req.Host.
+		req.Host = targetURL.Host
+	}
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := transport.RoundTrip(req)
+	res, err := ui.httpTransport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -228,15 +248,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				ui.backendErrors.Inc()
+			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
+				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -246,7 +271,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
 		return false
 	}
-	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
+	if slices.Contains(retryStatusCodes, res.StatusCode) {
+		_ = res.Body.Close()
+		if !rtbOK || !rtb.canRetry() {
+			// If we get an error from the retry_status_codes list, but cannot execute retry,
+			// we consider such a request an error as well.
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+					res.StatusCode, targetURL),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
+			return true
+		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
@@ -265,6 +303,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
 	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
 	copyBufPool.Put(copyBuf)
+	_ = res.Body.Close()
 	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
@@ -274,15 +313,6 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	return true
 }
 
-func hasInt(a []int, n int) bool {
-	for _, x := range a {
-		if x == n {
-			return true
-		}
-	}
-	return false
-}
-
 var copyBufPool bytesutil.ByteBufferPool
 
 func copyHeader(dst, src http.Header) {
@@ -392,8 +422,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var transportMap = make(map[string]*http.Transport)
-var transportMapLock sync.Mutex
+var (
+	transportMap     = make(map[string]*http.Transport)
+	transportMapLock sync.Mutex
+)
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)
@@ -421,7 +453,7 @@ func newTransport(insecureSkipVerify bool, caFile string) (*http.Transport, erro
 		tlsCfg.ClientSessionCache = tls.NewLRUClientSessionCache(0)
 		tlsCfg.InsecureSkipVerify = insecureSkipVerify
 		if caFile != "" {
-			data, err := fs.ReadFileOrHTTP(caFile)
+			data, err := fscore.ReadFileOrHTTP(caFile)
 			if err != nil {
 				return nil, fmt.Errorf("cannot read tls_ca_file: %w", err)
 			}

app/vmauth/target_url.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 )
 
@@ -49,18 +51,63 @@ func dropPrefixParts(path string, parts int) string {
 	return path
 }
 
-func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, int) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL, h http.Header) (*URLPrefix, HeadersConf) {
 	for _, e := range ui.URLMaps {
-		for _, sp := range e.SrcPaths {
-			if sp.match(u.Path) {
-				return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts
-			}
+		if !matchAnyRegex(e.SrcHosts, u.Host) {
+			continue
 		}
+		if !matchAnyRegex(e.SrcPaths, u.Path) {
+			continue
+		}
+		if !matchAnyQueryArg(e.SrcQueryArgs, u.Query()) {
+			continue
+		}
+		if !matchAnyHeader(e.SrcHeaders, h) {
+			continue
+		}
+
+		return e.URLPrefix, e.HeadersConf
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix, ui.HeadersConf, ui.DropSrcPathPrefixParts
+		return ui.URLPrefix, ui.HeadersConf
 	}
-	return nil, HeadersConf{}, 0
+	return nil, HeadersConf{}
+}
+
+func matchAnyRegex(rs []*Regex, s string) bool {
+	if len(rs) == 0 {
+		return true
+	}
+	for _, r := range rs {
+		if r.match(s) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchAnyQueryArg(qas []QueryArg, args url.Values) bool {
+	if len(qas) == 0 {
+		return true
+	}
+	for _, qa := range qas {
+		if slices.Contains(args[qa.Name], qa.Value) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchAnyHeader(headers []Header, h http.Header) bool {
+	if len(headers) == 0 {
+		return true
+	}
+	for _, header := range headers {
+		if slices.Contains(h.Values(header.Name), header.Value) {
+			return true
+		}
+	}
+	return false
 }
 
 func normalizeURL(uOrig *url.URL) *url.URL {

app/vmauth/target_url_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -89,19 +90,21 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, hc, dropSrcPathPrefixParts := ui.getURLPrefixAndHeaders(u)
+		up, hc := ui.getURLPrefixAndHeaders(u, nil)
 		if up == nil {
 			t.Fatalf("cannot determie backend: %s", err)
 		}
-		bu := up.getLeastLoadedBackendURL()
-		target := mergeURLs(bu.url, u, dropSrcPathPrefixParts)
+		bu := up.getBackendURL()
+		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
 		bu.put()
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
-		headersStr := fmt.Sprintf("%q", hc.RequestHeaders)
-		if headersStr != expectedRequestHeaders {
-			t.Fatalf("unexpected request headers; got %s; want %s", headersStr, expectedRequestHeaders)
+		if s := headersToString(hc.RequestHeaders); s != expectedRequestHeaders {
+			t.Fatalf("unexpected request headers; got %q; want %q", s, expectedRequestHeaders)
+		}
+		if s := headersToString(hc.ResponseHeaders); s != expectedResponseHeaders {
+			t.Fatalf("unexpected response headers; got %q; want %q", s, expectedResponseHeaders)
 		}
 		if !reflect.DeepEqual(up.retryStatusCodes, expectedRetryStatusCodes) {
 			t.Fatalf("unexpected retryStatusCodes; got %d; want %d", up.retryStatusCodes, expectedRetryStatusCodes)
@@ -109,47 +112,62 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		if up.loadBalancingPolicy != expectedLoadBalancingPolicy {
 			t.Fatalf("unexpected loadBalancingPolicy; got %q; want %q", up.loadBalancingPolicy, expectedLoadBalancingPolicy)
 		}
-		if dropSrcPathPrefixParts != expectedDropSrcPathPrefixParts {
-			t.Fatalf("unexpected dropSrcPathPrefixParts; got %d; want %d", dropSrcPathPrefixParts, expectedDropSrcPathPrefixParts)
+		if up.dropSrcPathPrefixParts != expectedDropSrcPathPrefixParts {
+			t.Fatalf("unexpected dropSrcPathPrefixParts; got %d; want %d", up.dropSrcPathPrefixParts, expectedDropSrcPathPrefixParts)
 		}
 	}
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "", "http://foo.bar/.", "[]", "[]", nil, "least_loaded", 0)
+	}, "", "http://foo.bar/.", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
 		HeadersConf: HeadersConf{
-			RequestHeaders: []Header{{
-				Name:  "bb",
-				Value: "aaa",
-			}},
+			RequestHeaders: []Header{
+				{
+					Name:  "bb",
+					Value: "aaa",
+				},
+			},
+			ResponseHeaders: []Header{
+				{
+					Name:  "x",
+					Value: "y",
+				},
+			},
 		},
 		RetryStatusCodes:       []int{503, 501},
 		LoadBalancingPolicy:    "first_available",
-		DropSrcPathPrefixParts: 2,
-	}, "/a/b/c", "http://foo.bar/c", `[{"bb" "aaa"}]`, `[]`, []int{503, 501}, "first_available", 2)
+		DropSrcPathPrefixParts: intp(2),
+	}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar/federate"),
-	}, "/", "http://foo.bar/federate", "[]", "[]", nil, "least_loaded", 0)
+	}, "/", "http://foo.bar/federate", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]", "[]", nil, "least_loaded", 0)
+	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/z", "https://sss:3894/x/y/z", "[]", "[]", nil, "least_loaded", 0)
+	}, "/z", "https://sss:3894/x/y/z", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]", "[]", nil, "least_loaded", 0)
+	}, "/../../aaa", "https://sss:3894/x/y/aaa", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]", "[]", nil, "least_loaded", 0)
+	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "", "", nil, "least_loaded", 0)
 
 	// Complex routing with `url_map`
 	ui := &UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/vmsingle/api/v1/query"}),
+				SrcHosts: getRegexs([]string{"host42"}),
+				SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}),
+				SrcQueryArgs: []QueryArg{
+					{
+						Name:  "db",
+						Value: "foo",
+					},
+				},
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
 				HeadersConf: HeadersConf{
 					RequestHeaders: []Header{
@@ -171,11 +189,13 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				},
 				RetryStatusCodes:       []int{503, 500, 501},
 				LoadBalancingPolicy:    "first_available",
-				DropSrcPathPrefixParts: 1,
+				DropSrcPathPrefixParts: intp(1),
 			},
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
-				URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
+				SrcPaths:               getRegexs([]string{"/api/v1/write"}),
+				URLPrefix:              mustParseURL("http://vminsert/0/prometheus"),
+				RetryStatusCodes:       []int{},
+				DropSrcPathPrefixParts: intp(0),
 			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
@@ -190,37 +210,46 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			}},
 		},
 		RetryStatusCodes:       []int{502},
-		DropSrcPathPrefixParts: 2,
+		DropSrcPathPrefixParts: intp(2),
 	}
-	f(ui, "/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0)
-	f(ui, "/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
+	f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up",
+		"xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1)
+	f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up",
+		"bb: aaa", "x: y", []int{502}, "least_loaded", 2)
+	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
+	f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", "bb: aaa", "x: y", []int{502}, "least_loaded", 2)
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
 			},
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/write"}),
 				URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
 			},
+			{
+				SrcHosts:  getRegexs([]string{"vmui\\..+"}),
+				URLPrefix: mustParseURL("http://vmui.host:1234/vmui/"),
+			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]", "[]", nil, "least_loaded", 0)
-	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]", "[]", nil, "least_loaded", 0)
-	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0)
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0)
-	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0)
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
+	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "", "", nil, "least_loaded", 0)
+	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "", "", nil, "least_loaded", 0)
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0)
+	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "", "", nil, "least_loaded", 0)
+	f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "", "", nil, "least_loaded", 0)
+
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
-	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0)
+	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "", "", nil, "least_loaded", 0)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
-	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]", "[]", nil, "least_loaded", 0)
+	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "", "", nil, "least_loaded", 0)
 }
 
 func TestCreateTargetURLFailure(t *testing.T) {
@@ -231,7 +260,7 @@ func TestCreateTargetURLFailure(t *testing.T) {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, hc, dropSrcPathPrefixParts := ui.getURLPrefixAndHeaders(u)
+		up, hc := ui.getURLPrefixAndHeaders(u, nil)
 		if up != nil {
 			t.Fatalf("unexpected non-empty up=%#v", up)
 		}
@@ -241,17 +270,22 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if hc.ResponseHeaders != nil {
 			t.Fatalf("unexpected non-empty response headers=%q", hc.ResponseHeaders)
 		}
-		if dropSrcPathPrefixParts != 0 {
-			t.Fatalf("unexpected non-zero dropSrcPathPrefixParts=%d", dropSrcPathPrefixParts)
-		}
 	}
 	f(&UserInfo{}, "/foo/bar")
 	f(&UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://foobar/baz"),
 			},
 		},
 	}, "/api/v1/write")
 }
+
+func headersToString(hs []Header) string {
+	a := make([]string, len(hs))
+	for i, h := range hs {
+		a[i] = fmt.Sprintf("%s: %s", h.Name, h.Value)
+	}
+	return strings.Join(a, "\n")
+}