docs/CHANGELOG.md: cut v1.71.0

This should handle the case when the original job_name has been changed in -promscrape.config ,
while the resulting job label remains the same because it is overriden via relabeling.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -4,14 +4,14 @@ about: Create a report to help us improve
 title: ''
 labels: ''
 assignees: ''
-
 ---
 
 **Describe the bug**
 A clear and concise description of what the bug is.
-It would be great [upgrading](https://victoriametrics.github.io/#how-to-upgrade) to [the latest avaialble release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+It would be a great [upgrading](https://docs.victoriametrics.com/#how-to-upgrade) 
+to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 and verifying whether the bug is reproducible there.
-It is also recommended reading [troubleshooting docs](https://victoriametrics.github.io/#troubleshooting).
+It is also recommended reading [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
 
 **To Reproduce**
 Steps to reproduce the behavior.
@@ -19,9 +19,22 @@ Steps to reproduce the behavior.
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 
+**Logs**
+Check if any warnings or errors were logged by VictoriaMetrics components
+or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
+
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
+For VictoriaMetrics health-state issues please provide full-length screenshots 
+of Grafana dashboards if possible:
+* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+
+See how to setup monitoring here:
+* [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+* [montioring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+
 **Version**
 The line returned when passing `--version` command line flag to binary. For example:
 ```
@@ -30,15 +43,5 @@ victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
 ```
 
 **Used command-line flags**
-Command-line flags are listed as `flag{name="httpListenAddr", value=":443"} 1` lines at `/metrics` page.
-See the following docs for details:
+Please provide applied command-line flags used for running VictoriaMetrics and its components. 
 
-* [monitoring for single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#monitoring)
-* [montioring for VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#monitoring)
-
-**Additional context**
-Add any other context about the problem here such as error logs from VictoriaMetrics and Prometheus,
-`/metrics` output, screenshots from the official Grafana dashboards for VictoriaMetrics:
-
-* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
-* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)

.github/dependabot.yml

@@ -0,0 +1,26 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "bundler"
+    directory: "/docs"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/app/vmui/packages/vmui/web"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/app/vmui/packages/vmui"
+    schedule:
+      interval: "weekly"

.github/workflows/check-licenses.yml

@@ -0,0 +1,23 @@
+name: license-check
+on:
+  push:
+    paths:
+      - 'vendor'
+  pull_request:
+    paths:
+      - 'vendor'
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.17
+        id: go
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Check License
+        run: |
+          make check-licenses

.github/workflows/github-pages.yml

@@ -1,30 +0,0 @@
-name: github-pages
-on:
-  push:
-    paths:
-      - 'docs/*'
-      - 'README.md'
-    branches:
-      - master
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: publish
-        shell: bash
-        env:
-          TOKEN: ${{secrets.CI_TOKEN}}
-        run: |
-          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git gpages
-          cp docs/* gpages
-          cp README.md gpages
-          cd gpages
-          git config --local user.email "info@victoriametrics.com"
-          git config --local user.name "Vika"
-          git add .
-          git commit -m "update github pages"
-          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git"
-          git push "${remote_repo}"
-          cd ..
-          rm -rf gpages

.github/workflows/main.yml

@@ -16,15 +16,15 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
-      - name: Dependencies
-        run: |
-          go get -u golang.org/x/lint/golint
-          go get -u github.com/kisielk/errcheck
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.29.0
       - name: Code checkout
         uses: actions/checkout@master
+      - name: Dependencies
+        run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
       - name: Build
         env:
           GO111MODULE: on
@@ -60,8 +60,7 @@ jobs:
           GOOS=darwin go build -mod=vendor ./app/vmctl
           CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
       - name: Publish coverage
-        uses: codecov/codecov-action@v1.0.6
+        uses: codecov/codecov-action@v2.1.0
         with:
-          token: ${{secrets.CODECOV_TOKEN}}
           file: ./coverage.txt
 

.github/workflows/wiki.yml

@@ -16,7 +16,7 @@ jobs:
           TOKEN: ${{secrets.CI_TOKEN}}
         run: |
           git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
-          cp docs/* wiki
+          cp -r docs/* wiki
           cd wiki
           git config --local user.email "info@victoriametrics.com"
           git config --local user.name "Vika"

.gitignore

@@ -15,3 +15,7 @@
 /package/temp-rpm-*
 /package/*.deb
 /package/*.rpm
+.DS_store
+Gemfile.lock
+/_site
+_site

.wwhrd.yml

@@ -0,0 +1,5 @@
+allowlist:
+  - Apache-2.0
+  - MIT
+  - BSD-3-Clause
+  - BSD-2-Clause

CODE_OF_CONDUCT.md

@@ -7,7 +7,7 @@ contributors and maintainers pledge to making participation in our project and
 our community a harassment-free experience for everyone, regardless of age, body
 size, disability, ethnicity, sex characteristics, gender identity and expression,
 level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
+appearance, race, religion or sexual identity and orientation.
 
 ## Our Standards
 
@@ -24,9 +24,9 @@ Examples of unacceptable behavior by participants include:
 
 * The use of sexualized language or imagery and unwelcome sexual attention or
  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
+* Trolling, insulting/derogatory comments and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as a physical or electronic
+* Publishing others' private information, such as physical or electronic
  address, without explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
  professional setting
@@ -38,26 +38,26 @@ behavior and are expected to take appropriate and fair corrective action in
 response to any instances of unacceptable behavior.
 
 Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
+reject comments, commits, code, wiki edits, issues and other contributions
+that are not aligned to this Code of Conduct or to ban temporarily or
 permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
+threatening, offensive or harmful.
 
 ## Scope
 
 This Code of Conduct applies both within project spaces and in public spaces
 when an individual is representing the project or its community. Examples of
 representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
+address, posting via an official social media account or acting as an appointed
 representative at an online or offline event. Representation of a project may be
 further defined and clarified by project maintainers.
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
+Instances of abusive, harassing or otherwise unacceptable behavior may be
 reported by contacting the project team at info@victoriametrics.com. All
 complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
+is deemed necessary and appropriate for the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.
 Further details of specific enforcement policies may be posted separately.
 

Makefile

@@ -1,5 +1,6 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
+DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
@@ -8,7 +9,7 @@ ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
+GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -23,6 +24,8 @@ all: \
 
 include app/*/Makefile
 include deployment/*/Makefile
+include snap/local/Makefile
+
 
 clean:
 	rm -rf bin/*
@@ -83,9 +86,12 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
-release-snap:
-	snapcraft
-	snapcraft upload "victoriametrics_$(PKG_TAG)_multi.snap" --release beta,edge,candidate
+
+publish-release:
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
 
 release: \
 	release-victoria-metrics \
@@ -173,6 +179,7 @@ release-vmutils-windows-generic: \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 
+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -189,7 +196,7 @@ lint: install-golint
 	golint app/...
 
 install-golint:
-	which golint || go install golang.org/x/lint/golint
+	which golint || GO111MODULE=off go get golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
@@ -204,7 +211,7 @@ errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
 
 install-errcheck:
-	which errcheck || go install github.com/kisielk/errcheck
+	which errcheck || GO111MODULE=off go get github.com/kisielk/errcheck
 
 check-all: fmt vet lint errcheck golangci-lint
 
@@ -253,14 +260,20 @@ quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || go install github.com/valyala/quicktemplate/qtc
+	which qtc || GO111MODULE=off go get github.com/valyala/quicktemplate/qtc
 
 
 golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.29.0
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+
+install-wwhrd:
+	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
+
+check-licenses: install-wwhrd
+	wwhrd check -f .wwhrd.yml
 
 copy-docs:
 	echo "---\nsort: ${ORDER}\n---\n" > ${DST}
@@ -270,13 +283,13 @@ copy-docs:
 # Cluster docs are supposed to be ordered as 9th.
 # For The rest of docs is ordered manually.t
 docs-sync:
+	cp README.md docs/README.md
 	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md ORDER=1 $(MAKE) copy-docs
-	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=2 $(MAKE) copy-docs
-	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=3 $(MAKE) copy-docs
-	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=4 $(MAKE) copy-docs
-	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=5 $(MAKE) copy-docs
-	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=6 $(MAKE) copy-docs
-	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=7 $(MAKE) copy-docs
-	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=8 $(MAKE) copy-docs
-
-
+	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=5 $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=6 $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=7 $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=8 $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=9 $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md ORDER=10 $(MAKE) copy-docs

app/victoria-metrics/main.go

@@ -3,10 +3,8 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
-	"path"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
@@ -26,9 +24,8 @@ import (
 
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
-	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
-		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
-		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
 		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
@@ -54,7 +51,7 @@ func main() {
 
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	storage.SetDedupInterval(*minScrapeInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -89,14 +86,19 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
-		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics.</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://victoriametrics.github.io/'>https://victoriametrics.github.io/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints: </br>")
-		writeAPIHelp(w, [][]string{
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/metrics", "available service metrics"},
-			{"/api/v1/status/tsdb", "tsdb status page"},
+		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
+		fmt.Fprintf(w, "Useful endpoints:</br>")
+		httpserver.WriteAPIHelp(w, [][2]string{
+			{"vmui", "Web UI"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"api/v1/status/tsdb", "tsdb status page"},
+			{"api/v1/status/top_queries", "top queries"},
+			{"api/v1/status/active_queries", "active queries"},
 		})
 		return true
 	}
@@ -112,20 +114,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	return false
 }
 
-func writeAPIHelp(w io.Writer, pathList [][]string) {
-	pathPrefix := httpserver.GetPathPrefix()
-	for _, p := range pathList {
-		p, doc := p[0], p[1]
-		p = path.Join(pathPrefix, p)
-		fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
-	}
-}
-
 func usage() {
 	const s = `
 victoria-metrics is a time series database and monitoring solution.
 
-See the docs at https://victoriametrics.github.io/
+See the docs at https://docs.victoriametrics.com/
 `
 	flagutil.Usage(s)
 }

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -11,6 +11,6 @@
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[
-	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"]]}
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"], ["{TIME_S}","0.5"]]}
       ]}}
 }

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -19,6 +19,7 @@
 	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-100s}","3"],
 	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-40s}","2"],

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -13,6 +13,6 @@
     "data":{"resultType":"matrix",
       "result":[
 	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
-	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"]]}
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}", "3"]]}
       ]}}
 }

app/vmagent/README.md

@@ -17,32 +17,36 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 
 ## Features
 
-* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
-  See [Quick Start](#quick-start) for details.
+* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter). See [Quick Start](#quick-start) for details.
+* Can read data from Kafka. See [these docs](#reading-metrics-from-kafka).
+* Can write data to Kafka. See [these docs](#writing-metrics-to-kafka).
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Accepts data via all ingestion protocols supported by VictoriaMetrics:
-  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
-  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
-  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents).
+  * DataDog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent).
+  * InfluxDB line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
-  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-import-data-in-native-format).
-  * Data in Prometheus exposition format. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
-  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-import-csv-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-native-format).
+  * Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
+  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as the connection
   to the remote storage is repaired. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
 * Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared with Prometheus.
-* Scrape targets can be spread among multiple `vmagent` instances when big number of targets must be scraped. See [these docs](#scraping-big-number-of-targets) for details.
-
+* Scrape targets can be spread among multiple `vmagent` instances when big number of targets must be scraped. See [these docs](#scraping-big-number-of-targets).
+* Can efficiently scrape targets that expose millions of time series such as [/federate endpoint in Prometheus](https://prometheus.io/docs/prometheus/latest/federation/). See [these docs](#stream-parsing-mode).
+* Can deal with [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues by limiting the number of unique time series at scrape time and before sending them to remote storage systems. See [these docs](#cardinality-limiter).
+* Can load scrape configs from multiple files. See [these docs](#loading-scrape-configs-from-multiple-files).
 
 ## Quick Start
 
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and configure the following flags to the `vmagent` binary in order to start scraping Prometheus targets:
 
-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`)
+* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`). The path can point either to local file or to http url.
 * `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified multiple times to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
@@ -51,13 +55,13 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-If you only need to collect Influx data, then the following command is sufficient:
+If you only need to collect InfluxDB data, then the following command is sufficient:
 
 ```
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://victoriametrics.github.io/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+Then send InfluxDB data to `http://vmagent-host:8429`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
 
 `vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags).
 
@@ -133,7 +137,11 @@ Also, Basic Auth can be enabled for the incoming `remote_write` requests with `-
 
 ### remote_write for clustered version
 
-While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx, Prometheus, Graphite) and scrape data from various targets, writes are always peformed in Promethes remote_write protocol. Therefore for the [clustered version](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html), `-remoteWrite.url` the command-line flag should be configured as `<schema>://<vminsert-host>:8480/insert/<customer-id>/prometheus/api/v1/write`
+While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx, Prometheus, Graphite) and scrape data from various targets, writes are always peformed in Promethes remote_write protocol. Therefore for the [clustered version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html), `-remoteWrite.url` the command-line flag should be configured as `<schema>://<vminsert-host>:8480/insert/<accountID>/prometheus/api/v1/write` according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). There is also support for multitenant writes. See [these docs](#multitenancy).
+
+## Multitenancy
+
+By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. But it can accept multitenant data if `-remoteWrite.multitenantURL` is set. In this case it accepts multitenant data at `http://vmagent:8429/insert/<accountID>/...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert/<accountID>/prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants.
 
 
 ## How to collect metrics in Prometheus format
@@ -171,10 +179,16 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
 * `openstack_sd_configs` - is for scraping OpenStack targets.
   See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
   [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `docker_sd_configs` - is for scraping Docker targets.
+  See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details.
 * `dockerswarm_sd_configs` - is for scraping Docker Swarm targets.
   See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
 * `eureka_sd_configs` - is for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
   See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
+* `digitalocean_sd_configs` is for scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/)
+  See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
+* `http_sd_configs` is for scraping targerts registered in http service discovery.
+  See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
 
 Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -184,7 +198,12 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
   to save network bandwidth.
 * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
   By default, `vmagent` uses keep-alive connections to scrape targets to reduce overhead on connection re-establishing.
-* `stream_parse: true` - for scraping targets in a streaming manner. This may be useful for targets exporting big number of metrics.
+* `series_limit: N` - for limiting the number of unique time series a single scrape target can expose. See [these docs](#cardinality-limiter).
+* `stream_parse: true` - for scraping targets in a streaming manner. This may be useful for targets exporting big number of metrics. See [these docs](#stream-parsing-mode).
+* `scrape_align_interval: duration` - for aligning scrapes to the given interval instead of using random offset in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time.
+* `scrape_offset: duration` - for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`.
+* `relabel_debug: true` - for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling).
+* `metric_relabel_debug: true` - for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling).
 
 Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
 command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
@@ -193,11 +212,36 @@ entries to 60s. Run `vmagent -help` in order to see default values for the `-pro
 The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
 
 
+## Loading scrape configs from multiple files
+
+`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url:
+
+```yml
+scrape_config_files:
+- configs/*.yml
+- single_scrape_config.yml
+- https://config-server/scrape_config.yml
+```
+
+Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
+
+```yml
+- job_name: foo
+  static_configs:
+  - targets: ["vmagent:8429"]
+- job_name: bar
+  kubernetes_sd_configs:
+  - role: pod
+```
+
+`vmagent` dynamically reloads these files on `SIGHUP` signal or on the request to `http://vmagent:8429/-/reload`.
+
+
 ## Adding labels to metrics
 
 Labels can be added to metrics by the following mechanisms:
 
-* The `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in the `-promscrape.config` file. They aren't added to metrics collected via other [data ingestion protocols](https://victoriametrics.github.io/#how-to-import-time-series-data).
+* The `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in the `-promscrape.config` file. They aren't added to metrics collected via other [data ingestion protocols](https://docs.victoriametrics.com/#how-to-import-time-series-data).
 * The `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`. For example, the following command will start `vmagent`, which will add `{datacenter="foobar"}` label to all the metrics pushed to all the configured remote storage systems (all the `-remoteWrite.url` flag values):
 
 ```
@@ -207,20 +251,37 @@ Labels can be added to metrics by the following mechanisms:
 
 ## Relabeling
 
-`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
-and also provides the following actions:
+`vmagent` and VictoriaMetrics support Prometheus-compatible relabeling.
+They provide the following additional actions on top of actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config):
 
 * `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`.
 * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
 * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
 * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
+* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+
+The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
+
+```yaml
+- action: keep_metrics
+  regex: "metric_a|metric_b|foo_.+"
+```
+
+```yaml
+- action: keep_metrics
+  regex:
+  - "metric_a"
+  - "metric_b"
+  - "foo_.+"
+```
 
 The relabeling can be defined in the following places:
 
-* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels.
-* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`.
-* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
-* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
+* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
+* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
+* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
 
 You can read more about relabeling in the following articles:
 
@@ -232,10 +293,52 @@ You can read more about relabeling in the following articles:
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
 
+## Prometheus staleness markers
+
+`vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases:
+
+* If they are passed to `vmagent` via [Prometheus remote_write protocol](#prometheus-remote_write-proxy).
+* If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metric.
+* If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target.
+* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
+* Stale markers are sent for all the scraped metrics on graceful shutdown of `vmagent`.
+
+Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.
+
+
+## Stream parsing mode
+
+By default `vmagent` reads the full response body from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics.
+
+Stream parsing mode is automatically enabled for scrape targets returning response bodies with sizes bigger than the `-promscrape.minResponseSizeForStreamParse` command-line flag value. Additionally, the stream parsing mode can be explicitly enabled in the following places:
+
+- Via `-promscrape.streamParse` command-line flag. In this case all the scrape targets defined in the file pointed by `-promscrape.config` are scraped in stream parsing mode.
+- Via `stream_parse: true` option at `scrape_configs` section. In this case all the scrape targets defined in this section are scraped in stream parsing mode.
+- Via `__stream_parse__=true` label, which can be set via [relabeling](#relabeling) at `relabel_configs` section. In this case stream parsing mode is enabled for the corresponding scrape targets. Typical use case: to set the label via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets exposing big number of metrics.
+
+Examples:
+
+```yml
+scrape_configs:
+- job_name: 'big-federate'
+  stream_parse: true
+  static_configs:
+  - targets:
+    - big-prometeus1
+    - big-prometeus2
+  honor_labels: true
+  metrics_path: /federate
+  params:
+    'match[]': ['{__name__!=""}']
+```
+
+Note that `sample_limit` and `series_limit` options cannot be used in stream parsing mode because the parsed data is pushed to remote storage as soon as it is parsed.
+
+
 ## Scraping big number of targets
 
 A single `vmagent` instance can scrape tens of thousands of scrape targets. Sometimes this isn't enough due to limitations on CPU, network, RAM, etc.
-In this case scrape targets can be split among multiple `vmagent` instances (aka `vmagent` horizontal scaling and clustering).
+In this case scrape targets can be split among multiple `vmagent` instances (aka `vmagent` horizontal scaling, sharding and clustering).
 Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values.
 The flag value must be in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster.
 The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag. For example, the following commands
@@ -257,7 +360,7 @@ start a cluster of three `vmagent` instances, where each target is scraped by tw
 ```
 
 If each target is scraped by multiple `vmagent` instances, then data deduplication must be enabled at remote storage pointed by `-remoteWrite.url`.
-See [these docs](https://victoriametrics.github.io/#deduplication) for details.
+See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
 
 ## Scraping targets via a proxy
@@ -295,12 +398,38 @@ scrape_configs:
     server_name: real-server-name
 ```
 
+## Cardinality limiter
+
+By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced in the following places:
+
+- Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually to all the scrape targets defined in the file pointed by `-promscrape.config`.
+- Via `series_limit` config option at `scrape_config` section. This limit is applied individually to all the scrape targets defined in the given `scrape_config`.
+- Via `__series_limit__` label, which can be set with [relabeling](#relabeling) at `relabel_configs` section. This limit is applied to the corresponding scrape targets. Typical use case: to set the limit via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets, which may expose too high number of time series.
+
+All the scraped metrics are dropped for time series exceeding the given limit. The exceeded limit can be [monitored](#monitoring) via `promscrape_series_limit_rows_dropped_total` metric.
+
+See also `sample_limit` option at [scrape_config section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
+
+By default `vmagent` doesn't limit the number of time series written to remote storage systems specified at `-remoteWrite.url`. The limit can be enforced by setting the following command-line flags:
+
+* `-remoteWrite.maxHourlySeries` - limits the number of unique time series `vmagent` can write to remote storage systems during the last hour. Useful for limiting the number of active time series.
+* `-remoteWrite.maxDailySeries` - limits the number of unique time series `vmagent` can write to remote storage systems during the last day. Useful for limiting daily churn rate.
+
+Both limits can be set simultaneously. If any of these limits is reached, then samples for new time series are dropped instead of sending them to remote storage systems. A sample of dropped series is put in the log with `WARNING` level.
+
+The exceeded limits can be [monitored](#monitoring) with the following metrics:
+
+* `vmagent_hourly_series_limit_rows_dropped_total` - the number of metrics dropped due to exceeded hourly limit on the number of unique time series.
+* `vmagent_daily_series_limit_rows_dropped_total` - the number of metrics dropped due to exceeded daily limit on the number of unique time series.
+
+These limits are approximate, so `vmagent` can underflow/overflow the limit by a small percentage (usually less than 1%).
+
 
 ## Monitoring
 
 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. We recommend setting up regular scraping of this page
 either through `vmagent` itself or by Prometheus so that the exported metrics may be analyzed later.
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add a review to the dashboard.
 
 `vmagent` also exports the status for various targets at the following handlers:
@@ -322,6 +451,14 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 * We recommend you increase the maximum number of open files in the system (`ulimit -n`) when scraping a big number of targets,
   as `vmagent` establishes at least a single TCP connection per target.
 
+* If `vmagent` uses too big amounts of memory, then the following options can help:
+  * Disabling staleness tracking with `-promscrape.noStaleMarkers` option. See [these docs](#prometheus-staleness-markers).
+  * Enabling stream parsing mode if `vmagent` scrapes targets with millions of metrics per target. See [these docs](#stream-parsing-mode).
+  * Reducing the number of output queues with `-remoteWrite.queues` command-line option.
+  * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kuberntes if `vmagent` runs under these systems.
+  * Reducing the number of CPU cores vmagent can use by passing `GOMAXPROCS=N` environment variable to `vmagent`, where `N` is the desired limit on CPU cores. Another option is to reduce CPU limits in Docker or Kubernetes if `vmagent` runs under these systems.
+  * Passing `-promscrape.dropOriginalLabels` command-line option to `vmagent`, so it drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page. This reduces memory usage when scraping big number of targets at the cost of reduced debuggability for improperly configured per-target relabeling.
+
 * When `vmagent` scrapes many unreliable targets, it can flood the error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
   and `http://vmagent-host:8429/api/v1/targets`.
@@ -329,39 +466,13 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 * The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
   This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default the `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM. Therefore big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if a big number of scrape targets are dropped during relabeling.
 
-* If `vmagent` scrapes a big number of targets then the `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
-  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly. It is also recommended increasing `-remoteWrite.maxBlockSize` and `-remoteWrite.maxRowsPerBlock` command-line options in this case. This can improve data ingestion performance to the configured remote storage systems at the cost of higher memory usage.
 
-* If `vmagent` scrapes targets with millions of metrics per target (for example, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
-  we recommend enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all of the scrape targets
-  by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
-
-  ```yml
-  scrape_configs:
-  - job_name: 'big-federate'
-    stream_parse: true
-    static_configs:
-    - targets:
-      - big-prometeus1
-      - big-prometeus2
-    honor_labels: true
-    metrics_path: /federate
-    params:
-      'match[]': ['{__name__!=""}']
-  ```
-
-  Note that `sample_limit` option doesn't work if stream parsing is enabled because the parsed data is pushed to remote storage as soon as it is parsed. Therefore the `sample_limit` option
- doesn't make sense during stream parsing.
-
-* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly.
-
-* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`.
-  Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data
-  if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
+* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`. Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
 
 * `vmagent` drops data blocks if remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
-* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos.
+* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos, which typically emit `out of order sample` errors. The best solution is to use remote storage with [backfilling support](https://docs.victoriametrics.com/#backfilling).
 
 * `vmagent` buffers scraped data at the `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
@@ -408,6 +519,108 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
     regex: true
   ```
 
+## Kafka integration
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read and write metrics from / to Kafka:
+
+* [Reading metrics from Kafka](#reading-metrics-from-kafka)
+* [Writing metrics to Kafka](#writing-metrics-to-kafka)
+
+The enterprise version of vmagent is available for evaluation at [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page in `vmutils-*-enteprise.tar.gz` archives and in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+
+### Reading metrics from Kafka
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
+
+* `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). Messages in this format can be sent by vmagent - see [these docs](#writing-metrics-to-kafka).
+* `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
+* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
+* `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
+* `jsonline` - [JSON line format](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format).
+
+Every Kafka message may contain multiple lines in `influx`, `prometheus`, `graphite` and `jsonline` format delimited by `\n`.
+
+`vmagent` consumes messages from Kafka topics specified by `-kafka.consumer.topic` command-line flag. Multiple topics can be specified by passing multiple `-kafka.consumer.topic` command-line flags to `vmagent`.
+
+`vmagent` consumes messages from Kafka brokers specified by `-kafka.consumer.topic.brokers` command-line flag. Multiple brokers can be specified per each `-kafka.consumer.topic` by passing a list of brokers delimited by `;`. For example, `-kafka.consumer.topic.brokers=host1:9092;host2:9092`.
+
+The following command starts `vmagent`, which reads metrics in InfluxDB line protocol format from Kafka broker at `localhost:9092` from the topic `metrics-by-telegraf` and sends them to remote storage at `http://localhost:8428/api/v1/write`:
+
+```bash
+./bin/vmagent -remoteWrite.url=http://localhost:8428/api/v1/write \
+       -kafka.consumer.topic.brokers=localhost:9092 \
+       -kafka.consumer.topic.format=influx \
+       -kafka.consumer.topic=metrics-by-telegraf \
+       -kafka.consumer.topic.groupID=some-id
+```
+
+It is expected that [Telegraf](https://github.com/influxdata/telegraf) sends metrics to the `metrics-by-telegraf` topic with the following config:
+
+```yaml
+[[outputs.kafka]]
+brokers = ["localhost:9092"]
+topic = "influx"
+data_format = "influx"
+```
+
+
+#### Command-line flags for Kafka consumer
+
+These command-line flags are available only in [enterprise](https://victoriametrics.com/enterprise.html) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+```
+  -kafka.consumer.topic array
+        Kafka topic names for data consumption.
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.password array
+        Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.username array
+        Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.brokers array
+        List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.defaultFormat string
+        Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+  -kafka.consumer.topic.format array
+        data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.groupID array
+        Defines group.id for topic
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.isGzipped array
+        Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+        Supports array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.options array
+        Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+        Supports an array of values separated by comma or specified via multiple flags.
+```
+
+### Writing metrics to Kafka
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
+
+Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+
+
+#### Kafka broker authorization and authentication
+
+Two types of auth are supported:
+
+* sasl with username and password:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SASL_SSL&sasl.mechanisms=PLAIN -remoteWrite.basicAuth.username=user -remoteWrite.basicAuth.password=password
+```
+
+* tls certificates:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SSL -remoteWrite.tlsCAFile=/opt/ca.pem -remoteWrite.tlsCertFile=/opt/cert.pem -remoteWrite.tlsKeyFile=/opt/key.pem
+```
+
 
 ## How to build from sources
 
@@ -416,7 +629,7 @@ We recommend using [binary releases](https://github.com/VictoriaMetrics/Victoria
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.15.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
 2. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds the `vmagent` binary and puts it into the `bin` folder.
 
@@ -445,7 +658,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.15.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
 2. Run `make vmagent-arm` or `make vmagent-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
    It builds `vmagent-arm` or `vmagent-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -486,16 +699,21 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
 
 vmagent collects metrics data via popular data ingestion protocols and routes them to VictoriaMetrics.
 
-See the docs at https://victoriametrics.github.io/vmagent.html .
+See the docs at https://docs.victoriametrics.com/vmagent.html .
 
+  -configAuthKey string
+    	Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+  -datadog.maxInsertRequestSize size
+    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dryRun
     	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
@@ -505,17 +723,17 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -graphiteTrimTimestamp duration
     	Trim timestamps for Graphite data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
     	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
     	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
     	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
@@ -527,26 +745,26 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 104857600)
   -influx.databaseNames array
     	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize size
-    	The maximum size in bytes for a single Influx line during parsing
+    	The maximum size in bytes for a single InfluxDB line during parsing
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
   -influxListenAddr string
-    	TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
+    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
   -influxMeasurementFieldSeparator string
-    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol (default "_")
+    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
     	Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
   -influxSkipSingleField
-    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field
+    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
   -influxTrimTimestamp duration
-    	Trim timestamps for Influx line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
     	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
     	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
@@ -556,19 +774,19 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -loggerTimezone string
     	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
     	The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16)
   -maxInsertRequestSize size
     	The maximum size in bytes of a single Prometheus remote_write API request
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
     	TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
   -opentsdbListenAddr string
@@ -581,15 +799,15 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -opentsdbhttpTrimTimestamp duration
     	Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.cluster.memberNum int
     	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
   -promscrape.cluster.membersCount int
     	The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
   -promscrape.cluster.replicationFactor int
-    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://victoriametrics.github.io/#deduplication (default 1)
+    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
     	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
   -promscrape.config.strictParse
@@ -600,6 +818,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	Wait time used by Consul service discovery. Default value is used if not set
   -promscrape.consulSDCheckInterval duration
     	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+  -promscrape.digitaloceanSDCheckInterval duration
+    	Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config for details (default 1m0s)
   -promscrape.disableCompression
     	Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.disableKeepAlive
@@ -610,6 +830,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
   -promscrape.dnsSDCheckInterval duration
     	Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
+  -promscrape.dockerSDCheckInterval duration
+    	Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details (default 30s)
   -promscrape.dockerswarmSDCheckInterval duration
     	Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details (default 30s)
   -promscrape.dropOriginalLabels
@@ -622,53 +844,97 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+    	Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets int
     	The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+  -promscrape.maxResponseHeadersSize size
+    	The maximum size of http response headers from Prometheus scrape targets
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
+  -promscrape.minResponseSizeForStreamParse size
+    	The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
+  -promscrape.noStaleMarkers
+    	Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
     	Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
+  -promscrape.seriesLimitPerTarget int
+    	Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
     	Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
-    	Whether to suppress 'duplicate scrape target' errors; see https://victoriametrics.github.io/vmagent.html#troubleshooting for details
+    	Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
     	Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
   -remoteWrite.basicAuth.password array
     	Optional basic auth password to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.basicAuth.passwordFile array
+    	Optional path to basic auth password to use for -remoteWrite.url. The file is re-read every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.basicAuth.username array
     	Optional basic auth username to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.bearerToken array
     	Optional bearer auth token to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.bearerTokenFile array
+    	Optional path to bearer token file to use for -remoteWrite.url. The token is re-read from the file every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.flushInterval duration
     	Interval for flushing the data to remote storage. This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url (default 1s)
   -remoteWrite.label array
     	Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.maxBlockSize size
-    	The maximum size in bytes of unpacked request to send to remote storage. It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics
+    	The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
+  -remoteWrite.maxDailySeries int
+    	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
   -remoteWrite.maxDiskUsagePerURL size
     	The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. Disk usage is unlimited if the value is set to 0
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -remoteWrite.maxHourlySeries int
+    	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+  -remoteWrite.maxRowsPerBlock int
+    	The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
+  -remoteWrite.multitenantURL array
+    	Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.oauth2.clientID array
+    	Optional OAuth2 clientID to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.oauth2.clientSecret array
+    	Optional OAuth2 clientSecret to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.oauth2.clientSecretFile array
+    	Optional OAuth2 clientSecretFile to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.oauth2.scopes array
+    	Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.oauth2.tokenUrl array
+    	Optional OAuth2 tokenURL to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.proxyURL array
     	Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.queues int
-    	The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage (default 4)
+    	The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs (default 8)
   -remoteWrite.rateLimit array
     	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
     	Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
-    	Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://victoriametrics.github.io/vmagent.html#relabeling for details
+    	Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+  -remoteWrite.relabelDebug
+    	Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
     	Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
     	Supports array of values separated by comma or specified via multiple flags.
@@ -682,33 +948,36 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCAFile array
     	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCertFile array
     	Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsInsecureSkipVerify array
     	Whether to skip tls verification when connecting to -remoteWrite.url
     	Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsKeyFile array
     	Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsServerName array
     	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tmpDataPath string
     	Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
-    	Optional path to relabel config for the corresponding -remoteWrite.url
+    	Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.urlRelabelDebug array
+    	Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
     	Supports array of values separated by comma or specified via multiple flags.
   -sortLabels
     	Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
   -tlsKeyFile string
     	Path to file with TLS key. Used only if -tls is set
   -version

app/vmagent/csvimport/request_handler.go

@@ -5,32 +5,35 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="csvimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
 )
 
 // InsertHandler processes csv data from req.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -64,8 +67,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,99 @@
+package datadog
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
+		for _, tag := range ss.Tags {
+			n := strings.IndexByte(tag, ':')
+			if n < 0 {
+				return fmt.Errorf("cannot find ':' in tag %q", tag)
+			}
+			name := tag[:n]
+			value := tag[n+1:]
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -8,34 +8,37 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="influx"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
 )
 
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(r io.Reader) error {
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, nil)
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(nil, db, rows, nil)
 		})
 	})
 }
@@ -43,7 +46,7 @@ func InsertHandlerForReader(r io.Reader) error {
 // InsertHandlerForHTTP processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandlerForHTTP(req *http.Request) error {
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -55,12 +58,12 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 		db := q.Get("db")
 		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, extraLabels)
+			return insertRows(at, db, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -130,8 +133,11 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.Push(&ctx.ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 
 	return nil

app/vmagent/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
@@ -19,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -40,13 +43,14 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
@@ -92,7 +96,9 @@ func main() {
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
+		})
 	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
@@ -148,61 +154,72 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
-		fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
+		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
+		fmt.Fprintf(w, "Useful endpoints:</br>")
+		httpserver.WriteAPIHelp(w, [][2]string{
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
+		})
 		return true
 	}
+
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := promremotewrite.InsertHandler(r); err != nil {
+		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/api/v1/import":
 		vmimportRequests.Inc()
-		if err := vmimport.InsertHandler(r); err != nil {
+		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
-		if err := csvimport.InsertHandler(r); err != nil {
+		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/api/v1/import/prometheus":
 		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(r); err != nil {
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
 			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/api/v1/import/native":
 		nativeimportRequests.Inc()
-		if err := native.InsertHandler(r); err != nil {
+		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandlerForHTTP(r); err != nil {
+		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -211,13 +228,56 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
@@ -237,9 +297,121 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	}
+	if remotewrite.MultitenancyEnabled() {
+		return processMultitenantRequest(w, r, path)
+	}
 	return false
 }
 
+func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
+		return false
+	}
+	if p.Prefix != "insert" {
+		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+		return true
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
+		return true
+	}
+	switch p.Suffix {
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(at, r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(at, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(at, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(at, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(at, r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/write", "influx/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/query":
+		influxQueryRequests.Inc()
+		influxutils.WriteDatabaseNames(w)
+		return true
+	case "datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	default:
+		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
+		return true
+	}
+}
+
 var (
 	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
 	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
@@ -256,14 +428,23 @@ var (
 	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
 
 	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
+	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 
@@ -271,7 +452,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://victoriametrics.github.io/vmagent.html .
+See the docs at https://docs.victoriametrics.com/vmagent.html .
 `
 	flagutil.Usage(s)
 }

app/vmagent/native/request_handler.go

@@ -5,36 +5,39 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="native"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(block *parser.Block) error {
-			return insertRows(block, extraLabels)
+			return insertRows(at, block, extraLabels)
 		})
 	})
 }
 
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -42,6 +45,9 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	// since relabeling can prevent from inserting the rows.
 	rowsLen := len(block.Values)
 	rowsInserted.Add(rowsLen)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsLen)
+	}
 	rowsPerInsert.Update(float64(rowsLen))
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
@@ -80,6 +86,6 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/prometheusimport/request_handler.go

@@ -1,24 +1,28 @@
 package prometheusimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -30,12 +34,21 @@ func InsertHandler(req *http.Request) error {
 	return writeconcurrencylimiter.Do(func() error {
 		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		}, nil)
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader with optional gzip format
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		}, nil)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -69,8 +82,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }

app/vmagent/promremotewrite/request_handler.go

@@ -1,38 +1,51 @@
 package promremotewrite
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
-			return insertRows(tss, extraLabels)
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, extraLabels)
 		})
 	})
 }
 
-func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, nil)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -68,8 +81,11 @@ func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Labe
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmagent/remotewrite/client.go

@@ -2,8 +2,6 @@ package remotewrite
 
 import (
 	"bytes"
-	"crypto/tls"
-	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -42,17 +40,36 @@ var (
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for -remoteWrite.url. "+
+		"The file is re-read every second. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for -remoteWrite.url. "+
+		"The token is re-read from the file every second. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+
+	oauth2ClientID = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2ClientSecret = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2TokenURL = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 )
 
 type client struct {
 	sanitizedURL   string
 	remoteWriteURL string
-	authHeader     string
 	fq             *persistentqueue.FastQueue
 	hc             *http.Client
 
+	sendBlock func(block []byte) bool
+	authCfg   *promauth.Config
+
 	rl rateLimiter
 
 	bytesSent       *metrics.Counter
@@ -62,16 +79,18 @@ type client struct {
 	errorsCount     *metrics.Counter
 	packetsDropped  *metrics.Counter
 	retriesCount    *metrics.Counter
+	sendDuration    *metrics.FloatCounter
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
-	tlsCfg, err := getTLSConfig(argIdx)
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
 	}
+	tlsCfg := authCfg.NewTLSConfig()
 	tr := &http.Transport{
 		Dial:                statDial,
 		TLSClientConfig:     tlsCfg,
@@ -86,32 +105,16 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		if !strings.Contains(pURL, "://") {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
 		}
-		urlProxy, err := url.Parse(pURL)
+		pu, err := url.Parse(pURL)
 		if err != nil {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
 		}
-		tr.Proxy = http.ProxyURL(urlProxy)
-	}
-	authHeader := ""
-	username := basicAuthUsername.GetOptionalArg(argIdx)
-	password := basicAuthPassword.GetOptionalArg(argIdx)
-	if len(username) > 0 || len(password) > 0 {
-		// See https://en.wikipedia.org/wiki/Basic_access_authentication
-		token := username + ":" + password
-		token64 := base64.StdEncoding.EncodeToString([]byte(token))
-		authHeader = "Basic " + token64
-	}
-	token := bearerToken.GetOptionalArg(argIdx)
-	if len(token) > 0 {
-		if authHeader != "" {
-			logger.Fatalf("`-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", token)
-		}
-		authHeader = "Bearer " + token
+		tr.Proxy = http.ProxyURL(pu)
 	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
-		authHeader:     authHeader,
+		authCfg:        authCfg,
 		fq:             fq,
 		hc: &http.Client{
 			Transport: tr,
@@ -119,6 +122,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		},
 		stopCh: make(chan struct{}),
 	}
+	c.sendBlock = c.sendBlockHTTP
+	return c
+}
+
+func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
@@ -132,6 +140,7 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
+	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -140,7 +149,6 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		}()
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
-	return c
 }
 
 func (c *client) MustStop() {
@@ -149,23 +157,48 @@ func (c *client) MustStop() {
 	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
 }
 
-func getTLSConfig(argIdx int) (*tls.Config, error) {
-	c := &promauth.TLSConfig{
+func getAuthConfig(argIdx int) (*promauth.Config, error) {
+	username := basicAuthUsername.GetOptionalArg(argIdx)
+	password := basicAuthPassword.GetOptionalArg(argIdx)
+	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
+	var basicAuthCfg *promauth.BasicAuthConfig
+	if username != "" || password != "" || passwordFile != "" {
+		basicAuthCfg = &promauth.BasicAuthConfig{
+			Username:     username,
+			Password:     promauth.NewSecret(password),
+			PasswordFile: passwordFile,
+		}
+	}
+
+	token := bearerToken.GetOptionalArg(argIdx)
+	tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
+
+	var oauth2Cfg *promauth.OAuth2Config
+	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
+	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
+	if clientSecretFile != "" || clientSecret != "" {
+		oauth2Cfg = &promauth.OAuth2Config{
+			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
+			ClientSecret:     promauth.NewSecret(clientSecret),
+			ClientSecretFile: clientSecretFile,
+			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
+			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
+		}
+	}
+
+	tlsCfg := &promauth.TLSConfig{
 		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
 		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
 		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
 		ServerName:         tlsServerName.GetOptionalArg(argIdx),
 		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
-	if c.CAFile == "" && c.CertFile == "" && c.KeyFile == "" && c.ServerName == "" && !c.InsecureSkipVerify {
-		return nil, nil
-	}
-	cfg, err := promauth.NewConfig(".", nil, nil, "", "", c)
+
+	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate TLS config: %w", err)
+		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
-	tlsCfg := cfg.NewTLSConfig()
-	return tlsCfg, nil
+	return authCfg, nil
 }
 
 func (c *client) runWorker() {
@@ -178,7 +211,9 @@ func (c *client) runWorker() {
 			return
 		}
 		go func() {
+			startTime := time.Now()
 			ch <- c.sendBlock(block)
+			c.sendDuration.Add(time.Since(startTime).Seconds())
 		}()
 		select {
 		case ok := <-ch:
@@ -207,9 +242,9 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlock returns false only if c.stopCh is closed.
+// sendBlockHTTP returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlock(block []byte) bool {
+func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
 	retryDuration := time.Second
 	retriesCount := 0
@@ -219,15 +254,15 @@ func (c *client) sendBlock(block []byte) bool {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
 	h.Set("Content-Encoding", "snappy")
 	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	if c.authHeader != "" {
-		req.Header.Set("Authorization", c.authHeader)
+	if ah := c.authCfg.GetAuthHeader(); ah != "" {
+		req.Header.Set("Authorization", ah)
 	}
 
 	startTime := time.Now()
@@ -239,7 +274,7 @@ again:
 		if retryDuration > time.Minute {
 			retryDuration = time.Minute
 		}
-		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := timerpool.Get(retryDuration)
 		select {
@@ -260,11 +295,9 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
-		// Just drop block on 409 status code like Prometheus does.
+		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
-		// drop block on 400 status code,
-		// not expected that remote server will be able to handle it on retry
-		// should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
 		_ = resp.Body.Close()
 		c.packetsDropped.Inc()
 		return true

app/vmagent/remotewrite/pendingseries.go

@@ -20,16 +20,10 @@ import (
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
-		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
+	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 )
 
-// the maximum number of rows to send per each block.
-const maxRowsPerBlock = 10000
-
-// the maximum number of labels to send per each block.
-const maxLabelsPerBlock = 40000
-
 type pendingSeries struct {
 	mu sync.Mutex
 	wr writeRequest
@@ -128,7 +122,6 @@ func (wr *writeRequest) reset() {
 }
 
 func (wr *writeRequest) flush() {
-	sortLabelsIfNeeded(wr.tss)
 	wr.wr.Timeseries = wr.tss
 	wr.adjustSampleValues()
 	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
@@ -154,10 +147,13 @@ func (wr *writeRequest) adjustSampleValues() {
 
 func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
-		if len(wr.samples) >= maxRowsPerBlock || len(wr.labels) >= maxLabelsPerBlock {
+		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss

app/vmagent/remotewrite/relabel.go

@@ -15,9 +15,16 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
-		"before sending them to -remoteWrite.url. See https://victoriametrics.github.io/vmagent.html#relabeling for details")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
+	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
+		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
+		"This is useful for debugging the relabeling configs")
 )
 
 var labelsGlobal []prompbmarshal.Label
@@ -31,23 +38,23 @@ func CheckRelabelConfigs() error {
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
-		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal, *relabelDebugGlobal)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
 		rcs.global = global
 	}
-	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
-		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), len(*remoteWriteURLs))
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url or -remoteWrite.multitenantURL args: %d",
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.
 			continue
 		}
-		prc, err := promrelabel.LoadRelabelConfigs(path)
+		prc, err := promrelabel.LoadRelabelConfigs(path, relabelDebug.GetOptionalArg(i))
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}

app/vmagent/remotewrite/remotewrite.go

@@ -3,9 +3,15 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"net/url"
+	"strconv"
 	"sync"
 	"sync/atomic"
+	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -13,6 +19,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 	xxhash "github.com/cespare/xxhash/v2"
 )
@@ -20,11 +28,14 @@ import (
 var (
 	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
-	queues = flag.Int("remoteWrite.queues", 4, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage")
+	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -38,16 +49,39 @@ var (
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
 		"By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
 		"This option may be used for improving data compression for the stored metrics")
+	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
+		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
+		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
+		`Enabled sorting for labels can slow down ingestion performance a bit`)
+	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
+	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 )
 
-var rwctxs []*remoteWriteCtx
+var (
+	// rwctxsDefault contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsDefault []*remoteWriteCtx
+
+	// rwctxsMap contains dynamically populated entries when -remoteWrite.multitenantURL is specified.
+	rwctxsMap     = make(map[tenantmetrics.TenantID][]*remoteWriteCtx)
+	rwctxsMapLock sync.Mutex
+
+	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
+	defaultAuthToken = &auth.Token{}
+)
+
+// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
+func MultitenancyEnabled() bool {
+	return len(*remoteWriteMultitenantURLs) > 0
+}
 
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Value
 
 // maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
 // since it may lead to high memory usage due to big number of buffers.
-var maxQueues = cgroup.AvailableCPUs() * 4
+var maxQueues = cgroup.AvailableCPUs() * 16
 
 // InitSecretFlags must be called after flag.Parse and before any logging.
 func InitSecretFlags() {
@@ -63,8 +97,29 @@ func InitSecretFlags() {
 //
 // Stop must be called for graceful shutdown.
 func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-remoteWrite.multitenantURL` command-line flag must be set")
+	}
+	if len(*remoteWriteURLs) > 0 && len(*remoteWriteMultitenantURLs) > 0 {
+		logger.Fatalf("cannot set both `-remoteWrite.url` and `-remoteWrite.multitenantURL` command-line flags")
+	}
+	if *maxHourlySeries > 0 {
+		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
+		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
+			return float64(hourlySeriesLimiter.MaxItems())
+		})
+		_ = metrics.NewGauge(`vmagent_hourly_series_limit_current_series`, func() float64 {
+			return float64(hourlySeriesLimiter.CurrentItems())
+		})
+	}
+	if *maxDailySeries > 0 {
+		dailySeriesLimiter = bloomfilter.NewLimiter(*maxDailySeries, 24*time.Hour)
+		_ = metrics.NewGauge(`vmagent_daily_series_limit_max_series`, func() float64 {
+			return float64(dailySeriesLimiter.MaxItems())
+		})
+		_ = metrics.NewGauge(`vmagent_daily_series_limit_current_series`, func() float64 {
+			return float64(dailySeriesLimiter.CurrentItems())
+		})
 	}
 	if *queues > maxQueues {
 		*queues = maxQueues
@@ -73,33 +128,23 @@ func Init() {
 		*queues = 1
 	}
 	initLabelsGlobal()
+
+	// Register SIGHUP handler for config reload before loadRelabelConfigs.
+	// This guarantees that the config will be re-read if the signal arrives just after loadRelabelConfig.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
 	rcs, err := loadRelabelConfigs()
 	if err != nil {
 		logger.Fatalf("cannot load relabel configs: %s", err)
 	}
 	allRelabelConfigs.Store(rcs)
 
-	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
-	if maxInmemoryBlocks > 200 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 200
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
-	for i, remoteWriteURL := range *remoteWriteURLs {
-		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
-		if *showRemoteWriteURL {
-			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
-		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
-		rwctxs = append(rwctxs, rwctx)
+	if len(*remoteWriteURLs) > 0 {
+		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
 
 	// Start config reloader.
-	sighupCh := procutil.NewSighupChan()
 	configReloaderWG.Add(1)
 	go func() {
 		defer configReloaderWG.Done()
@@ -121,6 +166,41 @@ func Init() {
 	}()
 }
 
+func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
+	if len(urls) == 0 {
+		logger.Panicf("BUG: urls must be non-empty")
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * *queues
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	rwctxs := make([]*remoteWriteCtx, len(urls))
+	for i, remoteWriteURLRaw := range urls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+		if at != nil {
+			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
+			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
+		}
+		if *showRemoteWriteURL {
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+		}
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+	}
+	return rwctxs
+}
+
 var stopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
@@ -131,16 +211,62 @@ func Stop() {
 	close(stopCh)
 	configReloaderWG.Wait()
 
-	for _, rwctx := range rwctxs {
+	for _, rwctx := range rwctxsDefault {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsDefault = nil
+
+	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
+	for _, rwctxs := range rwctxsMap {
+		for _, rwctx := range rwctxs {
+			rwctx.MustStop()
+		}
+	}
+	rwctxsMap = nil
+
+	if sl := hourlySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
+	if sl := dailySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
 }
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
 func Push(wr *prompbmarshal.WriteRequest) {
+	PushWithAuthToken(nil, wr)
+}
+
+// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
+//
+// Note that wr may be modified by Push due to relabeling and rounding.
+func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
+		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
+		at = defaultAuthToken
+	}
+	var rwctxs []*remoteWriteCtx
+	if at == nil {
+		rwctxs = rwctxsDefault
+	} else {
+		if len(*remoteWriteMultitenantURLs) == 0 {
+			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+		}
+		rwctxsMapLock.Lock()
+		tenantID := tenantmetrics.TenantID{
+			AccountID: at.AccountID,
+			ProjectID: at.ProjectID,
+		}
+		rwctxs = rwctxsMap[tenantID]
+		if rwctxs == nil {
+			rwctxs = newRemoteWriteCtxs(at, *remoteWriteMultitenantURLs)
+			rwctxsMap[tenantID] = rwctxs
+		}
+		rwctxsMapLock.Unlock()
+	}
+
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
 	pcsGlobal := rcs.global
@@ -148,6 +274,9 @@ func Push(wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -157,7 +286,7 @@ func Push(wr *prompbmarshal.WriteRequest) {
 			samplesCount += len(tss[i].Samples)
 			labelsCount += len(tss[i].Labels)
 			i++
-			if samplesCount >= maxRowsPerBlock || labelsCount >= maxLabelsPerBlock {
+			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
 			}
 		}
@@ -173,9 +302,9 @@ func Push(wr *prompbmarshal.WriteRequest) {
 			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
 			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
 		}
-		for _, rwctx := range rwctxs {
-			rwctx.Push(tssBlock)
-		}
+		sortLabelsIfNeeded(tssBlock)
+		tssBlock = limitSeriesCardinality(tssBlock)
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
 		if rctx != nil {
 			rctx.reset()
 		}
@@ -185,6 +314,104 @@ func Push(wr *prompbmarshal.WriteRequest) {
 	}
 }
 
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+	if len(tssBlock) == 0 {
+		// Nothing to push
+		return
+	}
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	for _, rwctx := range rwctxs {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			rwctx.Push(tssBlock)
+		}(rwctx)
+	}
+	wg.Wait()
+}
+
+// sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
+func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
+	if !*sortLabels {
+		return
+	}
+	for i := range tss {
+		promrelabel.SortLabels(tss[i].Labels)
+	}
+}
+
+func limitSeriesCardinality(tss []prompbmarshal.TimeSeries) []prompbmarshal.TimeSeries {
+	if hourlySeriesLimiter == nil && dailySeriesLimiter == nil {
+		return tss
+	}
+	dst := make([]prompbmarshal.TimeSeries, 0, len(tss))
+	for i := range tss {
+		labels := tss[i].Labels
+		h := getLabelsHash(labels)
+		if hourlySeriesLimiter != nil && !hourlySeriesLimiter.Add(h) {
+			hourlySeriesLimitRowsDropped.Add(len(tss[i].Samples))
+			logSkippedSeries(labels, "-remoteWrite.maxHourlySeries", hourlySeriesLimiter.MaxItems())
+			continue
+		}
+		if dailySeriesLimiter != nil && !dailySeriesLimiter.Add(h) {
+			dailySeriesLimitRowsDropped.Add(len(tss[i].Samples))
+			logSkippedSeries(labels, "-remoteWrite.maxDailySeries", dailySeriesLimiter.MaxItems())
+			continue
+		}
+		dst = append(dst, tss[i])
+	}
+	return dst
+}
+
+var (
+	hourlySeriesLimiter *bloomfilter.Limiter
+	dailySeriesLimiter  *bloomfilter.Limiter
+
+	hourlySeriesLimitRowsDropped = metrics.NewCounter(`vmagent_hourly_series_limit_rows_dropped_total`)
+	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
+)
+
+func getLabelsHash(labels []prompbmarshal.Label) uint64 {
+	bb := labelsHashBufPool.Get()
+	b := bb.B[:0]
+	for _, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, label.Value...)
+	}
+	h := xxhash.Sum64(b)
+	bb.B = b
+	labelsHashBufPool.Put(bb)
+	return h
+}
+
+var labelsHashBufPool bytesutil.ByteBufferPool
+
+func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
+	select {
+	case <-logSkippedSeriesTicker.C:
+		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
+	default:
+	}
+}
+
+var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b []byte
+	b = append(b, '{')
+	for i, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		if i+1 < len(labels) {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, '}')
+	return string(b)
+}
+
 var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
 
 type remoteWriteCtx struct {
@@ -197,17 +424,29 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
-	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+	// strip query params, otherwise changing params resets pq
+	pqURL := *remoteWriteURL
+	pqURL.RawQuery = ""
+	pqURL.Fragment = ""
+	h := xxhash.Sum64([]byte(pqURL.String()))
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
+	var c *client
+	switch remoteWriteURL.Scheme {
+	case "http", "https":
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+	default:
+		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+	}
+	c.init(argIdx, *queues, sanitizedURL)
+
 	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
 	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
@@ -226,7 +465,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 

app/vmagent/remotewrite/sort_labels.go

@@ -1,51 +0,0 @@
-package remotewrite
-
-import (
-	"flag"
-	"sort"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-)
-
-var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
-	`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
-	`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
-	`Enabled sorting for labels can slow down ingestion performance a bit`)
-
-// sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
-func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
-	if !*sortLabels {
-		return
-	}
-	// The slc is used for avoiding memory allocation when passing labels to sort.Sort.
-	slc := sortLabelsCtxPool.Get().(*sortLabelsCtx)
-	for i := range tss {
-		slc.labels = tss[i].Labels
-		sort.Sort(&slc.labels)
-	}
-	slc.labels = nil
-	sortLabelsCtxPool.Put(slc)
-}
-
-type sortLabelsCtx struct {
-	labels sortedLabels
-}
-
-var sortLabelsCtxPool = &sync.Pool{
-	New: func() interface{} {
-		return &sortLabelsCtx{}
-	},
-}
-
-type sortedLabels []prompbmarshal.Label
-
-func (sl *sortedLabels) Len() int { return len(*sl) }
-func (sl *sortedLabels) Less(i, j int) bool {
-	a := *sl
-	return a[i].Name < a[j].Name
-}
-func (sl *sortedLabels) Swap(i, j int) {
-	a := *sl
-	a[i], a[j] = a[j], a[i]
-}

app/vmagent/vmimport/request_handler.go

@@ -1,40 +1,54 @@
 package vmimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="vmimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -74,8 +88,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmalert/Makefile

@@ -56,6 +56,7 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
+	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
@@ -66,7 +67,17 @@ run-vmalert: vmalert
 		-remoteRead.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
-		-evaluationInterval=3s
+		-evaluationInterval=3s \
+		-rule.configCheckInterval=10s
+
+replay-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-external.label=cluster=east-1 \
+		-external.label=replica=a \
+		-replay.timeFrom=2021-05-11T07:21:43Z \
+		-replay.timeTo=2021-05-29T18:40:43Z
 
 vmalert-amd64:
 	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmalert-local-with-goarch

app/vmalert/README.md

@@ -1,28 +1,33 @@
 # vmalert
 
-`vmalert` executes a list of given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
+`vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured address.
+rules against configured `-datasource.url`. For sending alerting notifications
+vmalert relies on [Alertmanager]((https://github.com/prometheus/alertmanager)) configured via `-notifier.url` flag.
+Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+protocol and require `-remoteWrite.url` to be configured.
+Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
+implementation and aims to be compatible with its syntax.
 
 ## Features
 * Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
-* VictoriaMetrics [MetricsQL](https://victoriametrics.github.io/MetricsQL.html)
+* VictoriaMetrics [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html)
  support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
-* Keeps the alerts [state on restarts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmalert#alerts-state-on-restarts);
-* Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite) for details.
+* Keeps the alerts [state on restarts](#alerts-state-on-restarts);
+* Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
+* Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight without extra dependencies.
 
 ## Limitations
-* `vmalert` execute queries against remote datasource which has reliability risks because of network.
-It is recommended to configure alerts thresholds and rules expressions with understanding that network request
-may fail;
-* by default, rules execution is sequential within one group, but persisting of execution results to remote
-storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
-recording rule is reused in next one;
-* `vmalert` has no UI, just an API for getting groups and rules statuses.
+* `vmalert` execute queries against remote datasource which has reliability risks because of the network.
+It is recommended to configure alerts thresholds and rules expressions with the understanding that network 
+requests may fail;
+* by default, rules execution is sequential within one group, but persistence of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
+recording rule is reused in the next one;
 
 ## QuickStart
 
@@ -32,28 +37,37 @@ git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
 ```
-The build binary will be placed to `VictoriaMetrics/bin` folder.
+The build binary will be placed in `VictoriaMetrics/bin` folder.
 
 To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable VictoriaMetrics instance for rules execution;
-* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts and sending notifications.
-* remote write address - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-compatible storage address for storing recording rules results and alerts state in for of timeseries. This is optional.
+* datasource address - reachable MetricsQL endpoint to run queries against;
+* notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
+aggregating alerts, and sending notifications.
+* remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+  compatible storage to persist rules and alerts state info;
+* remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
 ```
-./bin/vmalert -rule=alert.rules \
+./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
-    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
-    -remoteWrite.url=http://localhost:8428 \ # remote write compatible storage to persist rules
-    -remoteRead.url=http://localhost:8428 \  # PromQL compatible datasource to restore alerts state from
+    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
+    -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
     -external.label=cluster=east-1 \         # External label to be applied for each rule
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
+Note there's a separate `remoteRead.url` to allow writing results of
+alerting/recording rules into a different storage than the initial data that's
+queried. This allows using `vmalert` to aggregate data from a short-term,
+high-frequency, high-cardinality storage into a long-term storage with
+decreased cardinality and a bigger interval between samples.
+
+See the full list of configuration flags in [configuration](#configuration) section.
+
 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
 to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts.
 
@@ -61,7 +75,7 @@ Configuration for [recording](https://prometheus.io/docs/prometheus/latest/confi
 and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very
 similar to Prometheus rules and configured using YAML. Configuration examples may be found
 in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
-Every `rule` belongs to `group` and every configuration file may contain arbitrary number of groups:
+Every `rule` belongs to a `group` and every configuration file may contain arbitrary number of groups:
 ```yaml
 groups:
   [ - <rule_group> ]
@@ -69,57 +83,85 @@ groups:
 
 ### Groups
 
-Each group has following attributes:
+Each group has the following attributes:
 ```yaml
 # The name of the group. Must be unique within a file.
 name: <string>
 
 # How often rules in the group are evaluated.
-[ interval: <duration> | default = global.evaluation_interval ]
+[ interval: <duration> | default = -evaluationInterval flag ]
 
-# How many rules execute at once. Increasing concurrency may speed
+# How many rules execute at once within a group. Increasing concurrency may speed
 # up round execution speed.
 [ concurrency: <integer> | default = 1 ]
 
 # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
-# By default "prometheus" rule type is used.
+# By default "prometheus" type is used.
 [ type: <string> ]
 
+# Warning: DEPRECATED
+# Please use `params` instead:
+#   params:
+#     extra_label: ["job=nodeexporter", "env=prod"]
+extra_filter_labels:
+  [ <labelname>: <labelvalue> ... ]
+
+# Optional list of HTTP URL parameters
+# applied for all rules requests within a group
+# For example:
+#  params:
+#    nocache: ["1"]                # disable caching for vmselect
+#    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+# see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+params:
+  [ <string>: [<string>, ...]]
+
+# Optional list of labels added to every rule within a group.
+# It has priority over the external labels.
+# Labels are commonly used for adding environment
+# or tenant-specific tag.
+labels:
+  [ <labelname>: <labelvalue> ... ]
+
 rules:
   [ - <rule> ... ]
 ```
 
 ### Rules
 
+Every rule contains `expr` field for [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
+or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. Vmalert will execute the configured
+expression and then act according to the Rule type.
+
 There are two types of Rules:
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allows to define alert conditions via [MetricsQL](https://victoriametrics.github.io/MetricsQL.html)
-and to send notifications about firing alerts to [Alertmanager](https://github.com/prometheus/alertmanager).
+Alerting rules allow defining alert conditions via `expr` field and to send notifications to
+[Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allow you to precompute frequently needed or computationally expensive expressions
-and save their result as a new set of time series.
+Recording rules allow defining `expr` which result will be then backfilled to configured
+`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
+expensive expressions and save their result as a new set of time series.
 
-`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.
 
 #### Alerting rules
 
-The syntax for alerting rule is following:
+The syntax for alerting rule is the following:
 ```yaml
 # The name of the alert. Must be a valid metric name.
 alert: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If type="graphite", then the expression
+# By default PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
 # Alerts are considered firing once they have been returned for this long.
-# Alerts which have not yet fired for long enough are considered pending.
+# Alerts which have not yet been fired for long enough are considered pending.
+# If param is omitted or set to 0 then alerts will be immediately considered
+# as firing once they return.
 [ for: <duration> | default = 0s ]
 
 # Labels to add or overwrite for each alert.
@@ -131,6 +173,11 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ```
 
+It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations
+to format data, iterate over it or execute expressions.
+Additionally, `vmalert` provides some extra templating functions
+listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go).
+
 #### Recording rules
 
 The syntax for recording rules is following:
@@ -138,12 +185,8 @@ The syntax for recording rules is following:
 # The name of the time series to output to. Must be a valid metric name.
 record: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If type="graphite", then the expression
+# By default MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
@@ -152,28 +195,176 @@ labels:
   [ <labelname>: <labelvalue> ]
 ```
 
-For recording rules to work `-remoteWrite.url` must specified.
+For recording rules to work `-remoteWrite.url` must be specified.
 
 
 ### Alerts state on restarts
 
-`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after reloading of `vmalert`
+`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after restart of `vmalert`
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
-* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or VMInsert (Cluster). `vmalert` will persist alerts state
+* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
 into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and may be queried from VM just as any other time series.
-The state stored to the configured address on every rule evaluation.
-* `-remoteRead.url` - URL to VictoriaMetrics (Single) or VMSelect (Cluster). `vmalert` will try to restore alerts state
+These are regular time series and maybe queried from VM just as any other time series.
+The state is stored to the configured address on every rule evaluation.
+* `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
 from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
-Both flags are required for the proper state restoring. Restore process may fail if time series are missing
-in configured `-remoteRead.url`, weren't updated in the last `1h` or received state doesn't match current `vmalert`
-rules configuration.
+Both flags are required for proper state restoration. Restore process may fail if time series are missing
+in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
+or received state doesn't match current `vmalert` rules configuration.
 
 
-### WEB
+### Multitenancy
+
+There are the following approaches exist for alerting and recording rules across
+[multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy):
+
+* To run a separate `vmalert` instance per each tenant.
+  The corresponding tenant must be specified in `-datasource.url` command-line flag
+  according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format).
+  For example, `/path/to/vmalert -datasource.url=http://vmselect:8481/select/123/prometheus`
+  would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line
+  flag must contain the url for the specific tenant as well.
+  For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording
+  rules to `AccountID=123`.
+
+* To specify `tenant` parameter per each alerting and recording group if
+  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used
+  with `-clusterMode` command-line flag. For example:
+
+```yaml
+groups:
+- name: rules_for_tenant_123
+  tenant: "123"
+  rules:
+    # Rules for accountID=123
+
+- name: rules_for_tenant_456:789
+  tenant: "456:789"
+  rules:
+    # Rules for accountID=456, projectID=789
+```
+
+If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
+contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
+`vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
+
+If `-clusterMode` is enabled and the `tenant` in a particular group is missing, then the tenant value
+is obtained from `-defaultTenant.prometheus` or `-defaultTenant.graphite` depending on the `type` of the group.
+
+The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files
+at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
+tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
+
+### Topology examples
+
+The following sections are showing how `vmalert` may be used and configured 
+for different scenarios. 
+
+Please note, not all flags in examples are required: 
+* `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
+you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+* `-notifier.url` is optional and is needed only if you have alerting rules.
+
+#### Single-node VictoriaMetrics
+
+The simplest configuration where one single-node VM server is used for
+rules execution, storing recording rules results and alerts state.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                    # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093          # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert single" width="500" src="vmalert_single.png">
+
+
+#### Cluster VictoriaMetrics
+
+In [cluster mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html)
+VictoriaMetrics has separate components for writing and reading path:
+`vminsert` and `vmselect` components respectively. `vmselect` is used for executing rules expressions
+and `vminsert` is used to persist recording rules results and alerts state.
+Cluster mode could have multiple `vminsert` and `vmselect` components. 
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                                # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for executing rules expressions
+    -remoteWrite.url=http://vminsert:8480/insert/0/prometheuss  # vminsert addr to persist alerts state and recording rules results
+    -remoteRead.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093                      # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert cluster" src="vmalert_cluster.png">
+
+In case when you want to spread the load on these components - add balancers before them and configure
+`vmalert` with balancer's addresses. Please, see more about VM's cluster architecture
+[here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview).
+
+#### HA vmalert
+
+For HA user can run multiple identically configured `vmalert` instances.
+It means all of them will execute the same rules, write state and results to
+the same destinations, and send alert notifications to multiple configured
+Alertmanagers.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml \                   # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager1:9093 \       # Multiple AlertManager addresses to send alerts when they trigger
+    -notifier.url=http://alertmanagerN:9093         # The same alert will be sent to all configured notifiers
+```
+
+<img alt="vmalert ha" width="800px" src="vmalert_ha.png">
+
+To avoid recording rules results and alerts state duplication in VictoriaMetrics server
+don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+
+Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
+all `vmalert`s are having the same config.
+
+Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
+for Alertmanagers for better reliability.
+
+This example uses single-node VM server for the sake of simplicity. 
+Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. 
+
+
+#### Downsampling and aggregation via vmalert
+
+Example shows how to build a topology where `vmalert` will process data from one cluster
+and write results into another. Such clusters may be called as "hot" (low retention,
+high-speed disks, used for operative monitoring) and "cold" (long term retention,
+slower/cheaper disks, low resolution data). With help of `vmalert`, user can setup
+recording rules to process raw data from "hot" cluster (by applying additional transformations
+or reducing resolution) and push results to "cold" cluster.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheuss   # vminsert addr to persist recording rules results
+```
+
+<img alt="vmalert multi cluster" src="vmalert_multicluster.png">
+
+Please note, [replay](#rules-backfilling) feature may be used for transforming historical data.
+
+Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only recording rules are used.
+
+
+### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
 * `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status" ` - get alert status by ID.
@@ -188,27 +379,140 @@ vmalert sends requests to `<-datasource.url>/render?format=json` during evaluati
 if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
 implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
 When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
-to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on query type.
+to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
+
+## Rules backfilling
+
+vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert
+can read the same rules configuration as normal, evaluate them on the given time range and backfill
+results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
+data source for backfilling.
+
+### How it works
+
+In `replay` mode vmalert works as a cli-tool and exits immediately after work is done.
+To run vmalert in `replay` mode:
+```
+./bin/vmalert -rule=path/to/your.rules \        # path to files with rules you usually use with vmalert
+    -datasource.url=http://localhost:8428 \     # PromQL/MetricsQL compatible datasource
+    -remoteWrite.url=http://localhost:8428 \    # remote write compatible storage to persist results
+    -replay.timeFrom=2021-05-11T07:21:43Z \     # time from begin replay
+    -replay.timeTo=2021-05-29T18:40:43Z         # time to finish replay
+```
+
+The output of the command will look like the following:
+```
+Replay mode:
+from:   2021-05-11 07:21:43 +0000 UTC   # set by -replay.timeFrom
+to:     2021-05-29 18:40:43 +0000 UTC   # set by -replay.timeTo
+max data points per request: 1000       # set by -replay.maxDatapointsPerQuery
+
+Group "ReplayGroup"
+interval:       1m0s
+requests to make:       27
+max range per request:  16h40m0s
+> Rule "type:vm_cache_entries:rate5m" (ID: 1792509946081842725)
+27 / 27 [----------------------------------------------------------------------------------------------------] 100.00% 78 p/s
+> Rule "go_cgo_calls_count:rate5m" (ID: 17958425467471411582)
+27 / 27 [-----------------------------------------------------------------------------------------------------] 100.00% ? p/s
+
+Group "vmsingleReplay"
+interval:       30s
+requests to make:       54
+max range per request:  8h20m0s
+> Rule "RequestErrorsToAPI" (ID: 17645863024999990222)
+54 / 54 [-----------------------------------------------------------------------------------------------------] 100.00% ? p/s
+> Rule "TooManyLogs" (ID: 9042195394653477652)
+54 / 54 [-----------------------------------------------------------------------------------------------------] 100.00% ? p/s
+2021-06-07T09:59:12.098Z        info    app/vmalert/replay.go:68        replay finished! Imported 511734 samples
+```
+
+In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
+executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
+to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
+of the configured `-datasource.url`. Returned data is then processed according to the rule type and
+backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
+Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
+Vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
+to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
+
+#### Recording rules
+
+The result of recording rules `replay` should match with results of normal rules evaluation.
+
+#### Alerting rules
+
+The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
+To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
+```
+ALERTS{alertname="your_alertname", alertstate="firing"}
+```
+Execute the query against storage which was used for `-remoteWrite.url` during the `replay`.
+
+### Additional configuration
+
+There are following non-required `replay` flags:
+
+* `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
+In two words, it affects the max time range for every `/query_range` request. The higher the value,
+the fewer requests will be issued during `replay`.
+* `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
+per rule before giving up.
+* `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
+(rules which depend on each other) rules. It is expected, that remote storage will be able to persist
+previously accepted data during the delay, so data will be available for the subsequent queries.
+Keep it equal or bigger than `-remoteWrite.flushInterval`.
+
+See full description for these flags in `./vmalert --help`.
+
+### Limitations
+
+* Graphite engine isn't supported yet;
+* `query` template function is disabled for performance reasons (might be changed in future);
+
+
+## Monitoring
+
+`vmalert` exports various metrics in Prometheus exposition format at `http://vmalert-host:8880/metrics` page.
+We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
+metrics may be analyzed later.
+
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
+If you have suggestions for improvements or have found a bug - please open an issue on github or add
+a review to the dashboard.
 
 
 ## Configuration
 
+### Flags
+
+Pass `-help` to `vmalert` in order to see the full list of supported
+command-line flags with their descriptions.
+
 The shortlist of configuration flags is the following:
 ```
   -datasource.appendTypePrefix
-    	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to VMSelect URL.
+    	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
   -datasource.basicAuth.password string
     	Optional basic auth password for -datasource.url
+  -datasource.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.bearerToken string
+    	Optional bearer auth token to use for -datasource.url.
+  -datasource.bearerTokenFile string
+    	Optional path to bearer token file to use for -datasource.url.
   -datasource.lookback duration
-    	Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+    	Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
   -datasource.maxIdleConnections int
-    	Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
+    	Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
   -datasource.queryStep duration
-    	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.
+    	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+  -datasource.roundDigits int
+    	Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
   -datasource.tlsCAFile string
-    	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
+    	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
     	Optional path to client-side TLS certificate file to use when connecting to -datasource.url
   -datasource.tlsInsecureSkipVerify
@@ -216,15 +520,17 @@ The shortlist of configuration flags is the following:
   -datasource.tlsKeyFile string
     	Optional path to client-side TLS certificate key to use when connecting to -datasource.url
   -datasource.tlsServerName string
-    	Optional TLS server name to use for connections to -datasource.url. By default the server name from -datasource.url is used
+    	Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-    	Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
+    	VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+  -disableAlertgroupLabel
+    	Whether to disable adding group's name as label to generated alerts and time series.
   -dryRun -rule
     	Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -evaluationInterval duration
@@ -234,23 +540,23 @@ The shortlist of configuration flags is the following:
     	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
   -external.label array
     	Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -external.url string
     	External URL is used as alert's source for sent alerts to the notifier
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
     	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
     	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
     	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
@@ -260,7 +566,7 @@ The shortlist of configuration flags is the following:
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
     	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
@@ -270,44 +576,54 @@ The shortlist of configuration flags is the following:
   -loggerTimezone string
     	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
     	Optional basic auth password for -notifier.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.basicAuth.username array
     	Optional basic auth username for -notifier.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsCAFile array
     	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsCertFile array
     	Optional path to client-side TLS certificate file to use when connecting to -notifier.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsInsecureSkipVerify array
     	Whether to skip tls verification when connecting to -notifier.url
     	Supports array of values separated by comma or specified via multiple flags.
   -notifier.tlsKeyFile array
     	Optional path to client-side TLS certificate key to use when connecting to -notifier.url
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsServerName array
     	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
+    	Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -remoteRead.basicAuth.password string
     	Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteRead.url
   -remoteRead.basicAuth.username string
     	Optional basic auth username for -remoteRead.url
+  -remoteRead.bearerToken string
+    	Optional bearer auth token to use for -remoteRead.url.
+  -remoteRead.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteRead.url.
+  -remoteRead.disablePathAppend
+    	Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.
+  -remoteRead.ignoreRestoreErrors
+    	Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
   -remoteRead.lookback duration
     	Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
   -remoteRead.tlsCAFile string
@@ -321,13 +637,21 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
     	Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-    	Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+    	Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
   -remoteWrite.basicAuth.password string
     	Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteWrite.url
   -remoteWrite.basicAuth.username string
     	Optional basic auth username for -remoteWrite.url
+  -remoteWrite.bearerToken string
+    	Optional bearer auth token to use for -remoteWrite.url.
+  -remoteWrite.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteWrite.url.
   -remoteWrite.concurrency int
     	Defines number of writers for concurrent writing into remote querier (default 1)
+  -remoteWrite.disablePathAppend
+    	Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
   -remoteWrite.flushInterval duration
     	Defines interval of flushes to remote write endpoint (default 5s)
   -remoteWrite.maxBatchSize int
@@ -345,7 +669,17 @@ The shortlist of configuration flags is the following:
   -remoteWrite.tlsServerName string
     	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-    	Optional URL to Victoria Metrics or VMInsert where to persist alerts state and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428
+    	Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+  -replay.maxDatapointsPerQuery int
+    	Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
+  -replay.ruleRetryAttempts int
+    	Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
+  -replay.rulesDelay duration
+    	Delay between rules evaluation within the group. Could be important if there are chained rules inside of the groupand processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule.Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
+  -replay.timeFrom string
+    	The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+  -replay.timeTo string
+    	The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
     	Path to the file with alert rules.
     	Supports patterns. Flag can be specified multiple times.
@@ -354,7 +688,11 @@ The shortlist of configuration flags is the following:
     	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
     	absolute path to all .yaml files in root.
     	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
-    	Supports array of values separated by comma or specified via multiple flags.
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -rule.configCheckInterval duration
+    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+  -rule.maxResolveDuration duration
+    	Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
   -rule.validateExpressions
     	Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
@@ -362,18 +700,38 @@ The shortlist of configuration flags is the following:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
   -tlsKeyFile string
     	Path to file with TLS key. Used only if -tls is set
   -version
     	Show VictoriaMetrics version
 ```
 
-Pass `-help` to `vmalert` in order to see the full list of supported
-command-line flags with their descriptions.
+### Hot config reload
+`vmalert` supports "hot" config reload via the following methods:
+* send SIGHUP signal to `vmalert` process;
+* send GET request to `/-/reload` endpoint;
+* configure `-rule.configCheckInterval` flag for periodic reload
+on config change.
+
+### URL params
+
+To set additional URL params for `datasource.url`, `remoteWrite.url` or `remoteRead.url`
+just add them in address: `-datasource.url=http://localhost:8428?nocache=1`.
+
+To set additional URL params for specific [group of rules](#Groups) modify
+the `params` group:
+```yaml
+groups:
+  - name: TestGroup
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
+```
+Please note, `params` are used only for executing rules expressions (requests to `datasource.url`).
+If there would be a conflict between URL params set in `datasource.url` flag and params in group definition
+the latter will have higher priority.
 
-To reload configuration without `vmalert` restart send SIGHUP signal
-or send GET request to `/-/reload` endpoint.
 
 ## Contributing
 
@@ -390,7 +748,7 @@ It is recommended using
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.15.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
@@ -407,7 +765,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.15.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmalert/alerting.go

@@ -19,15 +19,18 @@ import (
 
 // AlertingRule is basic alert entity
 type AlertingRule struct {
-	Type        datasource.Type
-	RuleID      uint64
-	Name        string
-	Expr        string
-	For         time.Duration
-	Labels      map[string]string
-	Annotations map[string]string
-	GroupID     uint64
-	GroupName   string
+	Type         datasource.Type
+	RuleID       uint64
+	Name         string
+	Expr         string
+	For          time.Duration
+	Labels       map[string]string
+	Annotations  map[string]string
+	GroupID      uint64
+	GroupName    string
+	EvalInterval time.Duration
+
+	q datasource.Querier
 
 	// guard status fields
 	mu sync.RWMutex
@@ -39,6 +42,9 @@ type AlertingRule struct {
 	// resets on every successful Exec
 	// may be used as Health state
 	lastExecError error
+	// stores the number of samples returned during
+	// the last evaluation
+	lastExecSamples int
 
 	metrics *alertingRuleMetrics
 }
@@ -47,28 +53,35 @@ type alertingRuleMetrics struct {
 	errors  *gauge
 	pending *gauge
 	active  *gauge
+	samples *gauge
 }
 
-func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
+func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
-		Type:        cfg.Type,
-		RuleID:      cfg.ID,
-		Name:        cfg.Alert,
-		Expr:        cfg.Expr,
-		For:         cfg.For.Duration(),
-		Labels:      cfg.Labels,
-		Annotations: cfg.Annotations,
-		GroupID:     group.ID(),
-		GroupName:   group.Name,
-		alerts:      make(map[uint64]*notifier.Alert),
-		metrics:     &alertingRuleMetrics{},
+		Type:         group.Type,
+		RuleID:       cfg.ID,
+		Name:         cfg.Alert,
+		Expr:         cfg.Expr,
+		For:          cfg.For.Duration(),
+		Labels:       cfg.Labels,
+		Annotations:  cfg.Annotations,
+		GroupID:      group.ID(),
+		GroupName:    group.Name,
+		EvalInterval: group.Interval,
+		q: qb.BuildWithParams(datasource.QuerierParams{
+			DataSourceType:     &group.Type,
+			EvaluationInterval: group.Interval,
+			QueryParams:        group.Params,
+		}),
+		alerts:  make(map[uint64]*notifier.Alert),
+		metrics: &alertingRuleMetrics{},
 	}
 
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StatePending {
@@ -79,8 +92,8 @@ func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
 		})
 	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StateFiring {
@@ -89,15 +102,21 @@ func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_error{%s}`, labels),
+	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			if ar.lastExecError == nil {
 				return 0
 			}
 			return 1
 		})
+	ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			return float64(ar.lastExecSamples)
+		})
 	return ar
 }
 
@@ -106,6 +125,7 @@ func (ar *AlertingRule) Close() {
 	metrics.UnregisterMetric(ar.metrics.active.name)
 	metrics.UnregisterMetric(ar.metrics.pending.name)
 	metrics.UnregisterMetric(ar.metrics.errors.name)
+	metrics.UnregisterMetric(ar.metrics.samples.name)
 }
 
 // String implements Stringer interface
@@ -119,15 +139,77 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+// ExecRange executes alerting rule on the given time range similarly to Exec.
+// It doesn't update internal states of the Rule and meant to be used just
+// to get time series for backfilling.
+// It returns ALERT and ALERT_FOR_STATE time series as result.
+func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
+	series, err := ar.q.QueryRange(ctx, ar.Expr, start, end)
+	if err != nil {
+		return nil, err
+	}
+	var result []prompbmarshal.TimeSeries
+	qFn := func(query string) ([]datasource.Metric, error) {
+		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
+	}
+	for _, s := range series {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			s.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			s.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
+		// extra labels could contain templates, so we expand them first
+		labels, err := expandLabels(s, qFn, ar)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+		for k, v := range labels {
+			// apply extra labels to datasource
+			// so the hash key will be consistent on restore
+			s.SetLabel(k, v)
+		}
+		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
+		if err != nil {
+			return nil, fmt.Errorf("failed to create alert: %s", err)
+		}
+		if ar.For == 0 { // if alert is instant
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
+		}
+
+		// if alert with For > 0
+		prevT := time.Time{}
+		for i := range s.Values {
+			at := time.Unix(s.Timestamps[i], 0)
+			if at.Sub(prevT) > ar.EvalInterval {
+				// reset to Pending if there are gaps > EvalInterval between DPs
+				a.State = notifier.StatePending
+				a.Start = at
+			} else if at.Sub(a.Start) >= ar.For {
+				a.State = notifier.StateFiring
+			}
+			prevT = at
+			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+		}
+	}
+	return result, nil
+}
+
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := q.Query(ctx, ar.Expr, ar.Type)
+func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := ar.q.Query(ctx, ar.Expr)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
 	ar.lastExecError = err
 	ar.lastExecTime = time.Now()
+	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
@@ -139,10 +221,17 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 		}
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query, ar.Type) }
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			m.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			m.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(m, qFn, ar)
 		if err != nil {
@@ -161,9 +250,9 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
-			if a.Value != m.Value {
+			if a.Value != m.Values[0] {
 				// update Value field with latest value
-				a.Value = m.Value
+				a.Value = m.Values[0]
 				// and re-exec template since Value can be used
 				// in annotations
 				a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
@@ -201,10 +290,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 			alertsFired.Inc()
 		}
 	}
-	if series {
-		return ar.toTimeSeries(ar.lastExecTime), nil
-	}
-	return nil, nil
+	return ar.toTimeSeries(ar.lastExecTime.Unix()), nil
 }
 
 func expandLabels(m datasource.Metric, q notifier.QueryFn, ar *AlertingRule) (map[string]string, error) {
@@ -214,13 +300,13 @@ func expandLabels(m datasource.Metric, q notifier.QueryFn, ar *AlertingRule) (ma
 	}
 	tpl := notifier.AlertTplData{
 		Labels: metricLabels,
-		Value:  m.Value,
+		Value:  m.Values[0],
 		Expr:   ar.Expr,
 	}
 	return notifier.ExecTemplate(q, ar.Labels, tpl)
 }
 
-func (ar *AlertingRule) toTimeSeries(timestamp time.Time) []prompbmarshal.TimeSeries {
+func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
 		if a.State == notifier.StateInactive {
@@ -244,6 +330,8 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 	ar.For = nr.For
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
+	ar.EvalInterval = nr.EvalInterval
+	ar.q = nr.q
 	return nil
 }
 
@@ -271,13 +359,10 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notif
 		GroupID: ar.GroupID,
 		Name:    ar.Name,
 		Labels:  map[string]string{},
-		Value:   m.Value,
+		Value:   m.Values[0],
 		Start:   start,
 		Expr:    ar.Expr,
 	}
-	// label defined here to make override possible by
-	// time series labels.
-	a.Labels[alertGroupNameLabel] = ar.GroupName
 	for _, l := range m.Labels {
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
@@ -317,6 +402,7 @@ func (ar *AlertingRule) RuleAPI() APIAlertingRule {
 		Expression:  ar.Expr,
 		For:         ar.For.String(),
 		LastError:   lastErr,
+		LastSamples: ar.lastExecSamples,
 		LastExec:    ar.lastExecTime,
 		Labels:      ar.Labels,
 		Annotations: ar.Annotations,
@@ -335,10 +421,11 @@ func (ar *AlertingRule) AlertsAPI() []*APIAlert {
 }
 
 func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
-	return &APIAlert{
+	aa := &APIAlert{
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", a.ID),
 		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
 
 		Name:        a.Name,
 		Expression:  ar.Expr,
@@ -346,8 +433,13 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Annotations: a.Annotations,
 		State:       a.State.String(),
 		ActiveAt:    a.Start,
-		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
 }
 
 const (
@@ -362,43 +454,42 @@ const (
 	alertStateLabel = "alertstate"
 
 	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
 	alertGroupNameLabel = "alertgroup"
 )
 
 // alertToTimeSeries converts the given alert with the given timestamp to timeseries
-func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	tss = append(tss, alertToTimeSeries(a, timestamp))
 	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
 	return tss
 }
 
-func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertMetricName
-	labels[alertNameLabel] = name
 	labels[alertStateLabel] = a.State.String()
-	return newTimeSeries(1, labels, timestamp)
+	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
 // alertForToTimeSeries returns a timeseries that represents
 // state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	labels[alertNameLabel] = name
-	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores the state of active alerts basing on previously written time series.
 // Restore restores only Start field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
@@ -407,7 +498,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		return fmt.Errorf("querier is nil")
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query, ar.Type) }
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
 
 	// account for external labels in filter
 	var labelsFilter string
@@ -420,29 +511,19 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr, ar.Type)
+	qMetrics, err := q.Query(ctx, expr)
 	if err != nil {
 		return err
 	}
 
 	for _, m := range qMetrics {
-		labels := m.Labels
-		m.Labels = make([]datasource.Label, 0)
-		// drop all extra labels, so hash key will
-		// be identical to time series received in Exec
-		for _, l := range labels {
-			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
-				continue
-			}
-			m.Labels = append(m.Labels, l)
-		}
-
-		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0), qFn)
+		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = hash(m)
 		a.State = notifier.StatePending
+		a.Restored = true
 		ar.alerts[a.ID] = a
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
 	}

app/vmalert/alerting_test.go

@@ -24,11 +24,10 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 			newTestAlertingRule("instant", 0),
 			&notifier.Alert{State: notifier.StateFiring},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant",
-				}, timestamp),
+				}),
 			},
 		},
 		{
@@ -38,13 +37,12 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				"instance": "bar",
 			}},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant extra labels",
 					"job":           "foo",
 					"instance":      "bar",
-				}, timestamp),
+				}),
 			},
 		},
 		{
@@ -54,48 +52,47 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				"__name__":      "bar",
 			}},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant labels override",
-				}, timestamp),
+				}),
 			},
 		},
 		{
 			newTestAlertingRule("for", time.Second),
 			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "for",
-				}, timestamp),
-				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
-					"__name__":     alertForStateMetricName,
-					alertNameLabel: "for",
-				}, timestamp),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
 			},
 		},
 		{
 			newTestAlertingRule("for pending", 10*time.Second),
 			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StatePending.String(),
-					alertNameLabel:  "for pending",
-				}, timestamp),
-				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
-					"__name__":     alertForStateMetricName,
-					alertNameLabel: "for pending",
-				}, timestamp),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
 			},
 		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			tc.rule.alerts[tc.alert.ID] = tc.alert
-			tss := tc.rule.toTimeSeries(timestamp)
+			tss := tc.rule.toTimeSeries(timestamp.Unix())
 			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
 				t.Fatalf("timeseries missmatch: %s", err)
 			}
@@ -105,23 +102,27 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 
 func TestAlertingRule_Exec(t *testing.T) {
 	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
 	testCases := []struct {
 		rule      *AlertingRule
 		steps     [][]datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
+		expAlerts []testAlert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
 			[][]datasource.Metric{},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("empty labels", 0),
 			[][]datasource.Metric{
-				{datasource.Metric{}},
+				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 			},
-			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -129,8 +130,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -139,8 +140,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -150,8 +151,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -162,8 +163,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -175,7 +176,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -187,8 +188,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -200,10 +201,10 @@ func TestAlertingRule_Exec(t *testing.T) {
 					metricWithLabels(t, "name", "foo2"),
 				},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -216,9 +217,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
 			// 3: fire third alert, set second inactive, delete first one
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -226,8 +227,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -236,8 +237,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -248,7 +249,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset and delete pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
@@ -258,8 +259,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -271,8 +272,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -285,8 +286,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 	}
@@ -294,11 +295,12 @@ func TestAlertingRule_Exec(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
+			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO(), fq, false); err != nil {
+				if _, err := tc.rule.Exec(context.TODO()); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -307,7 +309,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
 				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
 			}
-			for key, exp := range tc.expAlerts {
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := ta.labels
+				labels = append(labels, alertNameLabel)
+				labels = append(labels, tc.rule.Name)
+				h := hash(metricWithLabels(t, labels...))
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
 				got, ok := tc.rule.alerts[key]
 				if !ok {
 					t.Fatalf("expected to have key %d", key)
@@ -320,6 +330,171 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}
 }
 
+func TestAlertingRule_ExecRange(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		data      []datasource.Metric
+		expAlerts []*notifier.Alert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[]datasource.Metric{},
+			nil,
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[]datasource.Metric{
+				{Values: []float64{1}, Timestamps: []int64{1}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[]datasource.Metric{
+				metricWithLabels(t, "name", "foo"),
+			},
+			[]*notifier.Alert{
+				{
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
+				},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing-on-range", 0),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, Start: time.Unix(3, 0)},
+				{State: notifier.StatePending, Start: time.Unix(5, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for-firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, Start: time.Unix(5, 0)},
+				{State: notifier.StateFiring, Start: time.Unix(5, 0)},
+				{State: notifier.StatePending, Start: time.Unix(20, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("multi-series-for=>pending=>pending=>firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, Start: time.Unix(1, 0)},
+				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				//
+				{State: notifier.StatePending, Start: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+				{State: notifier.StatePending, Start: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+			},
+		},
+		{
+			newTestRuleWithLabels("multi-series-firing", "source", "vm"),
+			[]datasource.Metric{
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				//
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_ExecRange"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.q = fq
+			tc.rule.GroupID = fakeGroup.ID()
+			fq.add(tc.data...)
+			gotTS, err := tc.rule.ExecRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			var expTS []prompbmarshal.TimeSeries
+			var j int
+			for _, series := range tc.data {
+				for _, timestamp := range series.Timestamps {
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
+					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
+					j++
+				}
+			}
+			if len(gotTS) != len(expTS) {
+				t.Fatalf("expected %d time series; got %d", len(expTS), len(gotTS))
+			}
+			for i := range expTS {
+				got, exp := gotTS[i], expTS[i]
+				if !reflect.DeepEqual(got, exp) {
+					t.Fatalf("%d: expected \n%v but got \n%v", i, exp, got)
+				}
+			}
+		})
+	}
+}
+
 func TestAlertingRule_Restore(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
@@ -331,7 +506,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 				),
 			},
 			map[uint64]*notifier.Alert{
@@ -344,7 +518,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
+					alertNameLabel, "metric labels",
 					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
@@ -352,6 +526,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(metricWithLabels(t,
+					alertNameLabel, "metric labels",
+					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				)): {State: notifier.StatePending,
@@ -363,7 +539,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
 					// extra labels set by rule
@@ -410,6 +585,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
 			tc.rule.GroupID = fakeGroup.ID()
+			tc.rule.q = fq
 			fq.add(tc.metrics...)
 			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
 				t.Fatalf("unexpected err: %s", err)
@@ -437,17 +613,18 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	fq := &fakeQuerier{}
 	ar := newTestAlertingRule("test", 0)
 	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO(), fq, false)
+	_, err := ar.Exec(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO(), fq, false)
+	_, err = ar.Exec(context.TODO())
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -456,7 +633,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO(), fq, false)
+	_, err = ar.Exec(context.TODO())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -478,20 +655,20 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "foo")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						alertGroupNameLabel: "",
-						"region":            "east",
-						"instance":          "foo",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "bar")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						alertGroupNameLabel: "",
-						"region":            "east",
-						"instance":          "bar",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
 					},
 				},
 			},
@@ -514,22 +691,22 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "foo")): {
 					Labels: map[string]string{
-						alertGroupNameLabel: "",
-						"instance":          "foo",
-						"region":            "east",
+						alertNameLabel: "override label",
+						"instance":     "foo",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "foo" for region east`,
 						"description": `It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "bar")): {
 					Labels: map[string]string{
-						alertGroupNameLabel: "",
-						"instance":          "bar",
-						"region":            "east",
+						alertNameLabel: "override label",
+						"instance":     "bar",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "bar" for region east`,
@@ -538,14 +715,53 @@ func TestAlertingRule_Template(t *testing.T) {
 				},
 			},
 		},
+		{
+			&AlertingRule{
+				Name:      "ExtraTemplating",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"name":     "alert_{{ $labels.alertname }}",
+					"group":    "group_{{ $labels.alertgroup }}",
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
+					"name", "alert_ExtraTemplating",
+					alertGroupNameLabel, "Testing",
+					"group", "group_Testing",
+					"instance", "foo")): {
+					Labels: map[string]string{
+						alertNameLabel:      "ExtraTemplating",
+						"name":              "alert_ExtraTemplating",
+						alertGroupNameLabel: "Testing",
+						"group":             "group_Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
+						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+					},
+				},
+			},
+		},
 	}
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
 			tc.rule.GroupID = fakeGroup.ID()
+			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO(), fq, false); err != nil {
+			if _, err := tc.rule.Exec(context.TODO()); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -575,5 +791,5 @@ func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 }
 
 func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
-	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
+	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor, EvalInterval: waitFor}
 }

app/vmalert/config/config.go

@@ -5,19 +5,18 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
+	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
-	"time"
+
+	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metricsql"
-	"gopkg.in/yaml.v2"
 )
 
 // Group contains list of Rules grouped into
@@ -25,13 +24,23 @@ import (
 type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
-	Name        string        `yaml:"name"`
-	Interval    time.Duration `yaml:"interval,omitempty"`
-	Rules       []Rule        `yaml:"rules"`
-	Concurrency int           `yaml:"concurrency"`
+	Name        string             `yaml:"name"`
+	Interval    utils.PromDuration `yaml:"interval"`
+	Rules       []Rule             `yaml:"rules"`
+	Concurrency int                `yaml:"concurrency"`
+	// ExtraFilterLabels is a list label filters applied to every rule
+	// request withing a group. Is compatible only with VM datasources.
+	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+	// DEPRECATED: use Params field instead
+	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	// It has priority over the external labels.
+	Labels map[string]string `yaml:"labels"`
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -51,12 +60,20 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if g.Type.Get() == "" {
 		g.Type.Set(datasource.NewPrometheusType())
 	}
-	// update rules with empty type.
-	for i, r := range g.Rules {
-		if r.Type.Get() == "" {
-			r.Type.Set(g.Type)
-			r.ID = HashRule(r)
-			g.Rules[i] = r
+
+	// backward compatibility with deprecated `ExtraFilterLabels` param
+	if len(g.ExtraFilterLabels) > 0 {
+		if g.Params == nil {
+			g.Params = url.Values{}
+		}
+		// Sort extraFilters for consistent order for query args across runs.
+		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
+		for k, v := range g.ExtraFilterLabels {
+			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(extraFilters)
+		for _, extraFilter := range extraFilters {
+			g.Params.Add("extra_label", extraFilter)
 		}
 	}
 
@@ -71,9 +88,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
-	if len(g.Rules) == 0 {
-		return fmt.Errorf("group %q can't contain no rules", g.Name)
-	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -92,9 +106,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
-			if r.Type.Get() != "" {
-				exprValidator = r.Type.ValidateExpr
-			}
 			if err := exprValidator(r.Expr); err != nil {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
@@ -115,54 +126,17 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Type        datasource.Type   `yaml:"type,omitempty"`
-	Record      string            `yaml:"record,omitempty"`
-	Alert       string            `yaml:"alert,omitempty"`
-	Expr        string            `yaml:"expr"`
-	For         PromDuration      `yaml:"for"`
-	Labels      map[string]string `yaml:"labels,omitempty"`
-	Annotations map[string]string `yaml:"annotations,omitempty"`
+	Record      string             `yaml:"record,omitempty"`
+	Alert       string             `yaml:"alert,omitempty"`
+	Expr        string             `yaml:"expr"`
+	For         utils.PromDuration `yaml:"for"`
+	Labels      map[string]string  `yaml:"labels,omitempty"`
+	Annotations map[string]string  `yaml:"annotations,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
 }
 
-// PromDuration is Prometheus duration.
-type PromDuration struct {
-	milliseconds int64
-}
-
-// NewPromDuration returns PromDuration for given d.
-func NewPromDuration(d time.Duration) PromDuration {
-	return PromDuration{
-		milliseconds: d.Milliseconds(),
-	}
-}
-
-// MarshalYAML implements yaml.Marshaler interface.
-func (pd PromDuration) MarshalYAML() (interface{}, error) {
-	return pd.Duration().String(), nil
-}
-
-// UnmarshalYAML implements yaml.Unmarshaler interface.
-func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var s string
-	if err := unmarshal(&s); err != nil {
-		return err
-	}
-	ms, err := metricsql.DurationValue(s, 0)
-	if err != nil {
-		return err
-	}
-	pd.milliseconds = ms
-	return nil
-}
-
-// Duration returns duration for pd.
-func (pd *PromDuration) Duration() time.Duration {
-	return time.Duration(pd.milliseconds) * time.Millisecond
-}
-
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
@@ -193,7 +167,6 @@ func HashRule(r Rule) uint64 {
 		h.Write([]byte("alerting"))
 		h.Write([]byte(r.Alert))
 	}
-	h.Write([]byte(r.Type.Get()))
 	kv := sortMap(r.Labels)
 	for _, i := range kv {
 		h.Write([]byte(i.key))
@@ -225,6 +198,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
+	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -244,6 +218,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
+			if len(g.ExtraFilterLabels) > 0 {
+				isExtraFilterLabelsUsed = true
+			}
 			groups = append(groups, g)
 		}
 	}
@@ -253,6 +230,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
+	if isExtraFilterLabelsUsed {
+		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
+	}
 	return groups, nil
 }
 

app/vmalert/config/config_test.go

@@ -7,11 +7,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
 
 func TestMain(m *testing.M) {
@@ -96,10 +96,6 @@ func TestGroup_Validate(t *testing.T) {
 			group:  &Group{},
 			expErr: "group name must be set",
 		},
-		{
-			group:  &Group{Name: "test"},
-			expErr: "contain no rules",
-		},
 		{
 			group: &Group{Name: "test",
 				Rules: []Rule{
@@ -264,11 +260,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						For:  PromDuration{milliseconds: 10},
+						For:  utils.NewPromDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -280,11 +275,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						For:  PromDuration{milliseconds: 10},
+						For:  utils.NewPromDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -348,7 +342,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: NewPromDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: utils.NewPromDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -437,7 +431,7 @@ rules:
 `)
 	})
 
-	t.Run("Ok, `for` must change cs", func(t *testing.T) {
+	t.Run("`for` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup
 rules:
@@ -446,10 +440,118 @@ rules:
     for: 5m
 `, `
 name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`interval` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+interval: 2s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+interval: 4s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`concurrency` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+concurrency: 2
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+concurrency: 16
 rules:
   - alert: ExampleAlertWithFor
     expr: sum by(job) (up == 1)
 `)
 	})
 
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+}
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+
+	t.Run("extra labels", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+  env: prod
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+
+	t.Run("extra labels and params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+params:
+  nocache: ["1"]
+  extra_label: ["env=prod"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
+	})
 }

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -1,13 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 2s
-    concurrency: 2
-    type: prometheus
-    rules:
-      - alert: up
-        expr: up == 0
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules-update1-good.rules

@@ -1,12 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 30s
-    type: graphite
-    rules:
-      - alert: up
-        expr: filterSeries(time('host.2',20),'>','0')
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: duplicatedGroupDiffFiles
+    labels:
+       dc: gcp
     rules:
       - alert: VMRows
         for: 5m

app/vmalert/config/testdata/rules-query-good.rules

@@ -0,0 +1,15 @@
+groups:
+  - name: alertmanager.rules
+    rules:
+      - alert: AlertmanagerConfigInconsistent
+        annotations:
+          message: |
+            The configuration of the instances of the Alertmanager cluster `{{ $labels.namespace }}/{{ $labels.service }}` are out of sync.
+            {{ range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query }}
+            Configuration hash for pod {{ .Labels.pod }} is "{{ printf "%.f" .Value }}"
+            {{ end }}
+        expr: |
+          count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="openshift-monitoring"})) != 1
+        for: 5m
+        labels:
+          severity: critical

app/vmalert/config/testdata/rules-replay-good.rules

@@ -0,0 +1,39 @@
+groups:
+  - name: ReplayGroup
+    interval: 1m
+    concurrency: 1
+    rules:
+      - record: type:vm_cache_entries:rate5m
+        expr: sum(rate(vm_cache_entries[5m])) by (type)
+        labels:
+          recording: true
+      - record: go_cgo_calls_count:rate5m
+        expr: rate(go_cgo_calls_count{job="vmdb"}[5m])
+        labels:
+          recording: true
+
+  - name: vmsingleReplay
+    interval: 30s
+    concurrency: 2
+    rules:
+      - alert: RequestErrorsToAPI
+        expr: increase(vm_http_request_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=35&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
+          description: "Requests to path {{ $labels.path }} are receiving errors.
+            Please verify if clients are sending correct requests."
+
+      - alert: TooManyLogs
+        expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) by (job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=67&var-instance={{ $labels.instance }}"
+          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
+           Worth to check logs for specific error messages."

app/vmalert/config/testdata/rules0-good.rules

@@ -1,5 +1,8 @@
 groups:
   - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
     rules:
       - alert: VMRows
         for: 10s

app/vmalert/config/testdata/rules2-good.rules

@@ -2,6 +2,11 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
+    extra_filter_labels: # deprecated param, use `params` instead
+        job: victoriametrics
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by(instance) > 1

app/vmalert/config/testdata/rules3-good.rules

@@ -21,10 +21,3 @@ groups:
         annotations:
           summary: Too high connection number for {{$labels.instance}}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-      - alert: HostDown
-        type: graphite
-        expr: filterSeries(sumSeries(host.receiver.interface.up),'last','=', 0)
-        for: 3m
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules4-good.rules

@@ -0,0 +1,8 @@
+groups:
+  - name: TestEmptyRules
+    interval: 2s
+    concurrency: 2
+    rules:
+
+  - name: TestNoRules
+    type: prometheus

app/vmalert/datasource/datasource.go

@@ -2,21 +2,33 @@ package datasource
 
 import (
 	"context"
+	"net/url"
+	"time"
 )
 
-// Querier interface wraps Query method which
-// executes given query and returns list of Metrics
-// as result
+// Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string, engine Type) ([]Metric, error)
+	Query(ctx context.Context, query string) ([]Metric, error)
+	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
+}
+
+// QuerierBuilder builds Querier with given params.
+type QuerierBuilder interface {
+	BuildWithParams(params QuerierParams) Querier
+}
+
+// QuerierParams params for Querier.
+type QuerierParams struct {
+	DataSourceType     *Type
+	EvaluationInterval time.Duration
+	QueryParams        url.Values
 }
 
 // Metric is the basic entity which should be return by datasource
-// It represents single data point with full list of labels
 type Metric struct {
-	Labels    []Label
-	Timestamp int64
-	Value     float64
+	Labels     []Label
+	Timestamps []int64
+	Values     []float64
 }
 
 // SetLabel adds or updates existing one label

app/vmalert/datasource/datasource_test.go

@@ -0,0 +1,18 @@
+package datasource
+
+import "testing"
+
+func TestMetric_Label(t *testing.T) {
+	m := &Metric{}
+
+	m.AddLabel("foo", "bar")
+	checkEqualString(t, "bar", m.Label("foo"))
+
+	m.SetLabel("foo", "baz")
+	checkEqualString(t, "baz", m.Label("foo"))
+
+	m.SetLabel("qux", "quux")
+	checkEqualString(t, "quux", m.Label("qux"))
+
+	checkEqualString(t, "", m.Label("non-existing"))
+}

app/vmalert/datasource/init.go

@@ -4,43 +4,79 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
 
 var (
-	addr = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
-		" E.g. http://127.0.0.1:8428")
-	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to VMSelect URL.")
-	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
+		"E.g. http://127.0.0.1:8428")
+	appendTypePrefix      = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
+	bearerToken           = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile       = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
 	tlsKeyFile            = flag.String("datasource.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -datasource.url")
-	tlsCAFile             = flag.String("datasource.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -datasource.url. "+
-		"By default system CA is used")
-	tlsServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
-		"By default the server name from -datasource.url is used")
+	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
+	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
-	lookBack = flag.Duration("datasource.lookback", 0, "Lookback defines how far to look into past when evaluating queries. "+
-		"For example, if datasource.lookback=5m then param \"time\" with value now()-5m will be added to every query.")
+	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
-		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query.")
-	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, "Defines the number of idle (keep-alive connections) to configured datasource."+
-		"Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state.")
+		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
+		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
+	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
+// Param represents an HTTP GET param
+type Param struct {
+	Key, Value string
+}
+
 // Init creates a Querier from provided flag values.
-func Init() (Querier, error) {
+// Provided extraParams will be added as GET params for
+// each request.
+func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
+
 	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
-	tr.MaxIdleConns = *maxIdleConnections
-	c := &http.Client{Transport: tr}
-	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, *lookBack, *queryStep, *appendTypePrefix, c), nil
+	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
+
+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
+	if *roundDigits > 0 {
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
+	}
+
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
+	return &VMStorage{
+		c:                &http.Client{Transport: tr},
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		appendTypePrefix: *appendTypePrefix,
+		lookBack:         *lookBack,
+		queryStep:        *queryStep,
+		dataSourceType:   NewPrometheusType(),
+		extraParams:      extraParams,
+	}, nil
 }

app/vmalert/datasource/type.go

@@ -7,9 +7,6 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-const graphiteType = "graphite"
-const prometheusType = "prometheus"
-
 // Type represents data source type
 type Type struct {
 	name string
@@ -17,12 +14,16 @@ type Type struct {
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
-	return Type{name: prometheusType}
+	return Type{
+		name: "prometheus",
+	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
-	return Type{name: graphiteType}
+	return Type{
+		name: "graphite",
+	}
 }
 
 // NewRawType returns datasource type from raw string
@@ -44,19 +45,19 @@ func (t *Type) Set(d Type) {
 // String implements String interface with default value.
 func (t Type) String() string {
 	if t.name == "" {
-		return prometheusType
+		return "prometheus"
 	}
 	return t.name
 }
 
 // ValidateExpr validates query expression with datasource ql.
 func (t *Type) ValidateExpr(expr string) error {
-	switch t.name {
-	case graphiteType:
+	switch t.String() {
+	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
 			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
 		}
-	case "", prometheusType:
+	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
@@ -72,12 +73,13 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
+	if s == "" {
+		s = "prometheus"
+	}
 	switch s {
-	case "":
-		s = prometheusType
-	case graphiteType, prometheusType:
+	case "graphite", "prometheus":
 	default:
-		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, prometheusType, graphiteType)
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
 	t.name = s
 	return nil

app/vmalert/datasource/vm.go

@@ -2,205 +2,157 @@ package datasource
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"strconv"
+	"net/url"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
-type response struct {
-	Status string `json:"status"`
-	Data   struct {
-		ResultType string `json:"resultType"`
-		Result     []struct {
-			Labels map[string]string `json:"metric"`
-			TV     [2]interface{}    `json:"value"`
-		} `json:"result"`
-	} `json:"data"`
-	ErrorType string `json:"errorType"`
-	Error     string `json:"error"`
-}
-
-func (r response) metrics() ([]Metric, error) {
-	var ms []Metric
-	var m Metric
-	var f float64
-	var err error
-	for i, res := range r.Data.Result {
-		f, err = strconv.ParseFloat(res.TV[1].(string), 64)
-		if err != nil {
-			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
-		}
-		m.Labels = nil
-		for k, v := range r.Data.Result[i].Labels {
-			m.AddLabel(k, v)
-		}
-		m.Timestamp = int64(res.TV[0].(float64))
-		m.Value = f
-		ms = append(ms, m)
-	}
-	return ms, nil
-}
-
-type graphiteResponse []graphiteResponseTarget
-
-type graphiteResponseTarget struct {
-	Target     string            `json:"target"`
-	Tags       map[string]string `json:"tags"`
-	DataPoints [][2]float64      `json:"datapoints"`
-}
-
-func (r graphiteResponse) metrics() []Metric {
-	var ms []Metric
-	for _, res := range r {
-		if len(res.DataPoints) < 1 {
-			continue
-		}
-		var m Metric
-		// add only last value to the result.
-		last := res.DataPoints[len(res.DataPoints)-1]
-		m.Value = last[0]
-		m.Timestamp = int64(last[1])
-		for k, v := range res.Tags {
-			m.AddLabel(k, v)
-		}
-		ms = append(ms, m)
-	}
-	return ms
-}
-
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
+	authCfg          *promauth.Config
 	datasourceURL    string
-	basicAuthUser    string
-	basicAuthPass    string
 	appendTypePrefix bool
 	lookBack         time.Duration
 	queryStep        time.Duration
+
+	dataSourceType     Type
+	evaluationInterval time.Duration
+	extraParams        url.Values
+	disablePathAppend  bool
 }
 
-const queryPath = "/api/v1/query"
-const graphitePath = "/render"
+// Clone makes clone of VMStorage, shares http client.
+func (s *VMStorage) Clone() *VMStorage {
+	return &VMStorage{
+		c:                 s.c,
+		authCfg:           s.authCfg,
+		datasourceURL:     s.datasourceURL,
+		lookBack:          s.lookBack,
+		queryStep:         s.queryStep,
+		appendTypePrefix:  s.appendTypePrefix,
+		dataSourceType:    s.dataSourceType,
+		disablePathAppend: s.disablePathAppend,
+	}
+}
 
-const prometheusPrefix = "/prometheus"
-const graphitePrefix = "/graphite"
+// ApplyParams - changes given querier params.
+func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
+	if params.DataSourceType != nil {
+		s.dataSourceType = *params.DataSourceType
+	}
+	s.evaluationInterval = params.EvaluationInterval
+	s.extraParams = params.QueryParams
+	return s
+}
+
+// BuildWithParams - implements interface.
+func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
+	return s.Clone().ApplyParams(params)
+}
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
 	return &VMStorage{
-		c:                c,
-		basicAuthUser:    basicAuthUser,
-		basicAuthPass:    basicAuthPass,
-		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix: appendTypePrefix,
-		lookBack:         lookBack,
-		queryStep:        queryStep,
+		c:                 c,
+		authCfg:           authCfg,
+		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix:  appendTypePrefix,
+		lookBack:          lookBack,
+		queryStep:         queryStep,
+		dataSourceType:    NewPrometheusType(),
+		disablePathAppend: disablePathAppend,
 	}
 }
 
-// Query reads metrics from datasource by given query and type
-func (s *VMStorage) Query(ctx context.Context, query string, dataSourceType Type) ([]Metric, error) {
-	switch dataSourceType.name {
-	case "", prometheusType:
-		return s.queryDataSource(ctx, query, s.setPrometheusReqParams, parsePrometheusResponse)
-	case graphiteType:
-		return s.queryDataSource(ctx, query, s.setGraphiteReqParams, parseGraphiteResponse)
+// Query executes the given query and returns parsed response
+func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+	req, err := s.newRequestPOST()
+	if err != nil {
+		return nil, err
+	}
+
+	ts := time.Now()
+	switch s.dataSourceType.String() {
+	case "prometheus":
+		s.setPrometheusInstantReqParams(req, query, ts)
+	case "graphite":
+		s.setGraphiteReqParams(req, query, ts)
 	default:
-		return nil, fmt.Errorf("engine not found: %q", dataSourceType)
+		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
 	}
+
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	parseFn := parsePrometheusResponse
+	if s.dataSourceType.name != "prometheus" {
+		parseFn = parseGraphiteResponse
+	}
+	return parseFn(req, resp)
 }
 
-func (s *VMStorage) queryDataSource(
-	ctx context.Context,
-	query string,
-	setReqParams func(r *http.Request, query string),
-	processResponse func(r *http.Request, resp *http.Response,
-	) ([]Metric, error)) ([]Metric, error) {
+// QueryRange executes the given query on the given time range.
+// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
+// Graphite type isn't supported.
+func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
+	if s.dataSourceType.name != "prometheus" {
+		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
+	}
+	req, err := s.newRequestPOST()
+	if err != nil {
+		return nil, err
+	}
+	if start.IsZero() {
+		return nil, fmt.Errorf("start param is missing")
+	}
+	if end.IsZero() {
+		return nil, fmt.Errorf("end param is missing")
+	}
+	s.setPrometheusRangeReqParams(req, query, start, end)
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	return parsePrometheusResponse(req, resp)
+}
+
+func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	resp, err := s.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
+	}
+	return resp, nil
+}
+
+func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	req, err := http.NewRequest("POST", s.datasourceURL, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
-	if s.basicAuthPass != "" {
-		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	req.Header.Set("Content-Type", "application/json")
+	if s.authCfg != nil {
+		if auth := s.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
-	setReqParams(req, query)
-	resp, err := s.c.Do(req.WithContext(ctx))
-	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err)
-	}
-	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != http.StatusOK {
-		body, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
-	}
-	return processResponse(req, resp)
-}
-
-func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
-	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
-	}
-	r.URL.Path += queryPath
-	q := r.URL.Query()
-	q.Set("query", query)
-	if s.lookBack > 0 {
-		lookBack := time.Now().Add(-s.lookBack)
-		q.Set("time", fmt.Sprintf("%d", lookBack.Unix()))
-	}
-	if s.queryStep > 0 {
-		q.Set("step", s.queryStep.String())
-	}
-	r.URL.RawQuery = q.Encode()
-}
-
-func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string) {
-	if s.appendTypePrefix {
-		r.URL.Path += graphitePrefix
-	}
-	r.URL.Path += graphitePath
-	q := r.URL.Query()
-	q.Set("format", "json")
-	q.Set("target", query)
-	from := "-5min"
-	if s.lookBack > 0 {
-		lookBack := time.Now().Add(-s.lookBack)
-		from = strconv.FormatInt(lookBack.Unix(), 10)
-	}
-	q.Set("from", from)
-	q.Set("until", "now")
-	r.URL.RawQuery = q.Encode()
-}
-
-const (
-	statusSuccess, statusError, rtVector = "success", "error", "vector"
-)
-
-func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
-	r := &response{}
-	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL, err)
-	}
-	if r.Status == statusError {
-		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
-	}
-	if r.Status != statusSuccess {
-		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
-	}
-	if r.Data.ResultType != rtVector {
-		return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
-	}
-	return r.metrics()
-}
-
-func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
-	r := &graphiteResponse{}
-	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL, err)
-	}
-	return r.metrics(), nil
+	return req, nil
 }

app/vmalert/datasource/vm_graphite_api.go

@@ -0,0 +1,75 @@
+package datasource
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+type graphiteResponse []graphiteResponseTarget
+
+type graphiteResponseTarget struct {
+	Target     string            `json:"target"`
+	Tags       map[string]string `json:"tags"`
+	DataPoints [][2]float64      `json:"datapoints"`
+}
+
+func (r graphiteResponse) metrics() []Metric {
+	var ms []Metric
+	for _, res := range r {
+		if len(res.DataPoints) < 1 {
+			continue
+		}
+		var m Metric
+		// add only last value to the result.
+		last := res.DataPoints[len(res.DataPoints)-1]
+		m.Values = append(m.Values, last[0])
+		m.Timestamps = append(m.Timestamps, int64(last[1]))
+		for k, v := range res.Tags {
+			m.AddLabel(k, v)
+		}
+		ms = append(ms, m)
+	}
+	return ms
+}
+
+func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
+	r := &graphiteResponse{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
+	}
+	return r.metrics(), nil
+}
+
+const (
+	graphitePath   = "/render"
+	graphitePrefix = "/graphite"
+)
+
+func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestamp time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += graphitePrefix
+	}
+	r.URL.Path += graphitePath
+	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
+	q.Set("format", "json")
+	q.Set("target", query)
+	from := "-5min"
+	if s.lookBack > 0 {
+		lookBack := timestamp.Add(-s.lookBack)
+		from = strconv.FormatInt(lookBack.Unix(), 10)
+	}
+	q.Set("from", from)
+	q.Set("until", "now")
+	r.URL.RawQuery = q.Encode()
+}

app/vmalert/datasource/vm_prom_api.go

@@ -0,0 +1,173 @@
+package datasource
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+type promResponse struct {
+	Status    string `json:"status"`
+	ErrorType string `json:"errorType"`
+	Error     string `json:"error"`
+	Data      struct {
+		ResultType string          `json:"resultType"`
+		Result     json.RawMessage `json:"result"`
+	} `json:"data"`
+}
+
+type promInstant struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TV     [2]interface{}    `json:"value"`
+	} `json:"result"`
+}
+
+type promRange struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TVs    [][2]interface{}  `json:"values"`
+	} `json:"result"`
+}
+
+func (r promInstant) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range r.Result {
+		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
+		}
+		var m Metric
+		for k, v := range r.Result[i].Labels {
+			m.AddLabel(k, v)
+		}
+		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
+		m.Values = append(m.Values, f)
+		result = append(result, m)
+	}
+	return result, nil
+}
+
+func (r promRange) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range r.Result {
+		var m Metric
+		for _, tv := range res.TVs {
+			f, err := strconv.ParseFloat(tv[1].(string), 64)
+			if err != nil {
+				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
+			}
+			m.Values = append(m.Values, f)
+			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
+		}
+		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
+			return nil, fmt.Errorf("metric %v contains no values", res)
+		}
+		m.Labels = nil
+		for k, v := range r.Result[i].Labels {
+			m.AddLabel(k, v)
+		}
+		result = append(result, m)
+	}
+	return result, nil
+}
+
+const (
+	statusSuccess, statusError = "success", "error"
+	rtVector, rtMatrix         = "vector", "matrix"
+)
+
+func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
+	r := &promResponse{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err)
+	}
+	if r.Status == statusError {
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
+	}
+	if r.Status != statusSuccess {
+		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
+	}
+	switch r.Data.ResultType {
+	case rtVector:
+		var pi promInstant
+		if err := json.Unmarshal(r.Data.Result, &pi.Result); err != nil {
+			return nil, fmt.Errorf("umarshal err %s; \n %#v", err, string(r.Data.Result))
+		}
+		return pi.metrics()
+	case rtMatrix:
+		var pr promRange
+		if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
+			return nil, err
+		}
+		return pr.metrics()
+	default:
+		return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
+	}
+}
+
+const (
+	prometheusInstantPath = "/api/v1/query"
+	prometheusRangePath   = "/api/v1/query_range"
+	prometheusPrefix      = "/prometheus"
+)
+
+func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += prometheusPrefix
+	}
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusInstantPath
+	}
+	q := r.URL.Query()
+	if s.lookBack > 0 {
+		timestamp = timestamp.Add(-s.lookBack)
+	}
+	if s.evaluationInterval > 0 {
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
+		timestamp = timestamp.Truncate(s.evaluationInterval)
+	}
+	q.Set("time", fmt.Sprintf("%d", timestamp.Unix()))
+	r.URL.RawQuery = q.Encode()
+	s.setPrometheusReqParams(r, query)
+}
+
+func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += prometheusPrefix
+	}
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusRangePath
+	}
+	q := r.URL.Query()
+	q.Add("start", fmt.Sprintf("%d", start.Unix()))
+	q.Add("end", fmt.Sprintf("%d", end.Unix()))
+	r.URL.RawQuery = q.Encode()
+	s.setPrometheusReqParams(r, query)
+}
+
+func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
+	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
+	q.Set("query", query)
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+	}
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
+	}
+	r.URL.RawQuery = q.Encode()
+}

app/vmalert/datasource/vm_test.go

@@ -2,22 +2,32 @@ package datasource
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"reflect"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
 	ctx           = context.Background()
 	basicAuthName = "foo"
 	basicAuthPass = "bar"
-	query         = "vm_rows"
-	queryRender   = "constantLine(10)"
+	baCfg         = &promauth.BasicAuthConfig{
+		Username: basicAuthName,
+		Password: promauth.NewSecret(basicAuthPass),
+	}
+	query       = "vm_rows"
+	queryRender = "constantLine(10)"
 )
 
-func TestVMSelectQuery(t *testing.T) {
+func TestVMInstantQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
 		t.Errorf("should not be called")
@@ -63,32 +73,141 @@ func TestVMSelectQuery(t *testing.T) {
 		case 5:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
 		case 6:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]}]}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests"},"value":[1583786140,"2000"]}]}}`))
 		}
 	})
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+
+	p := NewPrometheusType()
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected invalid response status error got nil")
 	}
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected response body error got nil")
 	}
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected error status got nil")
 	}
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected unknown status got nil")
 	}
-	if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil {
+	if _, err := pq.Query(ctx, query); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
 	}
-	m, err := am.Query(ctx, query, NewPrometheusType())
+	m, err := pq.Query(ctx, query)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 2 {
+		t.Fatalf("expected 2 metrics got %d in %+v", len(m), m)
+	}
+	expected := []Metric{
+		{
+			Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
+			Timestamps: []int64{1583786142},
+			Values:     []float64{13763},
+		},
+		{
+			Labels:     []Label{{Value: "vm_requests", Name: "__name__"}},
+			Timestamps: []int64{1583786140},
+			Values:     []float64{2000},
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m, expected)
+	}
+
+	g := NewGraphiteType()
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+
+	m, err = gq.Query(ctx, queryRender)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
+	}
+	exp := Metric{
+		Labels:     []Label{{Value: "constantLine(10)", Name: "name"}},
+		Timestamps: []int64{1611758403},
+		Values:     []float64{10},
+	}
+	if !reflect.DeepEqual(m[0], exp) {
+		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+	}
+}
+
+func TestVMRangeQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		startTS := r.URL.Query().Get("start")
+		if startTS == "" {
+			t.Errorf("expected 'start' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(startTS, 10, 64); err != nil {
+			t.Errorf("failed to parse 'start' query param: %s", err)
+		}
+		endTS := r.URL.Query().Get("end")
+		if endTS == "" {
+			t.Errorf("expected 'end' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(endTS, 10, 64); err != nil {
+			t.Errorf("failed to parse 'end' query param: %s", err)
+		}
+		switch c {
+		case 0:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+
+	p := NewPrometheusType()
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+
+	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
+	expectError(t, err, "is missing")
+
+	_, err = pq.QueryRange(ctx, query, time.Time{}, time.Now())
+	expectError(t, err, "is missing")
+
+	start, end := time.Now().Add(-time.Minute), time.Now()
+
+	m, err := pq.QueryRange(ctx, query, start, end)
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -96,32 +215,321 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
-		Labels:    []Label{{Value: "vm_rows", Name: "__name__"}},
-		Timestamp: 1583786142,
-		Value:     13763,
+		Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
+		Timestamps: []int64{1583786142},
+		Values:     []float64{13763},
 	}
-	if m[0].Timestamp != expected.Timestamp &&
-		m[0].Value != expected.Value &&
-		m[0].Labels[0].Value != expected.Labels[0].Value &&
-		m[0].Labels[0].Name != expected.Labels[0].Name {
+	if !reflect.DeepEqual(m[0], expected) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
-	m, err = am.Query(ctx, queryRender, NewGraphiteType())
+
+	g := NewGraphiteType()
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+
+	_, err = gq.QueryRange(ctx, queryRender, start, end)
+	expectError(t, err, "is not supported")
+}
+
+func TestRequestParams(t *testing.T) {
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
 	if err != nil {
-		t.Fatalf("unexpected %s", err)
+		t.Fatalf("unexpected: %s", err)
 	}
-	if len(m) != 1 {
-		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
+	query := "up"
+	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+	testCases := []struct {
+		name       string
+		queryRange bool
+		vm         *VMStorage
+		checkFn    func(t *testing.T, r *http.Request)
+	}{
+		{
+			"prometheus path",
+			false,
+			&VMStorage{
+				dataSourceType: NewPrometheusType(),
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusInstantPath, r.URL.Path)
+			},
+		},
+		{
+			"prometheus path with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
+		{
+			"prometheus prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   NewPrometheusType(),
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
+			},
+		},
+		{
+			"prometheus prefix with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
+		{
+			"prometheus range path",
+			true,
+			&VMStorage{
+				dataSourceType: NewPrometheusType(),
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusRangePath, r.URL.Path)
+			},
+		},
+		{
+			"prometheus range path with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
+		{
+			"prometheus range prefix",
+			true,
+			&VMStorage{
+				dataSourceType:   NewPrometheusType(),
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
+			},
+		},
+		{
+			"prometheus range prefix with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
+		{
+			"graphite path",
+			false,
+			&VMStorage{
+				dataSourceType: NewGraphiteType(),
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"graphite prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   NewGraphiteType(),
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"default params",
+			false,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"default range params",
+			true,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("end=%d&query=%s&start=%d", timestamp.Unix(), query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"basic auth",
+			false,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"basic auth range",
+			true,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"lookback",
+			false,
+			&VMStorage{
+				lookBack: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Add(-time.Minute).Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"evaluation interval",
+			false,
+			&VMStorage{
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"lookback + evaluation interval",
+			false,
+			&VMStorage{
+				lookBack:           time.Minute,
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				tt := timestamp.Add(-time.Minute)
+				tt = tt.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"step override",
+			false,
+			&VMStorage{
+				queryStep: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"step to seconds",
+			false,
+			&VMStorage{
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params range",
+			true,
+			&VMStorage{
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
+					timestamp.Unix(), query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params",
+			false,
+			&VMStorage{
+				dataSourceType: NewGraphiteType(),
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
 	}
-	expected = Metric{
-		Labels:    []Label{{Value: "constantLine(10)", Name: "name"}},
-		Timestamp: 1611758403,
-		Value:     10,
-	}
-	if m[0].Timestamp != expected.Timestamp &&
-		m[0].Value != expected.Value &&
-		m[0].Labels[0].Value != expected.Labels[0].Value &&
-		m[0].Labels[0].Name != expected.Labels[0].Name {
-		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := tc.vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			switch tc.vm.dataSourceType.String() {
+			case "prometheus":
+				if tc.queryRange {
+					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+				} else {
+					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
+				}
+			case "graphite":
+				tc.vm.setGraphiteReqParams(req, query, timestamp)
+			}
+			tc.checkFn(t, req)
+		})
+	}
+}
+
+func checkEqualString(t *testing.T, exp, got string) {
+	t.Helper()
+	if got != exp {
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
+	}
+}
+
+func expectError(t *testing.T, err error, exp string) {
+	t.Helper()
+	if err == nil {
+		t.Errorf("expected non-nil error")
+	}
+	if !strings.Contains(err.Error(), exp) {
+		t.Errorf("expected error %q to contain %q", err, exp)
 	}
 }

app/vmalert/group.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"net/url"
 	"sync"
 	"time"
 
@@ -27,6 +28,9 @@ type Group struct {
 	Concurrency int
 	Checksum    string
 
+	Labels map[string]string
+	Params url.Values
+
 	doneCh     chan struct{}
 	finishedCh chan struct{}
 	// channel accepts new Group obj
@@ -49,17 +53,37 @@ func newGroupMetrics(name, file string) *groupMetrics {
 	return m
 }
 
-func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string]string) *Group {
+// merges group rule labels into result map
+// set2 has priority over set1.
+func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
+	r := map[string]string{}
+	for k, v := range set1 {
+		r[k] = v
+	}
+	for k, v := range set2 {
+		if prevV, ok := r[k]; ok {
+			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+				k, prevV, groupName, ruleName, k, v)
+		}
+		r[k] = v
+	}
+	return r
+}
+
+func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
 		Type:        cfg.Type,
 		Name:        cfg.Name,
 		File:        cfg.File,
-		Interval:    cfg.Interval,
+		Interval:    cfg.Interval.Duration(),
 		Concurrency: cfg.Concurrency,
 		Checksum:    cfg.Checksum,
-		doneCh:      make(chan struct{}),
-		finishedCh:  make(chan struct{}),
-		updateCh:    make(chan *Group),
+		Params:      cfg.Params,
+		Labels:      cfg.Labels,
+
+		doneCh:     make(chan struct{}),
+		finishedCh: make(chan struct{}),
+		updateCh:   make(chan *Group),
 	}
 	g.metrics = newGroupMetrics(g.Name, g.File)
 	if g.Interval == 0 {
@@ -70,33 +94,39 @@ func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string
 	}
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
-		// override rule labels with external labels
-		for k, v := range labels {
-			if prevV, ok := r.Labels[k]; ok {
-				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
-					k, prevV, g.Name, r.Name(), k, v)
-			}
-			if r.Labels == nil {
-				r.Labels = map[string]string{}
-			}
-			r.Labels[k] = v
+		var extraLabels map[string]string
+		// apply external labels
+		if len(labels) > 0 {
+			extraLabels = labels
 		}
-		rules[i] = g.newRule(r)
+		// apply group labels, it has priority on external labels
+		if len(cfg.Labels) > 0 {
+			extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels)
+		}
+		// apply rules labels, it has priority on other labels
+		if len(extraLabels) > 0 {
+			r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels)
+		}
+
+		rules[i] = g.newRule(qb, r)
 	}
 	g.Rules = rules
 	return g
 }
 
-func (g *Group) newRule(rule config.Rule) Rule {
+func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
 	if rule.Alert != "" {
-		return newAlertingRule(g, rule)
+		return newAlertingRule(qb, g, rule)
 	}
-	return newRecordingRule(g, rule)
+	return newRecordingRule(qb, g, rule)
 }
 
 // ID return unique group ID that consists of
 // rules file and group name
 func (g *Group) ID() uint64 {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -106,7 +136,8 @@ func (g *Group) ID() uint64 {
 }
 
 // Restore restores alerts state for group rules
-func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
+func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
+	labels = mergeLabels(g.Name, "", labels, g.Labels)
 	for _, rule := range g.Rules {
 		rr, ok := rule.(*AlertingRule)
 		if !ok {
@@ -115,6 +146,9 @@ func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time
 		if rr.For < 1 {
 			continue
 		}
+		// ignore g.ExtraFilterLabels on purpose, so it
+		// won't affect the restore procedure.
+		q := qb.BuildWithParams(datasource.QuerierParams{})
 		if err := rr.Restore(ctx, q, lookback, labels); err != nil {
 			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
 		}
@@ -160,19 +194,18 @@ func (g *Group) updateWith(newGroup *Group) error {
 	for _, nr := range rulesRegistry {
 		newRules = append(newRules, nr)
 	}
+	// note that g.Interval is not updated here
+	// so the value can be compared later in
+	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
+	g.Params = newGroup.Params
+	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
 }
 
-var (
-	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
-	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
-	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
-)
-
 func (g *Group) close() {
 	if g.doneCh == nil {
 		return
@@ -189,12 +222,12 @@ func (g *Group) close() {
 
 var skipRandSleepOnGroupStart bool
 
-func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
 
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
 		if randSleep < sleepOffset {
 			randSleep += uint64(g.Interval)
@@ -213,7 +246,16 @@ func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []not
 	}
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{querier, nts, rw}
+	e := &executor{rw: rw}
+	for _, nt := range nts {
+		ent := eNotifier{
+			Notifier:         nt,
+			alertsSent:       getOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", nt.Addr())),
+			alertsSendErrors: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", nt.Addr())),
+		}
+		e.notifiers = append(e.notifiers, ent)
+	}
+
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -242,36 +284,48 @@ func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []not
 		case <-t.C:
 			g.metrics.iterationTotal.Inc()
 			iterationStart := time.Now()
-
-			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
-			for err := range errs {
-				if err != nil {
-					logger.Errorf("group %q: %s", g.Name, err)
+			if len(g.Rules) > 0 {
+				resolveDuration := getResolveDuration(g.Interval)
+				errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
+				for err := range errs {
+					if err != nil {
+						logger.Errorf("group %q: %s", g.Name, err)
+					}
 				}
 			}
-
 			g.metrics.iterationDuration.UpdateDuration(iterationStart)
 		}
 	}
 }
 
+// resolveDuration for alerts is equal to 3 interval evaluations
+// so in case if vmalert stops sending updates for some reason,
+// notifier could automatically resolve the alert.
+func getResolveDuration(groupInterval time.Duration) time.Duration {
+	resolveInterval := groupInterval * 3
+	if *maxResolveDuration > 0 && (resolveInterval > *maxResolveDuration) {
+		return *maxResolveDuration
+	}
+	return resolveInterval
+}
+
 type executor struct {
-	querier   datasource.Querier
-	notifiers []notifier.Notifier
+	notifiers []eNotifier
 	rw        *remotewrite.Client
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
-	res := make(chan error, len(rules))
-	var returnSeries bool
-	if e.rw != nil {
-		returnSeries = true
-	}
+type eNotifier struct {
+	notifier.Notifier
+	alertsSent       *counter
+	alertsSendErrors *counter
+}
 
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
+	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, returnSeries, interval)
+			res <- e.exec(ctx, rule, resolveDuration)
 		}
 		close(res)
 		return res
@@ -284,7 +338,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, returnSeries, interval)
+				res <- e.exec(ctx, r, resolveDuration)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -296,21 +350,18 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 }
 
 var (
-	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
-	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
-	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+	alertsFired = metrics.NewCounter(`vmalert_alerts_fired_total`)
+
+	execTotal  = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
 
 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, returnSeries bool, interval time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
 	execTotal.Inc()
-	execStart := time.Now()
-	defer func() {
-		execDuration.UpdateDuration(execStart)
-	}()
 
-	tss, err := rule.Exec(ctx, e.querier, returnSeries)
+	tss, err := rule.Exec(ctx)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
@@ -333,10 +384,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, returnSeries bool, inter
 	for _, a := range ar.alerts {
 		switch a.State {
 		case notifier.StateFiring:
-			// set End to execStart + 3 intervals
-			// so notifier can resolve it automatically if `vmalert`
-			// won't be able to send resolve for some reason
-			a.End = time.Now().Add(3 * interval)
+			a.End = time.Now().Add(resolveDuration)
 			alerts = append(alerts, *a)
 		case notifier.StateInactive:
 			// set End to execStart to notify
@@ -349,11 +397,11 @@ func (e *executor) exec(ctx context.Context, rule Rule, returnSeries bool, inter
 		return nil
 	}
 
-	alertsSent.Add(len(alerts))
 	errGr := new(utils.ErrGroup)
 	for _, nt := range e.notifiers {
+		nt.alertsSent.Add(len(alerts))
 		if err := nt.Send(ctx, alerts); err != nil {
-			alertsSendErrors.Inc()
+			nt.alertsSendErrors.Inc()
 			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
 		}
 	}

app/vmalert/group_test.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
 
 func init() {
@@ -32,7 +34,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   config.NewPromDuration(time.Second),
+				For:   utils.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -44,7 +46,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   config.NewPromDuration(time.Second),
+				For:   utils.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -110,15 +112,16 @@ func TestUpdateWith(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			g := &Group{Name: "test"}
+			qb := &fakeQuerier{}
 			for _, r := range tc.currentRules {
 				r.ID = config.HashRule(r)
-				g.Rules = append(g.Rules, g.newRule(r))
+				g.Rules = append(g.Rules, g.newRule(qb, r))
 			}
 
 			ng := &Group{Name: "test"}
 			for _, r := range tc.newRules {
 				r.ID = config.HashRule(r)
-				ng.Rules = append(ng.Rules, ng.newRule(r))
+				ng.Rules = append(ng.Rules, ng.newRule(qb, r))
 			}
 
 			err := g.updateWith(ng)
@@ -156,11 +159,11 @@ func TestGroupStart(t *testing.T) {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
 	const evalInterval = time.Millisecond
-	g := newGroup(groups[0], evalInterval, map[string]string{"cluster": "east-1"})
-	g.Concurrency = 2
-
-	fn := &fakeNotifier{}
 	fs := &fakeQuerier{}
+	fn := &fakeNotifier{}
+
+	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
+	g.Concurrency = 2
 
 	const inst1, inst2, job = "foo", "bar", "baz"
 	m1 := metricWithLabels(t, "instance", inst1, "job", job)
@@ -177,7 +180,14 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert1.Labels["label"] = "bar"
 	alert1.Labels["host"] = inst1
-	alert1.ID = hash(m1)
+	// add service labels
+	alert1.Labels[alertNameLabel] = alert1.Name
+	alert1.Labels[alertGroupNameLabel] = g.Name
+	var labels1 []string
+	for k, v := range alert1.Labels {
+		labels1 = append(labels1, k, v)
+	}
+	alert1.ID = hash(metricWithLabels(t, labels1...))
 
 	alert2, err := r.newAlert(m2, time.Now(), nil)
 	if err != nil {
@@ -189,13 +199,20 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert2.Labels["label"] = "bar"
 	alert2.Labels["host"] = inst2
-	alert2.ID = hash(m2)
+	// add service labels
+	alert2.Labels[alertNameLabel] = alert2.Name
+	alert2.Labels[alertGroupNameLabel] = g.Name
+	var labels2 []string
+	for k, v := range alert2.Labels {
+		labels2 = append(labels2, k, v)
+	}
+	alert2.ID = hash(metricWithLabels(t, labels2...))
 
 	finished := make(chan struct{})
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), fs, []notifier.Notifier{fn}, nil)
+		g.start(context.Background(), []notifier.Notifier{fn}, nil)
 		close(finished)
 	}()
 
@@ -221,3 +238,27 @@ func TestGroupStart(t *testing.T) {
 	g.close()
 	<-finished
 }
+
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 3 * time.Minute},
+		{3 * time.Minute, 0, 9 * time.Minute},
+		{time.Minute, 2 * time.Minute, 2 * time.Minute},
+		{0, 0, 0},
+	}
+	defaultResolveDuration := *maxResolveDuration
+	defer func() { *maxResolveDuration = defaultResolveDuration }()
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			*maxResolveDuration = tc.maxDuration
+			got := getResolveDuration(tc.groupInterval)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
+}

app/vmalert/helpers_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -38,7 +39,15 @@ func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
 	fq.Unlock()
 }
 
-func (fq *fakeQuerier) Query(_ context.Context, _ string, _ datasource.Type) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fq
+}
+
+func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
+	return fq.Query(ctx, q)
+}
+
+func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
@@ -54,6 +63,7 @@ type fakeNotifier struct {
 	alerts []notifier.Alert
 }
 
+func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
@@ -68,9 +78,16 @@ func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 }
 
 func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
+	return metricWithValuesAndLabels(t, []float64{value}, labels...)
+}
+
+func metricWithValuesAndLabels(t *testing.T, values []float64, labels ...string) datasource.Metric {
 	t.Helper()
 	m := metricWithLabels(t, labels...)
-	m.Value = value
+	m.Values = values
+	for i := range values {
+		m.Timestamps = append(m.Timestamps, int64(i))
+	}
 	return m
 }
 
@@ -79,7 +96,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	if len(labels) == 0 || len(labels)%2 != 0 {
 		t.Fatalf("expected to get even number of labels")
 	}
-	m := datasource.Metric{}
+	m := datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}
 	for i := 0; i < len(labels); i += 2 {
 		m.Labels = append(m.Labels, datasource.Label{
 			Name:  labels[i],
@@ -160,6 +177,9 @@ func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
 	if !reflect.DeepEqual(a.Labels, b.Labels) {
 		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
 	}
+	if a.Type.String() != b.Type.String() {
+		return fmt.Errorf("expected to have Type %#v; got %#v", a.Type.String(), b.Type.String())
+	}
 	return nil
 }
 
@@ -185,7 +205,8 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 			}*/
 		}
 		if len(expTS.Labels) != len(gotTS.Labels) {
-			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+			return fmt.Errorf("expected number of labels %d (%v); got %d (%v)",
+				len(expTS.Labels), expTS.Labels, len(gotTS.Labels), gotTS.Labels)
 		}
 		for i, exp := range expTS.Labels {
 			got := gotTS.Labels[i]

app/vmalert/main.go

@@ -34,11 +34,16 @@ Examples:
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
+	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes")
+
 	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
+		"which is by default equal to 3 evaluation intervals of the parent group.")
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
@@ -47,10 +52,15 @@ eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
+
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's name as label to generated alerts and time series.")
 
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
+var alertURLGeneratorFn notifier.AlertURLGenerator
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -64,42 +74,68 @@ func main() {
 		notifier.InitTemplateFunc(u)
 		groups, err := config.Parse(*rulePath, true, true)
 		if err != nil {
-			logger.Fatalf(err.Error())
+			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
 		if len(groups) == 0 {
 			logger.Fatalf("No rules for validation. Please specify path to file(s) with alerting and/or recording rules using `-rule` flag")
 		}
 		return
 	}
+
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("failed to init `external.url`: %s", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("failed to init `external.alert.source`: %s", err)
+	}
+
+	if *replayFrom != "" || *replayTo != "" {
+		rw, err := remotewrite.Init(context.Background())
+		if err != nil {
+			logger.Fatalf("failed to init remoteWrite: %s", err)
+		}
+		notifier.InitTemplateFunc(eu)
+		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		if err != nil {
+			logger.Fatalf("cannot parse configuration file: %s", err)
+		}
+		// prevent queries from caching and boundaries aligning
+		// when querying VictoriaMetrics datasource.
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
+		if err != nil {
+			logger.Fatalf("failed to init datasource: %s", err)
+		}
+		if err := replay(groupsCfg, q, rw); err != nil {
+			logger.Fatalf("replay failed: %s", err)
+		}
+		return
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
-	if err := manager.start(ctx, *rulePath, *validateTemplates, *validateExpressions); err != nil {
+
+	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
+	groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+	if err != nil {
+		logger.Fatalf("cannot parse configuration file: %s", err)
+	}
+
+	// Register SIGHUP handler for config re-read just before manager.start call.
+	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
+	if err := manager.start(ctx, groupsCfg); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
 
-	go func() {
-		// init reload metrics with positive values to improve alerting conditions
-		configSuccess.Set(1)
-		configTimestamp.Set(fasttime.UnixTimestamp())
-		sigHup := procutil.NewSighupChan()
-		for {
-			<-sigHup
-			configReloads.Inc()
-			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
-			if err := manager.update(ctx, *rulePath, *validateTemplates, *validateExpressions, false); err != nil {
-				configReloadErrors.Inc()
-				configSuccess.Set(0)
-				logger.Errorf("error while reloading rules: %s", err)
-				continue
-			}
-			configSuccess.Set(1)
-			configTimestamp.Set(fasttime.UnixTimestamp())
-			logger.Infof("Rules reloaded successfully from %q", *rulePath)
-		}
-	}()
+	go configReload(ctx, manager, groupsCfg, sighupCh)
 
 	rh := &requestHandler{m: manager}
 	go httpserver.Serve(*httpListenAddr, rh.handler)
@@ -121,29 +157,19 @@ var (
 )
 
 func newManager(ctx context.Context) (*manager, error) {
-	q, err := datasource.Init()
+	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.url`: %w", err)
-	}
-	notifier.InitTemplateFunc(eu)
-	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.alert.source`: %w", err)
-	}
-	nts, err := notifier.Init(aug)
+	nts, err := notifier.Init(alertURLGeneratorFn)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
-
 	manager := &manager{
-		groups:    make(map[uint64]*Group),
-		querier:   q,
-		notifiers: nts,
-		labels:    map[string]string{},
+		groups:         make(map[uint64]*Group),
+		querierBuilder: q,
+		notifiers:      nts,
+		labels:         map[string]string{},
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -218,7 +244,66 @@ func usage() {
 	const s = `
 vmalert processes alerts and recording rules.
 
-See the docs at https://victoriametrics.github.io/vmalert.html .
+See the docs at https://docs.victoriametrics.com/vmalert.html .
 `
 	flagutil.Usage(s)
 }
+
+func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
+	var configCheckCh <-chan time.Time
+	if *rulesCheckInterval > 0 {
+		ticker := time.NewTicker(*rulesCheckInterval)
+		configCheckCh = ticker.C
+		defer ticker.Stop()
+	}
+
+	// init reload metrics with positive values to improve alerting conditions
+	configSuccess.Set(1)
+	configTimestamp.Set(fasttime.UnixTimestamp())
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-sighupCh:
+			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
+			configReloads.Inc()
+		case <-configCheckCh:
+		}
+		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("cannot parse configuration file: %s", err)
+			continue
+		}
+		if configsEqual(newGroupsCfg, groupsCfg) {
+			// set success to 1 since previous reload
+			// could have been unsuccessful
+			configSuccess.Set(1)
+			// config didn't change - skip it
+			continue
+		}
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("error while reloading rules: %s", err)
+			continue
+		}
+		groupsCfg = newGroupsCfg
+		configSuccess.Set(1)
+		configTimestamp.Set(fasttime.UnixTimestamp())
+		logger.Infof("Rules reloaded successfully from %q", *rulePath)
+	}
+}
+
+func configsEqual(a, b []config.Group) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i].Checksum != b[i].Checksum {
+			return false
+		}
+	}
+	return true
+}

app/vmalert/main_test.go

@@ -1,12 +1,17 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
 func TestGetExternalURL(t *testing.T) {
@@ -51,3 +56,105 @@ func TestGetAlertURLGenerator(t *testing.T) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
+
+func TestConfigReload(t *testing.T) {
+	originalRulePath := *rulePath
+	defer func() {
+		*rulePath = originalRulePath
+	}()
+
+	const (
+		rules1 = `
+groups:
+  - name: group-1
+    rules:
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job) (up == 1)
+      - record: handler:requests:rate5m 
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`
+		rules2 = `
+groups:
+  - name: group-1
+    rules:
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job) (up == 1)
+  - name: group-2
+    rules:
+      - record: handler:requests:rate5m 
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`
+	)
+
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeToFile(t, f.Name(), rules1)
+
+	*rulesCheckInterval = 200 * time.Millisecond
+	*rulePath = []string{f.Name()}
+	ctx, cancel := context.WithCancel(context.Background())
+
+	m := &manager{
+		querierBuilder: &fakeQuerier{},
+		groups:         make(map[uint64]*Group),
+		labels:         map[string]string{},
+		notifiers:      []notifier.Notifier{&fakeNotifier{}},
+		rw:             &remotewrite.Client{},
+	}
+
+	syncCh := make(chan struct{})
+	sighupCh := procutil.NewSighupChan()
+	go func() {
+		configReload(ctx, m, nil, sighupCh)
+		close(syncCh)
+	}()
+
+	lenLocked := func(m *manager) int {
+		m.groupsMu.RLock()
+		defer m.groupsMu.RUnlock()
+		return len(m.groups)
+	}
+
+	time.Sleep(*rulesCheckInterval * 2)
+	groupsLen := lenLocked(m)
+	if groupsLen != 1 {
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), rules2)
+	time.Sleep(*rulesCheckInterval * 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 2 {
+		fmt.Println(m.groups)
+		t.Fatalf("expected to have exactly 2 groups loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), rules1)
+	procutil.SelfSIGHUP()
+	time.Sleep(*rulesCheckInterval / 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 1 {
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), `corrupted`)
+	procutil.SelfSIGHUP()
+	time.Sleep(*rulesCheckInterval / 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 1 { // should remain unchanged
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	cancel()
+	<-syncCh
+}
+
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	err := ioutil.WriteFile(file, []byte(b), 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

app/vmalert/manager.go

@@ -3,7 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
-	"strings"
+	"net/url"
+	"sort"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -15,11 +16,12 @@ import (
 
 // manager controls group states
 type manager struct {
-	querier   datasource.Querier
-	notifiers []notifier.Notifier
+	querierBuilder datasource.QuerierBuilder
+	notifiers      []notifier.Notifier
 
 	rw *remotewrite.Client
-	rr datasource.Querier
+	// remote read builder.
+	rr datasource.QuerierBuilder
 
 	wg     sync.WaitGroup
 	labels map[string]string
@@ -49,8 +51,8 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
 }
 
-func (m *manager) start(ctx context.Context, path []string, validateTpl, validateExpr bool) error {
-	return m.update(ctx, path, validateTpl, validateExpr, true)
+func (m *manager) start(ctx context.Context, groupsCfg []config.Group) error {
+	return m.update(ctx, groupsCfg, true)
 }
 
 func (m *manager) close() {
@@ -63,10 +65,13 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
+func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) error {
 	if restore && m.rr != nil {
 		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
 		if err != nil {
+			if !*remoteReadIgnoreRestoreErrors {
+				return fmt.Errorf("failed to restore state for group %q: %w", group.Name, err)
+			}
 			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
 		}
 	}
@@ -74,25 +79,39 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
 	m.wg.Add(1)
 	id := group.ID()
 	go func() {
-		group.start(ctx, m.querier, m.notifiers, m.rw)
+		group.start(ctx, m.notifiers, m.rw)
 		m.wg.Done()
 	}()
 	m.groups[id] = group
+	return nil
 }
 
-func (m *manager) update(ctx context.Context, path []string, validateTpl, validateExpr, restore bool) error {
-	logger.Infof("reading rules configuration file from %q", strings.Join(path, ";"))
-	groupsCfg, err := config.Parse(path, validateTpl, validateExpr)
-	if err != nil {
-		return fmt.Errorf("cannot parse configuration file: %w", err)
-	}
-
+func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
+	var rrPresent, arPresent bool
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
-		ng := newGroup(cfg, *evaluationInterval, m.labels)
+		for _, r := range cfg.Rules {
+			if rrPresent && arPresent {
+				continue
+			}
+			if r.Record != "" {
+				rrPresent = true
+			}
+			if r.Alert != "" {
+				arPresent = true
+			}
+		}
+		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	if rrPresent && m.rw == nil {
+		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
+	}
+	if arPresent && m.notifiers == nil {
+		return fmt.Errorf("config contains alerting rules but `-notifier.url` isn't set")
+	}
+
 	type updateItem struct {
 		old *Group
 		new *Group
@@ -116,7 +135,9 @@ func (m *manager) update(ctx context.Context, path []string, validateTpl, valida
 		}
 	}
 	for _, ng := range groupsRegistry {
-		m.startGroup(ctx, ng, restore)
+		if err := m.startGroup(ctx, ng, restore); err != nil {
+			return err
+		}
 	}
 	m.groupsMu.Unlock()
 
@@ -140,12 +161,15 @@ func (g *Group) toAPI() APIGroup {
 
 	ag := APIGroup{
 		// encode as string to avoid rounding
-		ID:          fmt.Sprintf("%d", g.ID()),
+		ID: fmt.Sprintf("%d", g.ID()),
+
 		Name:        g.Name,
 		Type:        g.Type.String(),
 		File:        g.File,
 		Interval:    g.Interval.String(),
 		Concurrency: g.Concurrency,
+		Params:      urlValuesToStrings(g.Params),
+		Labels:      g.Labels,
 	}
 	for _, r := range g.Rules {
 		switch v := r.(type) {
@@ -157,3 +181,24 @@ func (g *Group) toAPI() APIGroup {
 	}
 	return ag
 }
+
+func urlValuesToStrings(values url.Values) []string {
+	if len(values) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		params := values[k]
+		for _, v := range params {
+			res = append(res, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return res
+}

app/vmalert/manager_test.go

@@ -5,13 +5,15 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 )
 
 func TestMain(m *testing.M) {
@@ -25,9 +27,8 @@ func TestMain(m *testing.M) {
 // starting with empty rules folder
 func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
-	path := []string{"foo/bar"}
-	err := m.update(context.Background(), path, true, true, false)
-	if err != nil {
+	cfg := loadCfg(t, []string{"foo/bar"}, true, true)
+	if err := m.update(context.Background(), cfg, false); err != nil {
 		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
 	}
 }
@@ -37,9 +38,9 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
-		groups:    make(map[uint64]*Group),
-		querier:   &fakeQuerier{},
-		notifiers: []notifier.Notifier{&fakeNotifier{}},
+		groups:         make(map[uint64]*Group),
+		querierBuilder: &fakeQuerier{},
+		notifiers:      []notifier.Notifier{&fakeNotifier{}},
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -50,8 +51,11 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 		"config/testdata/rules1-good.rules",
 		"config/testdata/rules2-good.rules",
 	}
+	evalInterval := *evaluationInterval
+	defer func() { *evaluationInterval = evalInterval }()
 	*evaluationInterval = time.Millisecond
-	if err := m.start(context.Background(), []string{paths[0]}, true, true); err != nil {
+	cfg := loadCfg(t, []string{paths[0]}, true, true)
+	if err := m.start(context.Background(), cfg); err != nil {
 		t.Fatalf("failed to start: %s", err)
 	}
 
@@ -64,8 +68,11 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			defer wg.Done()
 			for i := 0; i < iterations; i++ {
 				rnd := rand.Intn(len(paths))
-				path := []string{paths[rnd]}
-				_ = m.update(context.Background(), path, true, true, false)
+				cfg, err := config.Parse([]string{paths[rnd]}, true, true)
+				if err != nil { // update can fail and this is expected
+					continue
+				}
+				_ = m.update(context.Background(), cfg, false)
 			}
 		}()
 	}
@@ -108,18 +115,6 @@ func TestManagerUpdate(t *testing.T) {
 			Name: "ExampleAlertAlwaysFiring",
 			Expr: "sum by(job) (up == 1)",
 		}
-		ExampleAlertGraphite = &AlertingRule{
-			Name: "up graphite",
-			Expr: "filterSeries(time('host.1',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
-		ExampleAlertGraphite2 = &AlertingRule{
-			Name: "up",
-			Expr: "filterSeries(time('host.2',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
 	)
 
 	testCases := []struct {
@@ -143,7 +138,7 @@ func TestManagerUpdate(t *testing.T) {
 							Name:   "VMRows",
 							Expr:   "vm_rows > 0",
 							For:    5 * time.Minute,
-							Labels: map[string]string{"label": "bar"},
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
 							Annotations: map[string]string{
 								"summary":     "{{ $value }}",
 								"description": "{{$labels}}",
@@ -221,35 +216,25 @@ func TestManagerUpdate(t *testing.T) {
 				},
 			},
 		},
-		{
-			name:       "update prometheus to graphite type",
-			initPath:   "config/testdata/dir/rules-update0-good.rules",
-			updatePath: "config/testdata/dir/rules-update1-good.rules",
-			want: []*Group{
-				{
-					File:     "config/testdata/dir/rules-update1-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     datasource.NewGraphiteType(),
-					Name:     "TestUpdateGroup",
-					Rules: []Rule{
-						ExampleAlertGraphite2,
-						ExampleAlertGraphite,
-					},
-				},
-			},
-		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), querier: &fakeQuerier{}}
-			path := []string{tc.initPath}
-			if err := m.update(ctx, path, true, true, false); err != nil {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      []notifier.Notifier{&fakeNotifier{}},
+			}
+
+			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
+			if err := m.update(ctx, cfgInit, false); err != nil {
 				t.Fatalf("failed to complete initial rules update: %s", err)
 			}
 
-			path = []string{tc.updatePath}
-			_ = m.update(ctx, path, true, true, false)
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, true, true)
+			if err == nil { // update can fail and that's expected
+				_ = m.update(ctx, cfgUpdate, false)
+			}
 			if len(tc.want) != len(m.groups) {
 				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
 			}
@@ -267,3 +252,84 @@ func TestManagerUpdate(t *testing.T) {
 		})
 	}
 }
+
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        *remotewrite.Client
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&fakeNotifier{}},
+			nil,
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      tc.notifiers,
+				rw:             tc.rw,
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
+}
+
+func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
+	t.Helper()
+	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg
+}

app/vmalert/notifier/alert.go

@@ -14,17 +14,28 @@ import (
 // Alert the triggered alert
 // TODO: Looks like alert name isn't unique
 type Alert struct {
-	GroupID     uint64
-	Name        string
-	Labels      map[string]string
+	// GroupID contains the ID of the parent rules group
+	GroupID uint64
+	// Name represents Alert name
+	Name string
+	// Labels is the list of label-value pairs attached to the Alert
+	Labels map[string]string
+	// Annotations is the list of annotations generated on Alert evaluation
 	Annotations map[string]string
-	State       AlertState
-
-	Expr  string
+	// State represents the current state of the Alert
+	State AlertState
+	// Expr contains expression that was executed to generate the Alert
+	Expr string
+	// Start defines the moment of time when Alert has triggered
 	Start time.Time
-	End   time.Time
+	// End defines the moment of time when Alert supposed to expire
+	End time.Time
+	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	ID    uint64
+	// ID is the unique identifer for the Alert
+	ID uint64
+	// Restored is true if Alert was restored after restart
+	Restored bool
 }
 
 // AlertState type indicates the Alert state
@@ -90,13 +101,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
 	for key, text := range annotations {
-		r[key] = text
 		buf.Reset()
 		builder.Reset()
 		builder.Grow(len(tplHeader) + len(text))
 		builder.WriteString(tplHeader)
 		builder.WriteString(text)
 		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
 		}

app/vmalert/notifier/alert_test.go

@@ -83,14 +83,16 @@ func TestAlert_ExecTemplate(t *testing.T) {
 					{Name: "foo", Value: "bar"},
 					{Name: "baz", Value: "qux"},
 				},
-				Value: 1,
+				Values:     []float64{1},
+				Timestamps: []int64{1},
 			},
 			{
 				Labels: []datasource.Label{
 					{Name: "foo", Value: "garply"},
 					{Name: "baz", Value: "fred"},
 				},
-				Value: 2,
+				Values:     []float64{2},
+				Timestamps: []int64{1},
 			},
 		}, nil
 	}

app/vmalert/notifier/alertmanager.go

@@ -12,6 +12,7 @@ import (
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
+	addr          string
 	alertURL      string
 	basicAuthUser string
 	basicAuthPass string
@@ -19,6 +20,9 @@ type AlertManager struct {
 	client        *http.Client
 }
 
+// Addr returns address where alerts are sent.
+func (am AlertManager) Addr() string { return am.addr }
+
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
@@ -28,7 +32,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
 	req = req.WithContext(ctx)
 	if am.basicAuthPass != "" {
 		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
@@ -57,9 +61,10 @@ const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
 func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
-	addr := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
+	url := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
 	return &AlertManager{
-		alertURL:      addr,
+		addr:          alertManagerURL,
+		alertURL:      url,
 		argFunc:       fn,
 		client:        c,
 		basicAuthUser: user,

app/vmalert/notifier/alertmanager_test.go

@@ -10,6 +10,14 @@ import (
 	"time"
 )
 
+func TestAlertManager_Addr(t *testing.T) {
+	const addr = "http://localhost"
+	am := NewAlertManager(addr, "", "", nil, nil)
+	if am.Addr() != addr {
+		t.Errorf("expected to have %q; got %q", addr, am.Addr())
+	}
+}
+
 func TestAlertManager_Send(t *testing.T) {
 	const baUser, baPass = "foo", "bar"
 	mux := http.NewServeMux()

app/vmalert/notifier/init.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
 	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 
@@ -24,10 +24,6 @@ var (
 
 // Init creates a Notifier object based on provided flags.
 func Init(gen AlertURLGenerator) ([]Notifier, error) {
-	if len(*addrs) == 0 {
-		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
-	}
-
 	var notifiers []Notifier
 	for i, addr := range *addrs {
 		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)

app/vmalert/notifier/notifier.go

@@ -2,7 +2,12 @@ package notifier
 
 import "context"
 
-// Notifier is common interface for alert manager provider
+// Notifier is a common interface for alert manager provider
 type Notifier interface {
+	// Send sends the given list of alerts.
+	// Returns an error if fails to send the alerts.
+	// Must unblock if the given ctx is cancelled.
 	Send(ctx context.Context, alerts []Alert) error
+	// Addr returns address where alerts are sent.
+	Addr() string
 }

app/vmalert/notifier/template_func.go

@@ -28,44 +28,73 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
+// metric is private copy of datasource.Metric,
+// it is used for templating annotations,
+// Labels as map simplifies templates evaluation.
+type metric struct {
+	Labels    map[string]string
+	Timestamp int64
+	Value     float64
+}
+
+// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
+func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
+	mss := make([]metric, 0, len(ms))
+	for _, m := range ms {
+		labelsMap := make(map[string]string, len(m.Labels))
+		for _, labelValue := range m.Labels {
+			labelsMap[labelValue.Name] = labelValue.Value
+		}
+		mss = append(mss, metric{
+			Labels:    labelsMap,
+			Timestamp: m.Timestamps[0],
+			Value:     m.Values[0]})
+	}
+	return mss
+}
+
 // QueryFn is used to wrap a call to datasource into simple-to-use function
 // for templating functions.
 type QueryFn func(query string) ([]datasource.Metric, error)
 
-func funcsWithQuery(query QueryFn) textTpl.FuncMap {
-	fm := make(textTpl.FuncMap)
-	for k, fn := range tmplFunc {
-		fm[k] = fn
-	}
-	fm["query"] = func(q string) ([]datasource.Metric, error) {
-		return query(q)
-	}
-	return fm
-}
-
 var tmplFunc textTpl.FuncMap
 
 // InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
 	tmplFunc = textTpl.FuncMap{
-		"args": func(args ...interface{}) map[string]interface{} {
-			result := make(map[string]interface{})
-			for i, a := range args {
-				result[fmt.Sprintf("arg%d", i)] = a
-			}
-			return result
-		},
+		/* Strings */
+
+		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
+		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
+		// so for instance $1 represents the text of the first submatch.
+		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
 		"reReplaceAll": func(pattern, repl, text string) string {
 			re := regexp.MustCompile(pattern)
 			return re.ReplaceAllString(text, repl)
 		},
-		"safeHtml": func(text string) htmlTpl.HTML {
-			return htmlTpl.HTML(text)
-		},
-		"match":   regexp.MatchString,
-		"title":   strings.Title,
+
+		// match reports whether the string s
+		// contains any match of the regular expression pattern.
+		// alias for https://golang.org/pkg/regexp/#MatchString
+		"match": regexp.MatchString,
+
+		// title returns a copy of the string s with all Unicode letters
+		// that begin words mapped to their Unicode title case.
+		// alias for https://golang.org/pkg/strings/#Title
+		"title": strings.Title,
+
+		// toUpper returns s with all Unicode letters mapped to their upper case.
+		// alias for https://golang.org/pkg/strings/#ToUpper
 		"toUpper": strings.ToUpper,
+
+		// toLower returns s with all Unicode letters mapped to their lower case.
+		// alias for https://golang.org/pkg/strings/#ToLower
 		"toLower": strings.ToLower,
+
+		/* Numbers */
+
+		// humanize converts given number to a human readable format
+		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
 		"humanize": func(v float64) string {
 			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v)
@@ -91,6 +120,8 @@ func InitTemplateFunc(externalURL *url.URL) {
 			}
 			return fmt.Sprintf("%.4g%s", v, prefix)
 		},
+
+		// humanize1024 converts given number to a human readable format with 1024 as base
 		"humanize1024": func(v float64) string {
 			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v)
@@ -105,6 +136,8 @@ func InitTemplateFunc(externalURL *url.URL) {
 			}
 			return fmt.Sprintf("%.4g%s", v, prefix)
 		},
+
+		// humanizeDuration converts given seconds to a human readable duration
 		"humanizeDuration": func(v float64) string {
 			if math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v)
@@ -145,9 +178,13 @@ func InitTemplateFunc(externalURL *url.URL) {
 			}
 			return fmt.Sprintf("%.4g%ss", v, prefix)
 		},
+
+		// humanizePercentage converts given ratio value to a fraction of 100
 		"humanizePercentage": func(v float64) string {
 			return fmt.Sprintf("%.4g%%", v*100)
 		},
+
+		// humanizeTimestamp converts given timestamp to a human readable time equivalent
 		"humanizeTimestamp": func(v float64) string {
 			if math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v)
@@ -155,48 +192,115 @@ func InitTemplateFunc(externalURL *url.URL) {
 			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
 			return fmt.Sprint(t)
 		},
-		"pathPrefix": func() string {
-			return externalURL.Path
-		},
+
+		/* URLs */
+
+		// externalURL returns value of `external.url` flag
 		"externalURL": func() string {
 			return externalURL.String()
 		},
+
+		// pathPrefix returns a Path segment from the URL value in `external.url` flag
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+
+		// pathEscape escapes the string so it can be safely placed inside a URL path segment,
+		// replacing special characters (including /) with %XX sequences as needed.
+		// alias for https://golang.org/pkg/net/url/#PathEscape
 		"pathEscape": func(u string) string {
 			return url.PathEscape(u)
 		},
+
+		// queryEscape escapes the string so it can be safely placed
+		// inside a URL query.
+		// alias for https://golang.org/pkg/net/url/#QueryEscape
 		"queryEscape": func(q string) string {
 			return url.QueryEscape(q)
 		},
+
+		// crlfEscape replaces new line chars to skip URL encoding.
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890
 		"crlfEscape": func(q string) string {
 			q = strings.Replace(q, "\n", `\n`, -1)
 			return strings.Replace(q, "\r", `\r`, -1)
 		},
+
+		// quotesEscape escapes quote char
 		"quotesEscape": func(q string) string {
 			return strings.Replace(q, `"`, `\"`, -1)
 		},
-		// query function supposed to be substituted at funcsWithQuery().
-		// it is present here only for validation purposes, when there is no
-		// provided datasource.
-		"query": func(q string) ([]datasource.Metric, error) {
+
+		// query executes the MetricsQL/PromQL query against
+		// configured `datasource.url` address.
+		// For example, {{ query "foo" | first | value }} will
+		// execute "/api/v1/query?query=foo" request and will return
+		// the first value in response.
+		"query": func(q string) ([]metric, error) {
+			// query function supposed to be substituted at funcsWithQuery().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
 			// return non-empty slice to pass validation with chained functions in template
 			// see issue #989 for details
-			return []datasource.Metric{{}}, nil
+			return []metric{{}}, nil
 		},
-		"first": func(metrics []datasource.Metric) (datasource.Metric, error) {
+
+		// first returns the first by order element from the given metrics list.
+		// usually used alongside with `query` template function.
+		"first": func(metrics []metric) (metric, error) {
 			if len(metrics) > 0 {
 				return metrics[0], nil
 			}
-			return datasource.Metric{}, errors.New("first() called on vector with no elements")
+			return metric{}, errors.New("first() called on vector with no elements")
 		},
-		"label": func(label string, m datasource.Metric) string {
-			return m.Label(label)
+
+		// label returns the value of the given label name for the given metric.
+		// usually used alongside with `query` template function.
+		"label": func(label string, m metric) string {
+			return m.Labels[label]
 		},
-		"value": func(m datasource.Metric) float64 {
+
+		// value returns the value of the given metric.
+		// usually used alongside with `query` template function.
+		"value": func(m metric) float64 {
 			return m.Value
 		},
+
+		/* Helpers */
+
+		// Converts a list of objects to a map with keys arg0, arg1 etc.
+		// This is intended to allow multiple arguments to be passed to templates.
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+
+		// safeHtml marks string as HTML not requiring auto-escaping.
+		"safeHtml": func(text string) htmlTpl.HTML {
+			return htmlTpl.HTML(text)
+		},
 	}
 }
 
+func funcsWithQuery(query QueryFn) textTpl.FuncMap {
+	fm := make(textTpl.FuncMap)
+	for k, fn := range tmplFunc {
+		fm[k] = fn
+	}
+	fm["query"] = func(q string) ([]metric, error) {
+		result, err := query(q)
+		if err != nil {
+			return nil, err
+		}
+		return datasourceMetricsToTemplateMetrics(result), nil
+	}
+	return fm
+}
+
 // Time is the number of milliseconds since the epoch
 // (1970-01-01 00:00 UTC) excluding leap seconds.
 type Time int64

app/vmalert/recording.go

@@ -3,8 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
-	"hash/fnv"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -25,6 +25,8 @@ type RecordingRule struct {
 	Labels  map[string]string
 	GroupID uint64
 
+	q datasource.Querier
+
 	// guard status fields
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
@@ -33,12 +35,16 @@ type RecordingRule struct {
 	// resets on every successful Exec
 	// may be used as Health state
 	lastExecError error
+	// stores the number of samples returned during
+	// the last evaluation
+	lastExecSamples int
 
 	metrics *recordingRuleMetrics
 }
 
 type recordingRuleMetrics struct {
-	errors *gauge
+	errors  *gauge
+	samples *gauge
 }
 
 // String implements Stringer interface
@@ -52,81 +58,117 @@ func (rr *RecordingRule) ID() uint64 {
 	return rr.RuleID
 }
 
-func newRecordingRule(group *Group, cfg config.Rule) *RecordingRule {
+func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
 	rr := &RecordingRule{
-		Type:    cfg.Type,
+		Type:    group.Type,
 		RuleID:  cfg.ID,
 		Name:    cfg.Record,
 		Expr:    cfg.Expr,
 		Labels:  cfg.Labels,
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
+		q: qb.BuildWithParams(datasource.QuerierParams{
+			DataSourceType:     &group.Type,
+			EvaluationInterval: group.Interval,
+			QueryParams:        group.Params,
+		}),
 	}
 
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
 	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
-			rr.mu.Lock()
-			defer rr.mu.Unlock()
+			rr.mu.RLock()
+			defer rr.mu.RUnlock()
 			if rr.lastExecError == nil {
 				return 0
 			}
 			return 1
 		})
+	rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			rr.mu.RLock()
+			defer rr.mu.RUnlock()
+			return float64(rr.lastExecSamples)
+		})
 	return rr
 }
 
 // Close unregisters rule metrics
 func (rr *RecordingRule) Close() {
 	metrics.UnregisterMetric(rr.metrics.errors.name)
+	metrics.UnregisterMetric(rr.metrics.samples.name)
 }
 
-// Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
-	if !series {
-		return nil, nil
-	}
-
-	qMetrics, err := q.Query(ctx, rr.Expr, rr.Type)
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
-
-	rr.lastExecTime = time.Now()
-	rr.lastExecError = err
+// ExecRange executes recording rule on the given time range similarly to Exec.
+// It doesn't update internal states of the Rule and meant to be used just
+// to get time series for backfilling.
+func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
+	series, err := rr.q.QueryRange(ctx, rr.Expr, start, end)
 	if err != nil {
-		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+		return nil, err
 	}
-
-	duplicates := make(map[uint64]prompbmarshal.TimeSeries, len(qMetrics))
+	duplicates := make(map[string]struct{}, len(series))
 	var tss []prompbmarshal.TimeSeries
-	for _, r := range qMetrics {
-		ts := rr.toTimeSeries(r, rr.lastExecTime)
-		h := hashTimeSeries(ts)
-		if _, ok := duplicates[h]; ok {
-			rr.lastExecError = errDuplicate
-			return nil, errDuplicate
+	for _, s := range series {
+		ts := rr.toTimeSeries(s)
+		key := stringifyLabels(ts)
+		if _, ok := duplicates[key]; ok {
+			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", s.Labels, key, errDuplicate)
 		}
-		duplicates[h] = ts
+		duplicates[key] = struct{}{}
 		tss = append(tss, ts)
 	}
 	return tss, nil
 }
 
-func hashTimeSeries(ts prompbmarshal.TimeSeries) uint64 {
-	hash := fnv.New64a()
-	labels := ts.Labels
-	sort.Slice(labels, func(i, j int) bool {
-		return labels[i].Name < labels[j].Name
-	})
-	for _, l := range labels {
-		hash.Write([]byte(l.Name))
-		hash.Write([]byte(l.Value))
-		hash.Write([]byte("\xff"))
+// Exec executes RecordingRule expression via the given Querier.
+func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := rr.q.Query(ctx, rr.Expr)
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+
+	rr.lastExecTime = time.Now()
+	rr.lastExecError = err
+	rr.lastExecSamples = len(qMetrics)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
 	}
-	return hash.Sum64()
+
+	duplicates := make(map[string]struct{}, len(qMetrics))
+	var tss []prompbmarshal.TimeSeries
+	for _, r := range qMetrics {
+		ts := rr.toTimeSeries(r)
+		key := stringifyLabels(ts)
+		if _, ok := duplicates[key]; ok {
+			rr.lastExecError = errDuplicate
+			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
+		}
+		duplicates[key] = struct{}{}
+		tss = append(tss, ts)
+	}
+	return tss, nil
 }
 
-func (rr *RecordingRule) toTimeSeries(m datasource.Metric, timestamp time.Time) prompbmarshal.TimeSeries {
+func stringifyLabels(ts prompbmarshal.TimeSeries) string {
+	labels := ts.Labels
+	if len(labels) > 1 {
+		sort.Slice(labels, func(i, j int) bool {
+			return labels[i].Name < labels[j].Name
+		})
+	}
+	b := strings.Builder{}
+	for i, l := range labels {
+		b.WriteString(l.Name)
+		b.WriteString("=")
+		b.WriteString(l.Value)
+		if i != len(labels)-1 {
+			b.WriteString(",")
+		}
+	}
+	return b.String()
+}
+
+func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for _, l := range m.Labels {
 		labels[l.Name] = l.Value
@@ -136,12 +178,10 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric, timestamp time.Time)
 	for k, v := range rr.Labels {
 		labels[k] = v
 	}
-	return newTimeSeries(m.Value, labels, timestamp)
+	return newTimeSeries(m.Values, m.Timestamps, labels)
 }
 
 // UpdateWith copies all significant fields.
-// alerts state isn't copied since
-// it should be updated in next 2 Execs
 func (rr *RecordingRule) UpdateWith(r Rule) error {
 	nr, ok := r.(*RecordingRule)
 	if !ok {
@@ -149,6 +189,7 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	}
 	rr.Expr = nr.Expr
 	rr.Labels = nr.Labels
+	rr.q = nr.q
 	return nil
 }
 
@@ -161,13 +202,14 @@ func (rr *RecordingRule) RuleAPI() APIRecordingRule {
 	}
 	return APIRecordingRule{
 		// encode as strings to avoid rounding
-		ID:         fmt.Sprintf("%d", rr.ID()),
-		GroupID:    fmt.Sprintf("%d", rr.GroupID),
-		Name:       rr.Name,
-		Type:       rr.Type.String(),
-		Expression: rr.Expr,
-		LastError:  lastErr,
-		LastExec:   rr.lastExecTime,
-		Labels:     rr.Labels,
+		ID:          fmt.Sprintf("%d", rr.ID()),
+		GroupID:     fmt.Sprintf("%d", rr.GroupID),
+		Name:        rr.Name,
+		Type:        rr.Type.String(),
+		Expression:  rr.Expr,
+		LastError:   lastErr,
+		LastSamples: rr.lastExecSamples,
+		LastExec:    rr.lastExecTime,
+		Labels:      rr.Labels,
 	}
 }

app/vmalert/recording_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestRecoridngRule_ToTimeSeries(t *testing.T) {
+func TestRecoridngRule_Exec(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -24,9 +24,9 @@ func TestRecoridngRule_ToTimeSeries(t *testing.T) {
 				"__name__", "bar",
 			)},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(10, map[string]string{
+				newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "foo",
-				}, timestamp),
+				}),
 			},
 		},
 		{
@@ -37,18 +37,18 @@ func TestRecoridngRule_ToTimeSeries(t *testing.T) {
 				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
 			},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(1, map[string]string{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "foobarbaz",
 					"job":      "foo",
-				}, timestamp),
-				newTimeSeries(2, map[string]string{
+				}),
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "foobarbaz",
 					"job":      "bar",
-				}, timestamp),
-				newTimeSeries(3, map[string]string{
+				}),
+				newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "foobarbaz",
 					"job":      "baz",
-				}, timestamp),
+				}),
 			},
 		},
 		{
@@ -59,16 +59,16 @@ func TestRecoridngRule_ToTimeSeries(t *testing.T) {
 				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
 			[]prompbmarshal.TimeSeries{
-				newTimeSeries(2, map[string]string{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "job:foo",
 					"job":      "foo",
 					"source":   "test",
-				}, timestamp),
-				newTimeSeries(1, map[string]string{
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__": "job:foo",
 					"job":      "bar",
 					"source":   "test",
-				}, timestamp),
+				}),
 			},
 		},
 	}
@@ -76,7 +76,8 @@ func TestRecoridngRule_ToTimeSeries(t *testing.T) {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
-			tss, err := tc.rule.Exec(context.TODO(), fq, true)
+			tc.rule.q = fq
+			tss, err := tc.rule.Exec(context.TODO())
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -87,7 +88,88 @@ func TestRecoridngRule_ToTimeSeries(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ToTimeSeriesNegative(t *testing.T) {
+func TestRecoridngRule_ExecRange(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValuesAndLabels(t, []float64{10, 20, 30},
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{10, 20, 30},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foo",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+				metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+				metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}),
+				newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}),
+				newTimeSeries([]float64{4, 5, 6},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foobarbaz",
+						"job":      "baz",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "job:foo", Labels: map[string]string{
+				"source": "test",
+			}},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "bar",
+					"source":   "test",
+				}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			fq.add(tc.metrics...)
+			tc.rule.q = fq
+			tss, err := tc.rule.ExecRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestRecoridngRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
 		"job": "test",
 	}}
@@ -95,8 +177,8 @@ func TestRecoridngRule_ToTimeSeriesNegative(t *testing.T) {
 	fq := &fakeQuerier{}
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-
-	_, err := rr.Exec(context.TODO(), fq, true)
+	rr.q = fq
+	_, err := rr.Exec(context.TODO())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -111,7 +193,7 @@ func TestRecoridngRule_ToTimeSeriesNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO(), fq, true)
+	_, err = rr.Exec(context.TODO())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remoteread/init.go

@@ -10,11 +10,15 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
-		" state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state."+
-		" E.g. http://127.0.0.1:8428")
+	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
+		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
+	bearerToken           = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile       = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
 	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
@@ -22,11 +26,12 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
 )
 
 // Init creates a Querier from provided flag values.
 // Returns nil if addr flag wasn't set.
-func Init() (datasource.Querier, error) {
+func Init() (datasource.QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, nil
 	}
@@ -34,6 +39,10 @@ func Init() (datasource.Querier, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
 }

app/vmalert/remotewrite/init.go

@@ -10,10 +10,14 @@ import (
 )
 
 var (
-	addr = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
-		" and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
-	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
+		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
+		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
+	bearerToken           = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile       = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -27,6 +31,7 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
+	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 )
 
 // Init creates Client object from given flags.
@@ -41,14 +46,19 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
 	return NewClient(ctx, Config{
-		Addr:          *addr,
-		Concurrency:   *concurrency,
-		MaxQueueSize:  *maxQueueSize,
-		MaxBatchSize:  *maxBatchSize,
-		FlushInterval: *flushInterval,
-		BasicAuthUser: *basicAuthUsername,
-		BasicAuthPass: *basicAuthPassword,
-		Transport:     t,
+		Addr:              *addr,
+		AuthCfg:           authCfg,
+		Concurrency:       *concurrency,
+		MaxQueueSize:      *maxQueueSize,
+		MaxBatchSize:      *maxBatchSize,
+		FlushInterval:     *flushInterval,
+		DisablePathAppend: *disablePathAppend,
+		Transport:         t,
 	})
 }

app/vmalert/remotewrite/remotewrite.go

@@ -6,26 +6,30 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 // Client is an asynchronous HTTP client for writing
 // timeseries via remote write protocol.
 type Client struct {
-	addr           string
-	c              *http.Client
-	input          chan prompbmarshal.TimeSeries
-	baUser, baPass string
-	flushInterval  time.Duration
-	maxBatchSize   int
-	maxQueueSize   int
+	addr              string
+	c                 *http.Client
+	authCfg           *promauth.Config
+	input             chan prompbmarshal.TimeSeries
+	flushInterval     time.Duration
+	maxBatchSize      int
+	maxQueueSize      int
+	disablePathAppend bool
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
@@ -34,10 +38,8 @@ type Client struct {
 // Config is config for remote write.
 type Config struct {
 	// Addr of remote storage
-	Addr string
-
-	BasicAuthUser string
-	BasicAuthPass string
+	Addr    string
+	AuthCfg *promauth.Config
 
 	// Concurrency defines number of readers that
 	// concurrently read from the queue and flush data
@@ -56,6 +58,8 @@ type Config struct {
 	WriteTimeout time.Duration
 	// Transport will be used by the underlying http.Client
 	Transport *http.Transport
+	// DisablePathAppend can be used to not automatically append '/api/v1/write' to the remote write url
+	DisablePathAppend bool
 }
 
 const (
@@ -89,24 +93,25 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Transport == nil {
 		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
 	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
-		baUser:        cfg.BasicAuthUser,
-		baPass:        cfg.BasicAuthPass,
-		flushInterval: cfg.FlushInterval,
-		maxBatchSize:  cfg.MaxBatchSize,
-		maxQueueSize:  cfg.MaxQueueSize,
-		doneCh:        make(chan struct{}),
-		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
-	}
-	cc := defaultConcurrency
-	if cfg.Concurrency > 0 {
-		cc = cfg.Concurrency
+		addr:              strings.TrimSuffix(cfg.Addr, "/"),
+		authCfg:           cfg.AuthCfg,
+		flushInterval:     cfg.FlushInterval,
+		maxBatchSize:      cfg.MaxBatchSize,
+		maxQueueSize:      cfg.MaxQueueSize,
+		doneCh:            make(chan struct{}),
+		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+		disablePathAppend: cfg.DisablePathAppend,
 	}
+
 	for i := 0; i < cc; i++ {
 		c.run(ctx)
 	}
@@ -179,10 +184,11 @@ func (c *Client) run(ctx context.Context) {
 }
 
 var (
-	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -193,6 +199,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		return
 	}
 	defer prompbmarshal.ResetWriteRequest(wr)
+	defer bufferFlushDuration.UpdateDuration(time.Now())
 
 	data, err := wr.Marshal()
 	if err != nil {
@@ -228,20 +235,24 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
-	if c.baPass != "" {
-		req.SetBasicAuth(c.baUser, c.baPass)
+	if c.authCfg != nil {
+		if auth := c.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
+	}
+	if !c.disablePathAppend {
+		req.URL.Path = path.Join(req.URL.Path, writePath)
 	}
-	req.URL.Path += writePath
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
-			req.URL, err, len(data), r.Size())
+			req.URL.Redacted(), err, len(data), r.Size())
 	}
 	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != http.StatusNoContent {
+	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
-			resp.StatusCode, req.URL, body)
+			resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return nil
 }

app/vmalert/replay.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cheggaaa/pb/v3"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+var (
+	replayFrom = flag.String("replay.timeFrom", "",
+		"The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'")
+	replayTo = flag.String("replay.timeTo", "",
+		"The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'")
+	replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
+		"Delay between rules evaluation within the group. Could be important if there are chained rules inside of the group"+
+			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
+			"Keep it equal or bigger than -remoteWrite.flushInterval.")
+	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
+		"Max number of data points expected in one request. The higher the value, the less requests will be made during replay.")
+	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
+		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+)
+
+func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw *remotewrite.Client) error {
+	if *replayMaxDatapoints < 1 {
+		return fmt.Errorf("replay.maxDatapointsPerQuery can't be lower than 1")
+	}
+	tFrom, err := time.Parse(time.RFC3339, *replayFrom)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q: %s", *replayFrom, err)
+	}
+	tTo, err := time.Parse(time.RFC3339, *replayTo)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q: %s", *replayTo, err)
+	}
+	if !tTo.After(tFrom) {
+		return fmt.Errorf("replay.timeTo must be bigger than replay.timeFrom")
+	}
+	labels := make(map[string]string)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	fmt.Printf("Replay mode:"+
+		"\nfrom: \t%v "+
+		"\nto: \t%v "+
+		"\nmax data points per request: %d\n",
+		tFrom, tTo, *replayMaxDatapoints)
+
+	var total int
+	for _, cfg := range groupsCfg {
+		ng := newGroup(cfg, qb, *evaluationInterval, labels)
+		total += ng.replay(tFrom, tTo, rw)
+	}
+	logger.Infof("replay finished! Imported %d samples", total)
+	if rw != nil {
+		return rw.Close()
+	}
+	return nil
+}
+
+func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
+	var total int
+	step := g.Interval * time.Duration(*replayMaxDatapoints)
+	ri := rangeIterator{start: start, end: end, step: step}
+	iterations := int(end.Sub(start)/step) + 1
+	fmt.Printf("\nGroup %q"+
+		"\ninterval: \t%v"+
+		"\nrequests to make: \t%d"+
+		"\nmax range per request: \t%v\n",
+		g.Name, g.Interval, iterations, step)
+	for _, rule := range g.Rules {
+		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
+		bar := pb.StartNew(iterations)
+		ri.reset()
+		for ri.next() {
+			n, err := replayRule(rule, ri.s, ri.e, rw)
+			if err != nil {
+				logger.Fatalf("rule %q: %s", rule, err)
+			}
+			total += n
+			bar.Increment()
+		}
+		bar.Finish()
+		// sleep to let remote storage to flush data on-disk
+		// so chained rules could be calculated correctly
+		time.Sleep(*replayRulesDelay)
+	}
+	return total
+}
+
+func replayRule(rule Rule, start, end time.Time, rw *remotewrite.Client) (int, error) {
+	var err error
+	var tss []prompbmarshal.TimeSeries
+	for i := 0; i < *replayRuleRetryAttempts; i++ {
+		tss, err = rule.ExecRange(context.Background(), start, end)
+		if err == nil {
+			break
+		}
+		logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, rule, err)
+		time.Sleep(time.Second)
+	}
+	if err != nil { // means all attempts failed
+		return 0, err
+	}
+	if len(tss) < 1 {
+		return 0, nil
+	}
+	var n int
+	for _, ts := range tss {
+		if err := rw.Push(ts); err != nil {
+			return n, fmt.Errorf("remote write failure: %s", err)
+		}
+		n += len(ts.Samples)
+	}
+	return n, nil
+}
+
+type rangeIterator struct {
+	step       time.Duration
+	start, end time.Time
+
+	iter int
+	s, e time.Time
+}
+
+func (ri *rangeIterator) reset() {
+	ri.iter = 0
+	ri.s, ri.e = time.Time{}, time.Time{}
+}
+
+func (ri *rangeIterator) next() bool {
+	ri.s = ri.start.Add(ri.step * time.Duration(ri.iter))
+	if !ri.end.After(ri.s) {
+		return false
+	}
+	ri.e = ri.s.Add(ri.step)
+	if ri.e.After(ri.end) {
+		ri.e = ri.end
+	}
+	ri.iter++
+	return true
+}

app/vmalert/replay_test.go

@@ -0,0 +1,250 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+type fakeReplayQuerier struct {
+	fakeQuerier
+	registry map[string]map[string]struct{}
+}
+
+func (fr *fakeReplayQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fr
+}
+
+func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) ([]datasource.Metric, error) {
+	key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
+	dps, ok := fr.registry[q]
+	if !ok {
+		return nil, fmt.Errorf("unexpected query received: %q", q)
+	}
+	_, ok = dps[key]
+	if !ok {
+		return nil, fmt.Errorf("unexpected time range received: %q", key)
+	}
+	delete(dps, key)
+	if len(fr.registry[q]) < 1 {
+		delete(fr.registry, q)
+	}
+	return nil, nil
+}
+
+func TestReplay(t *testing.T) {
+	testCases := []struct {
+		name     string
+		from, to string
+		maxDP    int
+		cfg      []config.Group
+		qb       *fakeReplayQuerier
+	}{
+		{
+			name:  "one rule + one response",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:00.000Z",
+			maxDP: 10,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {"12:00:00+12:02:00": {}},
+				},
+			},
+		},
+		{
+			name:  "one rule + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "datapoints per step",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T15:02:30.000Z",
+			maxDP: 60,
+			cfg: []config.Group{
+				{Interval: utils.NewPromDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+13:00:00": {},
+						"13:00:00+14:00:00": {},
+						"14:00:00+15:00:00": {},
+						"15:00:00+15:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple recording rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple alerting rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
+				{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up) > 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up) < 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+	}
+
+	from, to, maxDP := *replayFrom, *replayTo, *replayMaxDatapoints
+	retries, delay := *replayRuleRetryAttempts, *replayRulesDelay
+	defer func() {
+		*replayFrom, *replayTo = from, to
+		*replayMaxDatapoints, *replayRuleRetryAttempts = maxDP, retries
+		*replayRulesDelay = delay
+	}()
+
+	*replayRuleRetryAttempts = 1
+	*replayRulesDelay = time.Millisecond
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			*replayFrom = tc.from
+			*replayTo = tc.to
+			*replayMaxDatapoints = tc.maxDP
+			if err := replay(tc.cfg, tc.qb, nil); err != nil {
+				t.Fatalf("replay failed: %s", err)
+			}
+			if len(tc.qb.registry) > 0 {
+				t.Fatalf("not all requests were sent: %#v", tc.qb.registry)
+			}
+		})
+	}
+}
+
+func TestRangeIterator(t *testing.T) {
+	testCases := []struct {
+		ri     rangeIterator
+		result [][2]time.Time
+	}{
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  5 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
+				{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
+				{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
+				{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
+				{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
+				{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  45 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+				{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:12.000Z"),
+				end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
+				step:  time.Second,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
+				{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
+				{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
+				{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
+				{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			var j int
+			for tc.ri.next() {
+				if len(tc.result) < j+1 {
+					t.Fatalf("unexpected result for iterator on step %d: %v - %v",
+						j, tc.ri.s, tc.ri.e)
+				}
+				s, e := tc.ri.s, tc.ri.e
+				expS, expE := tc.result[j][0], tc.result[j][1]
+				if s != expS {
+					t.Fatalf("expected to get start=%v; got %v", expS, s)
+				}
+				if e != expE {
+					t.Fatalf("expected to get end=%v; got %v", expE, e)
+				}
+				j++
+			}
+		})
+	}
+}
+
+func parseTime(t *testing.T, s string) time.Time {
+	t.Helper()
+	tt, err := time.Parse("2006-01-02T15:04:05.000Z", s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tt
+}

app/vmalert/rule.go

@@ -3,22 +3,21 @@ package main
 import (
 	"context"
 	"errors"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"time"
 )
 
 // Rule represents alerting or recording rule
 // that has unique ID, can be Executed and
 // updated with other Rule.
 type Rule interface {
-	// Returns unique ID that may be used for
+	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
 	// Exec executes the rule with given context
-	// and Querier. If returnSeries is true, Exec
-	// may return TimeSeries as result of execution
-	Exec(ctx context.Context, q datasource.Querier, returnSeries bool) ([]prompbmarshal.TimeSeries, error)
+	Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error)
+	// ExecRange executes the rule on the given time range
+	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error

app/vmalert/tpl/footer.qtpl

@@ -0,0 +1,36 @@
+{% func Footer() %}
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+{% endfunc %}

app/vmalert/tpl/footer.qtpl.go

@@ -0,0 +1,86 @@
+// Code generated by qtc from "footer.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/footer.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/footer.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+func StreamFooter(qw422016 *qt422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:1
+	qw422016.N().S(`
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+`)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func WriteFooter(qq422016 qtio422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:36
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	StreamFooter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func Footer() string {
+//line app/vmalert/tpl/footer.qtpl:36
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/footer.qtpl:36
+	WriteFooter(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	return qs422016
+//line app/vmalert/tpl/footer.qtpl:36
+}

app/vmalert/tpl/header.qtpl

@@ -0,0 +1,43 @@
+{% func Header(title string, pages []NavItem) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    {%= PrintNavItems(title, pages) %}
+    <main class="px-2">
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -0,0 +1,107 @@
+// Code generated by qtc from "header.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/header.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/header.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:1
+	qw422016.N().S(`
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert`)
+//line app/vmalert/tpl/header.qtpl:5
+	if title != "" {
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.N().S(` - `)
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.E().S(title)
+//line app/vmalert/tpl/header.qtpl:5
+	}
+//line app/vmalert/tpl/header.qtpl:5
+	qw422016.N().S(`</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    `)
+//line app/vmalert/tpl/header.qtpl:41
+	StreamPrintNavItems(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:41
+	qw422016.N().S(`
+    <main class="px-2">
+`)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:43
+	StreamHeader(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func Header(title string, pages []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:43
+	WriteHeader(qb422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:43
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:43
+}

app/vmalert/tpl/nav.qtpl

@@ -0,0 +1,25 @@
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func PrintNavItems(current string, items []NavItem) %}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}
+
+

app/vmalert/tpl/nav.qtpl.go

@@ -0,0 +1,96 @@
+// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/nav.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/nav.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/nav.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/nav.qtpl:2
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/nav.qtpl:8
+func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:8
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/nav.qtpl:13
+	for _, item := range items {
+//line app/vmalert/tpl/nav.qtpl:13
+		qw422016.N().S(`
+                <li class="nav-item">
+                    <a class="nav-link`)
+//line app/vmalert/tpl/nav.qtpl:15
+		if current == item.Name {
+//line app/vmalert/tpl/nav.qtpl:15
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/nav.qtpl:15
+		}
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`" href="`)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.E().S(item.Url)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/nav.qtpl:19
+	}
+//line app/vmalert/tpl/nav.qtpl:19
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	StreamPrintNavItems(qw422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func PrintNavItems(current string, items []NavItem) string {
+//line app/vmalert/tpl/nav.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/nav.qtpl:23
+	WritePrintNavItems(qb422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	return qs422016
+//line app/vmalert/tpl/nav.qtpl:23
+}

app/vmalert/utils.go

@@ -7,17 +7,21 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
-	ts := prompbmarshal.TimeSeries{}
-	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
-		Value:     value,
-		Timestamp: timestamp.UnixNano() / 1e6,
-	})
+func newTimeSeries(values []float64, timestamps []int64, labels map[string]string) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{
+		Samples: make([]prompbmarshal.Sample, len(values)),
+	}
+	for i := range values {
+		ts.Samples[i] = prompbmarshal.Sample{
+			Value:     values[i],
+			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
+		}
+	}
 	keys := make([]string, 0, len(labels))
 	for k := range labels {
 		keys = append(keys, k)
 	}
-	sort.Strings(keys)
+	sort.Strings(keys) // make order deterministic
 	for _, key := range keys {
 		ts.Labels = append(ts.Labels, prompbmarshal.Label{
 			Name:  key,

app/vmalert/utils/auth.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+// AuthConfig returns promauth.Config based on the given params
+func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*promauth.Config, error) {
+	var baCfg *promauth.BasicAuthConfig
+	if baUser != "" || baPass != "" || baFile != "" {
+		baCfg = &promauth.BasicAuthConfig{
+			Username:     baUser,
+			Password:     promauth.NewSecret(baPass),
+			PasswordFile: baFile,
+		}
+	}
+	return promauth.NewConfig(".", nil, baCfg, bearerToken, bearerTokenFile, nil, nil)
+}

app/vmalert/utils/prom_duration.go

@@ -0,0 +1,43 @@
+package utils
+
+import (
+	"time"
+
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// PromDuration is Prometheus duration.
+type PromDuration struct {
+	milliseconds int64
+}
+
+// NewPromDuration returns PromDuration for given d.
+func NewPromDuration(d time.Duration) PromDuration {
+	return PromDuration{
+		milliseconds: d.Milliseconds(),
+	}
+}
+
+// MarshalYAML implements yaml.Marshaler interface.
+func (pd PromDuration) MarshalYAML() (interface{}, error) {
+	return pd.Duration().String(), nil
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler interface.
+func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	ms, err := metricsql.DurationValue(s, 0)
+	if err != nil {
+		return err
+	}
+	pd.milliseconds = ms
+	return nil
+}
+
+// Duration returns duration for pd.
+func (pd *PromDuration) Duration() time.Duration {
+	return time.Duration(pd.milliseconds) * time.Millisecond
+}

app/vmalert/utils/tls_test.go

@@ -13,6 +13,7 @@ func TestTLSConfig(t *testing.T) {
 	}
 	if tlsCfg == nil {
 		t.Errorf("expected tlsConfig to be set, got nil")
+		return
 	}
 	if tlsCfg.ServerName != serverName {
 		t.Errorf("unexpected ServerName, want %s, got %s", serverName, tlsCfg.ServerName)

app/vmalert/vmalert_cluster.excalidraw

@@ -0,0 +1,525 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 794,
+      "versionNonce": 1855937036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 659,
+      "versionNonce": 247957684,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347948032,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1670,
+      "versionNonce": 2021681972,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 750.3317260742188,
+      "y": 226.5509033203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348054411
+    },
+    {
+      "type": "text",
+      "version": 1311,
+      "versionNonce": 1283453068,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 759.2862243652344,
+      "y": 238.68240356445312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert:8480",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 897,
+      "versionNonce": 1983434892,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 61.744873046875,
+      "y": 224.9600830078125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638348077724
+    },
+    {
+      "type": "text",
+      "version": 719,
+      "versionNonce": 457402292,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 70.69937133789062,
+      "y": 237.33523559570312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect:8481",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1098,
+      "versionNonce": 1480603788,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 371.7434387207031,
+      "y": 398.50787353515625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 864,
+      "versionNonce": 1115813900,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 393.73809814453125,
+      "y": 410.5976257324219,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348009180,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2405,
+      "versionNonce": 959767732,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 556.6860961914062,
+      "y": 252.2582773408825,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 184.9195556640625,
+      "height": 1.6022679018915937,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.0344528515859526,
+        "gap": 10.57574462890625
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.039393828258510157,
+        "gap": 8.72607421875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.9195556640625,
+          -1.6022679018915937
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1173,
+      "versionNonce": 1248255756,
+      "isDeleted": false,
+      "id": "wRO0q9xKPHc8e8XPPsQWh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.0439244722469,
+      "y": 246.80533728741074,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 157.86774649373126,
+      "height": 0.1417938392881979,
+      "seed": 656189364,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.13736472619498497,
+        "gap": 16.306295254315614
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.013200835330936087,
+        "gap": 14.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -157.86774649373126,
+          0.1417938392881979
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 557,
+      "versionNonce": 995289780,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 552.4987182617188,
+      "y": 212.27996826171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 76,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348059525,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist alerts state\n\n\nand recording rules",
+      "baseline": 72,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 803,
+      "versionNonce": 1576507444,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 290.34130859375,
+      "y": 210.56927490234375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1471,
+      "versionNonce": 1361321140,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 484.18669893674246,
+      "y": 302.3424013553929,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 1.0484739253853945,
+      "height": 84.72775855671654,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348083348,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.010768924644894236,
+        "gap": 14.650750964767894
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.051051952959743775,
+        "gap": 11.437713623046818
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0484739253853945,
+          84.72775855671654
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 576,
+      "versionNonce": 2112088460,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 530.5612182617188,
+      "y": 318.60687255859375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348023735,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 1835119500,
+      "isDeleted": false,
+      "id": "ff5OkfgmkKLifS13_TFj3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 291.37474060058594,
+      "y": 261.4861297607422,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 264004620,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "restore state",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_ha.excalidraw

@@ -0,0 +1,911 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 906,
+      "versionNonce": 448716468,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 302.2676696777344,
+      "y": 275.59356689453125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "OTKoeHmKtqCxFArDbY-sP"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 762,
+      "versionNonce": 223660724,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 318.0538024902344,
+      "y": 296.6778106689453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2"
+      ],
+      "updated": 1638355102070,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert1",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 2067,
+      "versionNonce": 817347852,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 329.6426086425781,
+      "y": 90.3275146484375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355058507
+    },
+    {
+      "type": "text",
+      "version": 1717,
+      "versionNonce": 2040797452,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 349.01177978515625,
+      "y": 102.45901489257812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1553,
+      "versionNonce": 39381044,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 173.67257690429688,
+      "y": 470.5100402832031,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "OTKoeHmKtqCxFArDbY-sP",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 1309,
+      "versionNonce": 1112872844,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 189.667236328125,
+      "y": 482.59979248046875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager1:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 835,
+      "versionNonce": 2036483596,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 525.83837890625,
+      "y": 147.33470153808594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 777,
+      "versionNonce": 990468492,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 617.0690307617188,
+      "y": 376.9822692871094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638355227582,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1112,
+      "versionNonce": 999136652,
+      "isDeleted": false,
+      "id": "mIu-d0lmShCxzMLD5iA_p",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 492.29505920410156,
+      "y": 272.3052215576172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 756416780,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "S_dOHQrhGmu8SFJzobJK7",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "Ar-hcDLlzVSoTPs2MaywO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 970,
+      "versionNonce": 430760500,
+      "isDeleted": false,
+      "id": "ZqIR6SaLNDQl8s0zZbdnE",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 508.08119201660156,
+      "y": 293.38946533203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 1477404084,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355105020,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalertN",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "qZFezRGU4Chwxgvb2451t",
+      "type": "line",
+      "x": 449.93653869628906,
+      "y": 306.30010986328125,
+      "width": 19.48321533203125,
+      "height": 0.3865966796875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1833426828,
+      "version": 69,
+      "versionNonce": 871948300,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          19.48321533203125,
+          0.3865966796875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": null,
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": null
+    },
+    {
+      "type": "rectangle",
+      "version": 1649,
+      "versionNonce": 93981708,
+      "isDeleted": false,
+      "id": "gNHiZJKo0ap69ALDobtZ-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518.5596466064453,
+      "y": 471.4539489746094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 454422412,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "Ar-hcDLlzVSoTPs2MaywO",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 1412,
+      "versionNonce": 75038348,
+      "isDeleted": false,
+      "id": "O-zgjZBvt4RI1PrkBNQnb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 534.3148040771484,
+      "y": 483.543701171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1026268980,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanagerN:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "m9_BptFOFxbV2sS_xJDu2",
+      "type": "arrow",
+      "x": 378.57185562792466,
+      "y": 262.1596984863281,
+      "width": 79.96146539019026,
+      "height": 111.53411865234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 141224884,
+      "version": 521,
+      "versionNonce": 283254540,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          79.96146539019026,
+          -111.53411865234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.23996018586441184,
+        "gap": 13.433868408203125
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.14207100087207922,
+        "gap": 15.550811767578125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "S_dOHQrhGmu8SFJzobJK7",
+      "type": "arrow",
+      "x": 558.1990515577129,
+      "y": 263.4271240234375,
+      "width": 88.18940317574186,
+      "height": 114.15780639648438,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1004789812,
+      "version": 314,
+      "versionNonce": 1259171724,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355105019,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -88.18940317574186,
+          -114.15780639648438
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4315094049079832,
+        "gap": 8.878097534179688
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": 0.14840834302306677,
+        "gap": 14.194549560546875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "fsGFp4NW4JlrCdF0HR3uA",
+      "type": "arrow",
+      "x": 356.6613630516658,
+      "y": 354.95416259765625,
+      "width": 278.74351860741064,
+      "height": 106.90020751953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 712384692,
+      "version": 334,
+      "versionNonce": 266493324,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          278.74351860741064,
+          106.90020751953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.7721210277890177,
+        "gap": 7.228485107421875
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4493598001028791,
+        "gap": 9.599578857421875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "OTKoeHmKtqCxFArDbY-sP",
+      "type": "arrow",
+      "x": 361.02563221226757,
+      "y": 356.2252197265625,
+      "width": 95.3594006000182,
+      "height": 105.52130126953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1992252596,
+      "version": 466,
+      "versionNonce": 472553100,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355221749,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -95.3594006000182,
+          105.52130126953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.39325294425055324,
+        "gap": 8.499542236328125
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.400636314282374,
+        "gap": 8.763519287109375
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "XhPgLRBk-YhWAFcSQi9TJ",
+      "type": "arrow",
+      "x": 561.7189961327852,
+      "y": 349.238037109375,
+      "width": 266.271510370216,
+      "height": 112.383544921875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 588869260,
+      "version": 453,
+      "versionNonce": 584723212,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355150131,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -266.271510370216,
+          112.383544921875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": -0.7084007026732048,
+        "gap": 4.8007049560546875
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.4180430111580123,
+        "gap": 8.888458251953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "Ar-hcDLlzVSoTPs2MaywO",
+      "type": "arrow",
+      "x": 557.5999880535809,
+      "y": 352.71795654296875,
+      "width": 112.87813113656136,
+      "height": 106.3255615234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1508782476,
+      "version": 331,
+      "versionNonce": 1137561908,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355153683,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          112.87813113656136,
+          106.3255615234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4358123206211808,
+        "gap": 8.280624389648438
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4783744286829653,
+        "gap": 12.410430908203125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "WqHnv9_g3SdkLZuy4WSHa",
+      "type": "text",
+      "x": 186.5891876220703,
+      "y": 523.145263671875,
+      "width": 272,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1310147468,
+      "version": 281,
+      "versionNonce": 949157044,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "text": "Alertmanagers in cluster mode",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "id": "yA0kgvdF71wZbHJ7Cg2p8",
+      "type": "rectangle",
+      "x": 159.41685485839844,
+      "y": 440.7666015625,
+      "width": 625.9590148925781,
+      "height": 110.13494873046878,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1465986996,
+      "version": 366,
+      "versionNonce": 196040844,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939
+    },
+    {
+      "id": "D6fkQH1E_MFbCuL697ArO",
+      "type": "arrow",
+      "x": 422.4853057861328,
+      "y": 494.8779132311229,
+      "width": 90.5517578125,
+      "height": 0.12427004064892344,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 2025208204,
+      "version": 316,
+      "versionNonce": 2144102156,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          90.5517578125,
+          0.12427004064892344
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.08064204025505255,
+        "gap": 8.706283569335938
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": -0.05976220061952895,
+        "gap": 5.5225830078125
+      },
+      "startArrowhead": "arrow",
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 463,
+      "versionNonce": 2118914484,
+      "isDeleted": false,
+      "id": "p4EDvSKPqZzfM_SReybeq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 274.6086883544922,
+      "y": 56.77886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 394,
+      "height": 19,
+      "seed": 1214462348,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355074872,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VictoriaMetrics with deduplication enabled",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 712,
+      "versionNonce": 737833612,
+      "isDeleted": false,
+      "id": "YfNFeOucMo8BJk2P2JRex",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 229.77923583984375,
+      "y": 227.84378051757812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 448.5542297363281,
+      "height": 134.06329345703122,
+      "seed": 932207756,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355133244
+    },
+    {
+      "type": "text",
+      "version": 427,
+      "versionNonce": 1458527796,
+      "isDeleted": false,
+      "id": "oldM7Q_aJtpWd2jXQ0iKf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 238.9442901611328,
+      "y": 230.862548828125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 225,
+      "height": 38,
+      "seed": 2131899572,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355259358,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "vmalerts with identical \nconfigurations",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_multicluster.excalidraw

@@ -0,0 +1,915 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 791,
+      "versionNonce": 48874036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "pD9DcILMxa6GaR1U5YyMO",
+        "HPEwr85wL4IedW0AgdArp",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 658,
+      "versionNonce": 1653816076,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347796775,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 802,
+      "versionNonce": 995326644,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.05322265625,
+      "y": 228.65371704101562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 624,
+      "versionNonce": 707755700,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 640.7635803222656,
+      "y": 241.02886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701075,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 802,
+      "versionNonce": 1974403340,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 401.241943359375,
+      "y": 342.4627990722656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 178,
+      "height": 38,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347863144,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute aggregating\nrecording rules",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "jrFZeFNsxlss7IsEZpA-J",
+      "type": "rectangle",
+      "x": 594.1509857177734,
+      "y": 192.09194946289062,
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 48379060,
+      "version": 665,
+      "versionNonce": 617456652,
+      "isDeleted": false,
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347763716
+    },
+    {
+      "id": "6ibhLp94HJFIdfP5HCEv8",
+      "type": "text",
+      "x": 609.7205657958984,
+      "y": 199.81723022460938,
+      "width": 225,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1053164852,
+      "version": 256,
+      "versionNonce": 538595124,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347761092,
+      "text": "VM cluster with raw data",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "type": "rectangle",
+      "version": 914,
+      "versionNonce": 1576666164,
+      "isDeleted": false,
+      "id": "R5v-yZCVJ97BkJqr0Qb7H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 807.5664215087891,
+      "y": 287.8628387451172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1794212620,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "bZXA8PH9gYu-clotqJ4f7",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 725,
+      "versionNonce": 267455500,
+      "isDeleted": false,
+      "id": "pWRC_smX7TuOI8_8UrA4H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 840.0209197998047,
+      "y": 300.2379913330078,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 421856180,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 843,
+      "versionNonce": 1244193972,
+      "isDeleted": false,
+      "id": "EqROOfYulSPsZm7ovxfQN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 604.9091339111328,
+      "y": 345.2847442626953,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 2043521972,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347718537
+    },
+    {
+      "type": "text",
+      "version": 676,
+      "versionNonce": 143370892,
+      "isDeleted": false,
+      "id": "ddQH1nnmT7HbKW7Xmv4zx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 642.6199798583984,
+      "y": 357.90354919433594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 335223180,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "bZXA8PH9gYu-clotqJ4f7",
+      "type": "arrow",
+      "x": 785.2667708463345,
+      "y": 367.2342882272049,
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1153029388,
+      "version": 420,
+      "versionNonce": 1726025228,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347704644,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "EqROOfYulSPsZm7ovxfQN",
+        "focus": 0.5247890891198189,
+        "gap": 8.36404562660789
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": -0.6931056940383433,
+        "gap": 9.943033572650961
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "arrow",
+      "version": 528,
+      "versionNonce": 1908259468,
+      "isDeleted": false,
+      "id": "MGdu6GDIPNBAaEUr0Gt-a",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 781.4456641707259,
+      "y": 248.777857603323,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1695061516,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347737174,
+      "startBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 883,
+      "versionNonce": 845352076,
+      "isDeleted": false,
+      "id": "4pW8hvBu3bo1eMtFvg_gS",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 601.6520690917969,
+      "y": 473.7677536010742,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1678097076,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 703,
+      "versionNonce": 519371956,
+      "isDeleted": false,
+      "id": "w_i2hO06oLa0bWbyAfFzU",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 639.3624267578125,
+      "y": 486.14290618896484,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 340753036,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 745,
+      "versionNonce": 879581324,
+      "isDeleted": false,
+      "id": "U5U-67wL5fPwvzlxOAF5A",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 592.7498321533203,
+      "y": 437.2059860229492,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "seed": 912540724,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347776748
+    },
+    {
+      "type": "text",
+      "version": 345,
+      "versionNonce": 1628116148,
+      "isDeleted": false,
+      "id": "0IGVrvICMVeZp2RCaqTeP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 608.3194122314453,
+      "y": 444.93126678466797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 291,
+      "height": 19,
+      "seed": 68700428,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347784373,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VM cluster with aggregated data",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 996,
+      "versionNonce": 2112461580,
+      "isDeleted": false,
+      "id": "9QMf0HXPE1W4M9S-zMJVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 806.1652679443359,
+      "y": 532.9768753051758,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 523607476,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "9WBH34em4CdU2OwzqYIl5",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 804,
+      "versionNonce": 1660832052,
+      "isDeleted": false,
+      "id": "WOrOD5vn6EtMUQRJomt3-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 838.6197662353516,
+      "y": 545.3520278930664,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 1321420684,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 925,
+      "versionNonce": 2052807604,
+      "isDeleted": false,
+      "id": "4dtJZpXEUxSK3biwFG7vd",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.5079803466797,
+      "y": 590.3987808227539,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1738524468,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "9WBH34em4CdU2OwzqYIl5",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 756,
+      "versionNonce": 2040967820,
+      "isDeleted": false,
+      "id": "rAbyooo-X08-86qjoK0WR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 641.2188262939453,
+      "y": 603.0175857543945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 948598284,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "9WBH34em4CdU2OwzqYIl5"
+      ],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 660,
+      "versionNonce": 1560678196,
+      "isDeleted": false,
+      "id": "9WBH34em4CdU2OwzqYIl5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.8656172818814,
+      "y": 612.3483247872634,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "seed": 1141424308,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": 0.5247890891198189,
+        "gap": 8.364045626608004
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": -0.6931056940383404,
+        "gap": 9.943033572650847
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 768,
+      "versionNonce": 1653264948,
+      "isDeleted": false,
+      "id": "5W90eDBjtfZvkSuQTG0Iw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.0445106062728,
+      "y": 493.8918941633816,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1852298380,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4pW8hvBu3bo1eMtFvg_gS",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "type": "arrow",
+      "x": 423.70701599121094,
+      "y": 428.34434509277344,
+      "width": 179.54901123046875,
+      "height": 182.9937744140625,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 389863732,
+      "version": 47,
+      "versionNonce": 1364399028,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347805500,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          179.54901123046875,
+          -182.9937744140625
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "type": "arrow",
+      "x": 424.7585906982422,
+      "y": 441.12806701660156,
+      "width": 174.29449462890625,
+      "height": 170.56607055664062,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 981082124,
+      "version": 92,
+      "versionNonce": 1261546252,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347812431,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          174.29449462890625,
+          170.56607055664062
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": -0.8207814548026305,
+        "gap": 4.45489501953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 775,
+      "versionNonce": 95530764,
+      "isDeleted": false,
+      "id": "o-yIJ_WZzVubhbVODvhcC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 407.27012634277344,
+      "y": 489.33091735839844,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 775116812,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347824876,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_single.excalidraw

@@ -0,0 +1,354 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 795,
+      "versionNonce": 1195460364,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 660,
+      "versionNonce": 1034125236,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1690,
+      "versionNonce": 236788620,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 344.9837951660156,
+      "y": 64.64248657226562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348176044
+    },
+    {
+      "type": "text",
+      "version": 1331,
+      "versionNonce": 945335476,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 364.35296630859375,
+      "y": 76.77398681640625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1099,
+      "versionNonce": 671872012,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 360.6495666503906,
+      "y": 382.2536315917969,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 865,
+      "versionNonce": 635839156,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 382.64422607421875,
+      "y": 394.3433837890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2444,
+      "versionNonce": 361351692,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.0465703465111,
+      "y": 204.98379516601562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.7962088410123442,
+      "height": 86.86798095703125,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "gap": 10.57574462890625,
+        "focus": 0.0344528515859526
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "gap": 8.72607421875,
+        "focus": -0.039393828258510157
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.7962088410123442,
+          -86.86798095703125
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 640,
+      "versionNonce": 1892951180,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 500.609619140625,
+      "y": 134.11544799804688,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348163424,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1472,
+      "versionNonce": 768565516,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.5245361328125,
+      "y": 300.5478957313172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.3521007656264601,
+      "height": 70.26802223743277,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "startBinding": {
+        "elementId": "E9Run6wCm2chQ6JHrmc_y",
+        "focus": 1.1925203824459496,
+        "gap": 11.77154541015625
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.0441077573536454,
+        "gap": 11.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.3521007656264601,
+          70.26802223743277
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 577,
+      "versionNonce": 1646003636,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 498.29608154296875,
+      "y": 298.6573791503906,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/web.go

@@ -4,55 +4,80 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var (
+	once     = sync.Once{}
+	apiLinks [][2]string
+	navItems []tpl.NavItem
+)
+
+func initLinks() {
+	pathPrefix := httpserver.GetPathPrefix()
+	apiLinks = [][2]string{
+		{path.Join(pathPrefix, "api/v1/groups"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
+		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
+		{path.Join(pathPrefix, "flags"), "command-line flags"},
+		{path.Join(pathPrefix, "metrics"), "list of application metrics"},
+		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
+	}
+	navItems = []tpl.NavItem{
+		{Name: "vmalert", Url: pathPrefix},
+		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
+		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
+		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
+	}
+}
+
 type requestHandler struct {
 	m *manager
 }
 
-var pathList = [][]string{
-	{"/api/v1/groups", "list all loaded groups and rules"},
-	{"/api/v1/alerts", "list all active alerts"},
-	{"/api/v1/groupID/alertID/status", "get alert status by ID"},
-	// /metrics is served by httpserver by default
-	{"/metrics", "list of application metrics"},
-	{"/-/reload", "reload configuration"},
-}
-
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	once.Do(func() {
+		initLinks()
+	})
+
 	switch r.URL.Path {
 	case "/":
 		if r.Method != "GET" {
 			return false
 		}
-		for _, path := range pathList {
-			p, doc := path[0], path[1]
-			fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
-		}
+		WriteWelcome(w)
+		return true
+	case "/alerts":
+		WriteListAlerts(w, rh.groupAlerts())
+		return true
+	case "/groups":
+		WriteListGroups(w, rh.groups())
 		return true
 	case "/api/v1/groups":
 		data, err := rh.listGroups()
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
 		data, err := rh.listAlerts()
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -64,14 +89,26 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		if !strings.HasSuffix(r.URL.Path, "/status") {
 			return false
 		}
-		// /api/v1/<groupName>/<alertID>/status
-		data, err := rh.alert(r.URL.Path)
+		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.Write(data)
+
+		// /api/v1/<groupID>/<alertID>/status
+		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
+			data, err := json.Marshal(alert)
+			if err != nil {
+				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+				return true
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.Write(data)
+			return true
+		}
+
+		// <groupID>/<alertID>/status
+		WriteAlert(w, alert)
 		return true
 	}
 }
@@ -83,20 +120,25 @@ type listGroupsResponse struct {
 	Status string `json:"status"`
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) groups() []APIGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	lr := listGroupsResponse{Status: "success"}
+	var groups []APIGroup
 	for _, g := range rh.m.groups {
-		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+		groups = append(groups, g.toAPI())
 	}
 
 	// sort list of alerts for deterministic output
-	sort.Slice(lr.Data.Groups, func(i, j int) bool {
-		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	sort.Slice(groups, func(i, j int) bool {
+		return groups[i].Name < groups[j].Name
 	})
 
+	return groups
+}
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	lr := listGroupsResponse{Status: "success"}
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{
@@ -114,6 +156,30 @@ type listAlertsResponse struct {
 	Status string `json:"status"`
 }
 
+func (rh *requestHandler) groupAlerts() []GroupAlerts {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	var groupAlerts []GroupAlerts
+	for _, g := range rh.m.groups {
+		var alerts []*APIAlert
+		for _, r := range g.Rules {
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			alerts = append(alerts, a.AlertsAPI()...)
+		}
+		if len(alerts) > 0 {
+			groupAlerts = append(groupAlerts, GroupAlerts{
+				Group:  g.toAPI(),
+				Alerts: alerts,
+			})
+		}
+	}
+	return groupAlerts
+}
+
 func (rh *requestHandler) listAlerts() ([]byte, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
@@ -144,18 +210,17 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
-func (rh *requestHandler) alert(path string) ([]byte, error) {
+func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", 3)
 	if len(parts) != 3 {
 		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
 			StatusCode: http.StatusBadRequest,
 		}
 	}
-
 	groupID, err := uint64FromPath(parts[0])
 	if err != nil {
 		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
@@ -168,7 +233,7 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 	if err != nil {
 		return nil, errResponse(err, http.StatusNotFound)
 	}
-	return json.Marshal(resp)
+	return resp, nil
 }
 
 func uint64FromPath(path string) (uint64, error) {

app/vmalert/web.qtpl

@@ -0,0 +1,305 @@
+{% package main %}
+
+{% import (
+    "time"
+    "sort"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+) %}
+
+
+{% func Welcome() %}
+    {%= tpl.Header("vmalert", navItems) %}
+    <p>
+        API:<br>
+        {% for _, p := range apiLinks  %}
+            {%code
+                p, doc := p[0], p[1]
+            %}
+        	<a href="{%s p %}">{%s p %}</a> - {%s doc %}<br/>
+        {% endfor %}
+    </p>
+    {%= tpl.Footer() %}
+{% endfunc %}
+
+{% func ListGroups(groups []APIGroup) %}
+    {%= tpl.Header("Groups", navItems) %}
+    {%  if len(groups) > 0 %}
+        {%code
+            rOk := make(map[string]int)
+            rNotOk := make(map[string]int)
+            for _, g := range groups {
+                for _, r := range g.AlertingRules{
+                    if r.LastError != "" {
+                        rNotOk[g.Name]++
+                    } else {
+                        rOk[g.Name]++
+                    }
+                }
+                 for _, r := range g.RecordingRules{
+                    if r.LastError != "" {
+                        rNotOk[g.Name]++
+                    } else {
+                        rOk[g.Name]++
+                    }
+                }
+            }
+        %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+        {% for _, g := range groups  %}
+              <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%s g.Interval %})</a>
+                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
+                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+                {% if len(g.Params) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra params
+                    {% for _, param := range g.Params %}
+                            <span class="float-left badge bg-primary">{%s param %}</span>
+                    {% endfor %}
+                    </div>
+                {% endif %}
+            </div>
+            <div class="collapse" id="rules-{%s g.ID %}">
+                <table class="table table-striped table-hover table-sm">
+                    <thead>
+                        <tr>
+                            <th scope="col">Rule</th>
+                            <th scope="col" title="Shows if rule's execution ended with error">Error</th>
+                            <th scope="col" title="How many samples were produced by the rule">Samples</th>
+                            <th scope="col" title="How many seconds ago rule was executed">Updated</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                    {% for _, ar := range g.AlertingRules %}
+                        <tr{% if ar.LastError != "" %} class="alert-danger"{% endif %}>
+                            <td>
+                                <b>alert:</b> {%s ar.Name %} (for: {%v ar.For %})<br>
+                                <code><pre>{%s ar.Expression %}</pre></code><br>
+                                {% if len(ar.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range ar.Labels %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                {% endfor %}
+                            </td>
+                            <td><div class="error-cell">{%s ar.LastError %}</div></td>
+                            <td>{%d ar.LastSamples %}</td>
+                            <td>{%f.3 time.Since(ar.LastExec).Seconds() %}s ago</td>
+                        </tr>
+                    {% endfor %}
+                    {% for _, rr := range g.RecordingRules  %}
+                        <tr>
+                            <td>
+                                <b>record:</b> {%s rr.Name %}<br>
+                                <code><pre>{%s rr.Expression %}</pre></code>
+                                {% if len(rr.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range rr.Labels %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                {% endfor %}
+                            </td>
+                            <td><div class="error-cell">{%s rr.LastError %}</div></td>
+                            <td>{%d rr.LastSamples %}</td>
+                            <td>{%f.3 time.Since(rr.LastExec).Seconds() %}s ago</td>
+                        </tr>
+                    {% endfor %}
+                 </tbody>
+                </table>
+            </div>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+
+{% func ListAlerts(groupAlerts []GroupAlerts) %}
+    {%= tpl.Header("Alerts", navItems) %}
+    {% if len(groupAlerts) > 0 %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         {% for _, ga := range groupAlerts %}
+            {%code g := ga.Group %}
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
+                <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
+                <br>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+            </div>
+            {%code
+                var keys []string
+                alertsByRule := make(map[string][]*APIAlert)
+                for _, alert := range ga.Alerts {
+                    if len(alertsByRule[alert.RuleID]) < 1 {
+                        keys = append(keys, alert.RuleID)
+                    }
+                    alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
+                }
+                sort.Strings(keys)
+            %}
+            <div class="collapse" id="rules-{%s g.ID %}">
+                {% for _, ruleID := range keys %}
+                    {%code
+                        defaultAR := alertsByRule[ruleID][0]
+                        var labelKeys []string
+                        for k := range defaultAR.Labels {
+                            labelKeys = append(labelKeys, k)
+                        }
+                        sort.Strings(labelKeys)
+                    %}
+                    <br>
+                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                    <br>
+                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr>
+                                <th scope="col">Labels</th>
+                                <th scope="col">State</th>
+                                <th scope="col">Active at</th>
+                                <th scope="col">Value</th>
+                                <th scope="col">Link</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                        {% for _, ar := range alertsByRule[ruleID] %}
+                            <tr>
+                                <td>
+                                    {% for _, k := range labelKeys %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
+                                    {% endfor %}
+                                </td>
+                                <td>{%= badgeState(ar.State) %}</td>
+                                <td>
+                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                </td>
+                                <td>{%s ar.Value %}</td>
+                                <td>
+                                    <a href="/{%s g.ID %}/{%s ar.ID %}/status">Details</a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                     </tbody>
+                    </table>
+                {% endfor %}
+            </div>
+            <br>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func Alert(alert *APIAlert) %}
+    {%= tpl.Header("", navItems) %}
+    {%code
+        var labelKeys []string
+        for k := range alert.Labels {
+            labelKeys = append(labelKeys, k)
+        }
+        sort.Strings(labelKeys)
+
+        var annotationKeys []string
+        for k := range alert.Annotations {
+            annotationKeys = append(annotationKeys, k)
+        }
+        sort.Strings(annotationKeys)
+    %}
+    <div class="display-6 pb-3 mb-3">{%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Active at
+        </div>
+        <div class="col">
+          {%s alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+        </div>
+      </div>
+      </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Expr
+        </div>
+        <div class="col">
+          <code><pre>{%s alert.Expression %}</pre></code>
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Labels
+        </div>
+        <div class="col">
+           {% for _, k := range labelKeys %}
+                <span class="m-1 badge bg-primary">{%s k %}={%s alert.Labels[k] %}</span>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Annotations
+        </div>
+        <div class="col">
+           {% for _, k := range annotationKeys %}
+                <b>{%s k %}:</b><br>
+                <p>{%s alert.Annotations[k] %}</p>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Group
+        </div>
+        <div class="col">
+           <a target="_blank" href="/groups#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
+        </div>
+      </div>
+    </div>
+     <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Source link
+        </div>
+        <div class="col">
+           <a target="_blank" href="{%s alert.SourceLink %}">Link</a>
+        </div>
+      </div>
+    </div>
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func badgeState(state string) %}
+{%code
+    badgeClass := "bg-warning text-dark"
+    if state == "firing" {
+        badgeClass = "bg-danger"
+    }
+%}
+<span class="badge {%s badgeClass %}">{%s state %}</span>
+{% endfunc %}
+
+{% func badgeRestored() %}
+<span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
+{% endfunc %}

app/vmalert/web_types.go

@@ -9,6 +9,7 @@ import (
 type APIAlert struct {
 	ID          string            `json:"id"`
 	Name        string            `json:"name"`
+	RuleID      string            `json:"rule_id"`
 	GroupID     string            `json:"group_id"`
 	Expression  string            `json:"expression"`
 	State       string            `json:"state"`
@@ -16,6 +17,8 @@ type APIAlert struct {
 	Labels      map[string]string `json:"labels"`
 	Annotations map[string]string `json:"annotations"`
 	ActiveAt    time.Time         `json:"activeAt"`
+	SourceLink  string            `json:"source"`
+	Restored    bool              `json:"restored"`
 }
 
 // APIGroup represents Group for WEB view
@@ -26,6 +29,8 @@ type APIGroup struct {
 	File           string             `json:"file"`
 	Interval       string             `json:"interval"`
 	Concurrency    int                `json:"concurrency"`
+	Params         []string           `json:"params"`
+	Labels         map[string]string  `json:"labels,omitempty"`
 	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
 	RecordingRules []APIRecordingRule `json:"recording_rules"`
 }
@@ -39,6 +44,7 @@ type APIAlertingRule struct {
 	Expression  string            `json:"expression"`
 	For         string            `json:"for"`
 	LastError   string            `json:"last_error"`
+	LastSamples int               `json:"last_samples"`
 	LastExec    time.Time         `json:"last_exec"`
 	Labels      map[string]string `json:"labels"`
 	Annotations map[string]string `json:"annotations"`
@@ -46,12 +52,19 @@ type APIAlertingRule struct {
 
 // APIRecordingRule represents RecordingRule for WEB view
 type APIRecordingRule struct {
-	ID         string            `json:"id"`
-	Name       string            `json:"name"`
-	Type       string            `json:"type"`
-	GroupID    string            `json:"group_id"`
-	Expression string            `json:"expression"`
-	LastError  string            `json:"last_error"`
-	LastExec   time.Time         `json:"last_exec"`
-	Labels     map[string]string `json:"labels"`
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	Type        string            `json:"type"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	LastError   string            `json:"last_error"`
+	LastSamples int               `json:"last_samples"`
+	LastExec    time.Time         `json:"last_exec"`
+	Labels      map[string]string `json:"labels"`
+}
+
+// GroupAlerts represents a group of alerts for WEB view
+type GroupAlerts struct {
+	Group  APIGroup
+	Alerts []*APIAlert
 }

app/vmauth/README.md

@@ -1,9 +1,9 @@
 # vmauth
 
-`vmauth` is a simple auth proxy and router for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
-It reads username and password from [Basic Auth headers](https://en.wikipedia.org/wiki/Basic_access_authentication),
-matches them against configs pointed by `-auth.config` command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
-
+`vmauth` is a simple auth proxy, router and [load balancer](#load-balancing) for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
+It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) and `Bearer token` is supported),
+matches them against configs pointed by [-auth.config](#auth-config) command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
+The `-auth.config` can point to either local file or to http url.
 
 ## Quick start
 
@@ -17,31 +17,42 @@ and pass the following flag to `vmauth` binary in order to start authorizing and
 After that `vmauth` starts accepting HTTP requests on port `8427` and routing them according to the provided [-auth.config](#auth-config).
 The port can be modified via `-httpListenAddr` command-line flag.
 
-The auth config can be reloaded by passing `SIGHUP` signal to `vmauth`.
+The auth config can be reloaded either by passing `SIGHUP` signal to `vmauth` or by querying `/-/reload` http endpoint.
 
 Docker images for `vmauth` are available [here](https://hub.docker.com/r/victoriametrics/vmauth/tags).
 
 Pass `-help` to `vmauth` in order to see all the supported command-line flags with their descriptions.
 
 Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML,
-accounting and rate limiting such as [vmgateway](https://victoriametrics.github.io/vmgateway.html).
+accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.com/vmgateway.html).
 
+## Load balancing
+
+Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
 ## Auth config
 
-Auth config is represented in the following simple `yml` format:
+`-auth.config` is represented in the following simple `yml` format:
 
 ```yml
-
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.
 
 users:
   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
   # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be proxied to http://localhost:8428 .
@@ -50,42 +61,67 @@ users:
   password: "***"
   url_prefix: "http://localhost:8428"
 
-  # The user for querying account 123 in VictoriaMetrics cluster
-  # See https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format
+  # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://vmselect:8481/select/123/prometheus .
-  # For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect:8481/select/123/prometheus/api/v1/select
+  # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+  # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
+- username: "local-single-node"
+  password: "***"
+  url_prefix: "http://localhost:8428?extra_label=team=dev"
+
+  # The user for querying account 123 in VictoriaMetrics cluster
+  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+  # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
+  #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
+  #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
 - username: "cluster-select-account-123"
   password: "***"
-  url_prefix: "http://vmselect:8481/select/123/prometheus"
+  url_prefix:
+  - "http://vmselect1:8481/select/123/prometheus"
+  - "http://vmselect2:8481/select/123/prometheus"
 
   # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
-  # See https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format
+  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://vminsert:8480/insert/42/prometheus .
-  # For example, http://vmauth:8427/api/v1/write is proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+  # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
+  #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
+  #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write
 - username: "cluster-insert-account-42"
   password: "***"
-  url_prefix: "http://vminsert:8480/insert/42/prometheus"
-
+  url_prefix:
+  - "http://vminsert1:8480/insert/42/prometheus"
+  - "http://vminsert2:8480/insert/42/prometheus"
 
   # A single user for querying and inserting data:
   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
-  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to http://vmselect:8481/select/42/prometheus.
-  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect:8480/select/42/prometheus/api/v1/query
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
+  #     - http://vmselect1:8481/select/42/prometheus
+  #     - http://vmselect2:8481/select/42/prometheus
+  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
+  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
   url_map:
-  - src_paths: ["/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^/]+/values"]
-    url_prefix: "http://vmselect:8481/select/42/prometheus"
+  - src_paths:
+    - "/api/v1/query"
+    - "/api/v1/query_range"
+    - "/api/v1/label/[^/]+/values"
+    url_prefix:
+    - "http://vmselect1:8481/select/42/prometheus"
+    - "http://vmselect2:8481/select/42/prometheus"
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"
 ```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
-
 ## Security
 
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
@@ -101,21 +137,29 @@ Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable
 
 Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.
 
+It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
 
 ## Monitoring
 
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
-either via [vmagent](https://victoriametrics.github.io/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
+either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
+`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+
+```yml
+users:
+- username: "secret_user"
+  name: "foobar"
+  # other config options here
+```
 
 ## How to build from sources
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.
 
-
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.15.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
 2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth` binary and puts it into the `bin` folder.
 
@@ -138,7 +182,6 @@ by setting it via `<ROOT_IMAGE>` environment variable. For example, the followin
 ROOT_IMAGE=scratch make package-vmauth
 ```
 
-
 ## Profiling
 
 `vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
@@ -159,7 +202,6 @@ The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
 
-
 ## Advanced usage
 
 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
@@ -169,40 +211,42 @@ Pass `-help` command-line arg to `vmauth` in order to see all the configuration
 
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
 
-See the docs at https://victoriametrics.github.io/vmauth.html .
+See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
-    	Path to auth config. See https://victoriametrics.github.io/vmauth.html for details on the format of this auth config
+    	Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
     	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
     	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
     	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
     	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
     	TCP address to listen for http connections (default ":8427")
+  -logInvalidAuthTokens
+    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
     	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
@@ -212,20 +256,24 @@ See the docs at https://victoriametrics.github.io/vmauth.html .
   -loggerTimezone string
     	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero value disables the rate limit
+    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxIdleConnsPerBackend int
+    	The maximum number of idle connections vmauth can open per each backend host (default 100)
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -reloadAuthKey string
+    	Auth key for /-/reload http endpoint. It must be passed as authKey=...
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
   -tlsKeyFile string
     	Path to file with TLS key. Used only if -tls is set
   -version

app/vmauth/auth_config.go

@@ -4,14 +4,16 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/metrics"
@@ -19,30 +21,60 @@ import (
 )
 
 var (
-	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://victoriametrics.github.io/vmauth.html "+
-		"for details on the format of this auth config")
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
 )
 
 // AuthConfig represents auth config.
 type AuthConfig struct {
-	Users []UserInfo `yaml:"users"`
+	Users []UserInfo `yaml:"users,omitempty"`
 }
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	BearerToken string   `yaml:"bearer_token"`
-	Username    string   `yaml:"username"`
-	Password    string   `yaml:"password"`
-	URLPrefix   string   `yaml:"url_prefix"`
-	URLMap      []URLMap `yaml:"url_map"`
+	Name        string     `yaml:"name,omitempty"`
+	BearerToken string     `yaml:"bearer_token,omitempty"`
+	Username    string     `yaml:"username,omitempty"`
+	Password    string     `yaml:"password,omitempty"`
+	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMap      []URLMap   `yaml:"url_map,omitempty"`
+	Headers     []Header   `yaml:"headers,omitempty"`
 
 	requests *metrics.Counter
 }
 
+// Header is `Name: Value` http header, which must be added to the proxied request.
+type Header struct {
+	Name  string
+	Value string
+}
+
+// UnmarshalYAML unmarshals h from f.
+func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
+	var s string
+	if err := f(&s); err != nil {
+		return err
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
+	}
+	h.Name = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
+}
+
+// MarshalYAML marshals h to yaml.
+func (h *Header) MarshalYAML() (interface{}, error) {
+	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
+	return s, nil
+}
+
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths"`
-	URLPrefix string     `yaml:"url_prefix"`
+	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+	Headers   []Header   `yaml:"headers,omitempty"`
 }
 
 // SrcPath represents an src path
@@ -51,6 +83,76 @@ type SrcPath struct {
 	re        *regexp.Regexp
 }
 
+// URLPrefix represents pased `url_prefix`
+type URLPrefix struct {
+	n    uint32
+	urls []*url.URL
+}
+
+func (up *URLPrefix) getNextURL() *url.URL {
+	n := atomic.AddUint32(&up.n, 1)
+	idx := n % uint32(len(up.urls))
+	return up.urls[idx]
+}
+
+// UnmarshalYAML unmarshals up from yaml.
+func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
+	var v interface{}
+	if err := f(&v); err != nil {
+		return err
+	}
+	var urls []string
+	switch x := v.(type) {
+	case string:
+		urls = []string{x}
+	case []interface{}:
+		if len(x) == 0 {
+			return fmt.Errorf("`url_prefix` must contain at least a single url")
+		}
+		us := make([]string, len(x))
+		for i, xx := range x {
+			s, ok := xx.(string)
+			if !ok {
+				return fmt.Errorf("`url_prefix` must contain array of strings; got %T", xx)
+			}
+			us[i] = s
+		}
+		urls = us
+	default:
+		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
+	}
+	pus := make([]*url.URL, len(urls))
+	for i, u := range urls {
+		pu, err := url.Parse(u)
+		if err != nil {
+			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
+		}
+		pus[i] = pu
+	}
+	up.urls = pus
+	return nil
+}
+
+// MarshalYAML marshals up to yaml.
+func (up *URLPrefix) MarshalYAML() (interface{}, error) {
+	var b []byte
+	if len(up.urls) == 1 {
+		u := up.urls[0].String()
+		b = strconv.AppendQuote(b, u)
+		return string(b), nil
+	}
+	b = append(b, '[')
+	for i, pu := range up.urls {
+		u := pu.String()
+		b = strconv.AppendQuote(b, u)
+		if i+1 < len(up.urls) {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, ']')
+	return string(b), nil
+}
+
 func (sp *SrcPath) match(s string) bool {
 	prefix, ok := sp.re.LiteralPrefix()
 	if ok {
@@ -88,6 +190,12 @@ func initAuthConfig() {
 	if len(*authConfigPath) == 0 {
 		logger.Fatalf("missing required `-auth.config` command-line flag")
 	}
+
+	// Register SIGHUP handler for config re-read just before readAuthConfig call.
+	// This guarantees that the config will be re-read if the signal arrives during readAuthConfig call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
 	m, err := readAuthConfig(*authConfigPath)
 	if err != nil {
 		logger.Fatalf("cannot load auth config from `-auth.config=%s`: %s", *authConfigPath, err)
@@ -97,7 +205,7 @@ func initAuthConfig() {
 	authConfigWG.Add(1)
 	go func() {
 		defer authConfigWG.Done()
-		authConfigReloader()
+		authConfigReloader(sighupCh)
 	}()
 }
 
@@ -106,8 +214,7 @@ func stopAuthConfig() {
 	authConfigWG.Wait()
 }
 
-func authConfigReloader() {
-	sighupCh := procutil.NewSighupChan()
+func authConfigReloader(sighupCh <-chan os.Signal) {
 	for {
 		select {
 		case <-stopCh:
@@ -130,9 +237,9 @@ var authConfigWG sync.WaitGroup
 var stopCh chan struct{}
 
 func readAuthConfig(path string) (map[string]*UserInfo, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := fs.ReadFileOrHTTP(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read %q: %w", path, err)
+		return nil, err
 	}
 	m, err := parseAuthConfig(data)
 	if err != nil {
@@ -169,46 +276,68 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if byUsername[ui.Username] {
 			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
 		}
-		authToken := getAuthToken(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[authToken] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", authToken, ui.BearerToken, ui.Username)
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
 		}
-		if len(ui.URLPrefix) > 0 {
-			urlPrefix, err := sanitizeURLPrefix(ui.URLPrefix)
-			if err != nil {
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
+		}
+		if ui.URLPrefix != nil {
+			if err := ui.URLPrefix.sanitize(); err != nil {
 				return nil, err
 			}
-			ui.URLPrefix = urlPrefix
 		}
 		for _, e := range ui.URLMap {
 			if len(e.SrcPaths) == 0 {
-				return nil, fmt.Errorf("missing `src_paths`")
+				return nil, fmt.Errorf("missing `src_paths` in `url_map`")
 			}
-			urlPrefix, err := sanitizeURLPrefix(e.URLPrefix)
-			if err != nil {
+			if e.URLPrefix == nil {
+				return nil, fmt.Errorf("missing `url_prefix` in `url_map`")
+			}
+			if err := e.URLPrefix.sanitize(); err != nil {
 				return nil, err
 			}
-			e.URLPrefix = urlPrefix
 		}
-		if len(ui.URLMap) == 0 && len(ui.URLPrefix) == 0 {
+		if len(ui.URLMap) == 0 && ui.URLPrefix == nil {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
 		if ui.BearerToken != "" {
+			name := "bearer_token"
+			if ui.Name != "" {
+				name = ui.Name
+			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
-			ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total{username="bearer_token"}`)
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byBearerToken[ui.BearerToken] = true
 		}
 		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+			name := ui.Username
+			if ui.Name != "" {
+				name = ui.Name
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byUsername[ui.Username] = true
 		}
-		byAuthToken[authToken] = ui
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
+func getAuthTokens(bearerToken, username, password string) (string, string) {
+	if bearerToken != "" {
+		// Accept the bearerToken as Basic Auth username with empty password
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
+	}
+	at := getAuthToken("", username, password)
+	return at, at
+}
+
 func getAuthToken(bearerToken, username, password string) string {
 	if bearerToken != "" {
 		return "Bearer " + bearerToken
@@ -218,21 +347,28 @@ func getAuthToken(bearerToken, username, password string) string {
 	return "Basic " + token64
 }
 
-func sanitizeURLPrefix(urlPrefix string) (string, error) {
+func (up *URLPrefix) sanitize() error {
+	for i, pu := range up.urls {
+		puNew, err := sanitizeURLPrefix(pu)
+		if err != nil {
+			return err
+		}
+		up.urls[i] = puNew
+	}
+	return nil
+}
+
+func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
 	// Remove trailing '/' from urlPrefix
-	for strings.HasSuffix(urlPrefix, "/") {
-		urlPrefix = urlPrefix[:len(urlPrefix)-1]
+	for strings.HasSuffix(urlPrefix.Path, "/") {
+		urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
 	}
 	// Validate urlPrefix
-	target, err := url.Parse(urlPrefix)
-	if err != nil {
-		return "", fmt.Errorf("invalid `url_prefix: %q`: %w", urlPrefix, err)
+	if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
+		return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)
 	}
-	if target.Scheme != "http" && target.Scheme != "https" {
-		return "", fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)
-	}
-	if target.Host == "" {
-		return "", fmt.Errorf("missing hostname in `url_prefix %q`", urlPrefix)
+	if urlPrefix.Host == "" {
+		return nil, fmt.Errorf("missing hostname in `url_prefix %q`", urlPrefix.Host)
 	}
 	return urlPrefix, nil
 }

app/vmauth/auth_config_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"regexp"
 	"testing"
 
@@ -55,6 +56,33 @@ users:
 - username: foo
   url_prefix: http:///bar
 `)
+	f(`
+users:
+- username: foo
+  url_prefix:
+    bar: baz
+`)
+	f(`
+users:
+- username: foo
+  url_prefix:
+  - [foo]
+`)
+
+	// Invalid headers
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  headers: foobar
+`)
+
+	// empty url_prefix
+	f(`
+users:
+- username: foo
+  url_prefix: []
+`)
 
 	// Username and bearer_token in a single config
 	f(`
@@ -102,6 +130,24 @@ users:
   - src_paths: ["/foo/bar"]
 `)
 
+	// Invalid url_prefix in url_map
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ["/foo/bar"]
+    url_prefix: foo.bar
+`)
+
+	// empty url_prefix in url_map
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foo/bar']
+    url_prefix: []
+`)
+
 	// Missing src_paths in url_map
 	f(`
 users:
@@ -118,6 +164,27 @@ users:
   - src_paths: ['fo[obar']
     url_prefix: http://foobar
 `)
+
+	// Invalid headers in url_map (missing ':')
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+    - foobar
+`)
+	// Invalid headers in url_map (dictionary instead of array)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+      aaa: bbb
+`)
 }
 
 func TestParseAuthConfigSuccess(t *testing.T) {
@@ -143,7 +210,26 @@ users:
 		getAuthToken("", "foo", "bar"): {
 			Username:  "foo",
 			Password:  "bar",
-			URLPrefix: "http://aaa:343/bbb",
+			URLPrefix: mustParseURL("http://aaa:343/bbb"),
+		},
+	})
+
+	// Multiple url_prefix entries
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix:
+  - http://node1:343/bbb
+  - http://node2:343/bbb
+`, map[string]*UserInfo{
+		getAuthToken("", "foo", "bar"): {
+			Username: "foo",
+			Password: "bar",
+			URLPrefix: mustParseURLs([]string{
+				"http://node1:343/bbb",
+				"http://node2:343/bbb",
+			}),
 		},
 	})
 
@@ -157,11 +243,11 @@ users:
 `, map[string]*UserInfo{
 		getAuthToken("", "foo", ""): {
 			Username:  "foo",
-			URLPrefix: "http://foo",
+			URLPrefix: mustParseURL("http://foo"),
 		},
 		getAuthToken("", "bar", ""): {
 			Username:  "bar",
-			URLPrefix: "https://bar/x",
+			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
 
@@ -173,18 +259,60 @@ users:
   - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"]
     url_prefix: http://vmselect/select/0/prometheus
   - src_paths: ["/api/v1/write"]
-    url_prefix: http://vminsert/insert/0/prometheus
+    url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
+    headers:
+    - "foo: bar"
+    - "xxx: y"
 `, map[string]*UserInfo{
 		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
 			URLMap: []URLMap{
 				{
 					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
-					URLPrefix: "http://vmselect/select/0/prometheus",
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
-					URLPrefix: "http://vminsert/insert/0/prometheus",
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMap: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
 				},
 			},
 		},
@@ -222,3 +350,21 @@ func areEqualConfigs(a, b map[string]*UserInfo) error {
 	}
 	return nil
 }
+
+func mustParseURL(u string) *URLPrefix {
+	return mustParseURLs([]string{u})
+}
+
+func mustParseURLs(us []string) *URLPrefix {
+	pus := make([]*url.URL, len(us))
+	for i, u := range us {
+		pu, err := url.Parse(u)
+		if err != nil {
+			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
+		}
+		pus[i] = pu
+	}
+	return &URLPrefix{
+		urls: pus,
+	}
+}