docs/CHANGELOG.md: cut v1.75.0

These errors include `broken pipe` and `reset by peer`.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2334

.dockerignore

@@ -4,3 +4,4 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
+.vscode

.github/dependabot.yml

@@ -21,6 +21,6 @@ updates:
     schedule:
       interval: "daily"
   - package-ecosystem: "npm"
-    directory: "/app/vmui"
+    directory: "/app/vmui/packages/vmui"
     schedule:
       interval: "weekly"

.github/workflows/check-licenses.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/main.yml

@@ -16,15 +16,15 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
-      - name: Dependencies
-        run: |
-          go get -u golang.org/x/lint/golint
-          go get -u github.com/kisielk/errcheck
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.29.0
       - name: Code checkout
         uses: actions/checkout@master
+      - name: Dependencies
+        run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
       - name: Build
         env:
           GO111MODULE: on
@@ -60,7 +60,7 @@ jobs:
           GOOS=darwin go build -mod=vendor ./app/vmctl
           CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
       - name: Publish coverage
-        uses: codecov/codecov-action@v2.0.3
+        uses: codecov/codecov-action@v2.1.0
         with:
           file: ./coverage.txt
 

.gitignore

@@ -4,6 +4,7 @@
 *.pprof
 /bin
 .idea
+.vscode
 *.test
 *.swp
 /gocache-for-docker

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2021 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -24,6 +24,8 @@ all: \
 
 include app/*/Makefile
 include deployment/*/Makefile
+include snap/local/Makefile
+
 
 clean:
 	rm -rf bin/*
@@ -84,9 +86,12 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
-release-snap:
-	snapcraft
-	snapcraft upload "victoriametrics_$(PKG_TAG)_multi.snap" --release beta,edge,candidate
+
+publish-release:
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
 
 release: \
 	release-victoria-metrics \
@@ -95,66 +100,82 @@ release: \
 release-victoria-metrics: \
 	release-victoria-metrics-amd64 \
 	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64
+	release-victoria-metrics-arm64 \
+	release-victoria-metrics-darwin-amd64 \
+	release-victoria-metrics-darwin-arm64
 
 release-victoria-metrics-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic
 
 release-victoria-metrics-arm:
-	GOARCH=arm $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm $(MAKE) release-victoria-metrics-generic
 
 release-victoria-metrics-arm64:
-	GOARCH=arm64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic
 
-release-victoria-metrics-generic: victoria-metrics-$(GOARCH)-prod
+release-victoria-metrics-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt
 
 release-vmutils: \
 	release-vmutils-amd64 \
 	release-vmutils-arm64 \
 	release-vmutils-arm \
+	release-vmutils-darwin-amd64  \
+	release-vmutils-darwin-arm64 \
 	release-vmutils-windows-amd64
 
 release-vmutils-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-generic
+	OSARCH=amd64 $(MAKE) release-vmutils-generic
 
 release-vmutils-arm64:
-	GOARCH=arm64 $(MAKE) release-vmutils-generic
+	OSARCH=arm64 $(MAKE) release-vmutils-generic
 
 release-vmutils-arm:
-	GOARCH=arm $(MAKE) release-vmutils-generic
+	OSARCH=arm $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic
 
 release-vmutils-windows-amd64:
 	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic
 
 release-vmutils-generic: \
-	vmagent-$(GOARCH)-prod \
-	vmalert-$(GOARCH)-prod \
-	vmauth-$(GOARCH)-prod \
-	vmbackup-$(GOARCH)-prod \
-	vmrestore-$(GOARCH)-prod \
-	vmctl-$(GOARCH)-prod
+	vmagent-$(OSARCH)-prod \
+	vmalert-$(OSARCH)-prod \
+	vmauth-$(OSARCH)-prod \
+	vmbackup-$(OSARCH)-prod \
+	vmrestore-$(OSARCH)-prod \
+	vmctl-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > vmutils-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt
 
 release-vmutils-windows-generic: \
 	vmagent-windows-$(GOARCH)-prod \
@@ -174,6 +195,7 @@ release-vmutils-windows-generic: \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 
+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -261,7 +283,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.40.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.44.1
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

app/victoria-metrics/Makefile

@@ -27,6 +27,12 @@ victoria-metrics-ppc64le-prod:
 victoria-metrics-386-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
 
+victoria-metrics-darwin-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
+
+victoria-metrics-darwin-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 

app/victoria-metrics/main.go

@@ -25,9 +25,9 @@ import (
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
 )
 
 func main() {
@@ -51,7 +51,7 @@ func main() {
 
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	storage.SetDedupInterval(*minScrapeInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -90,13 +90,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/vmui", "Web UI"},
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/metrics", "available service metrics"},
-			{"/api/v1/status/tsdb", "tsdb status page"},
-			{"/api/v1/status/top_queries", "top queries"},
-			{"/api/v1/status/active_queries", "active queries"},
+			{"vmui", "Web UI"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"api/v1/status/tsdb", "tsdb status page"},
+			{"api/v1/status/top_queries", "top queries"},
+			{"api/v1/status/active_queries", "active queries"},
 		})
 		return true
 	}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -19,6 +19,7 @@
 	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-100s}","3"],
 	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-40s}","2"],

app/vmagent/Makefile

@@ -27,6 +27,15 @@ vmagent-ppc64le-prod:
 vmagent-386-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-386
 
+vmagent-darwin-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
+
+vmagent-darwin-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
+
+vmagent-windows-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
+
 package-vmagent:
 	APP_NAME=vmagent $(MAKE) package-via-docker
 
@@ -81,6 +90,3 @@ vmagent-pure:
 
 vmagent-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch
-
-vmagent-windows-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64

app/vmagent/README.md

@@ -12,22 +12,24 @@ or any other Prometheus-compatible storage systems that support the `remote_writ
 While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
 and RAM friendly to scrape metrics from Prometheus-compatible exporters into VictoriaMetrics.
 Also, we found that our user's infrastructure are like snowflakes in that no two are alike. Therefore we decided to add more flexibility
-to `vmagent` such as the ability to push metrics instead of pulling them. We did our best and will continue to improve vmagent.
+to `vmagent` such as the ability to push metrics additionally to pulling them. We did our best and will continue to improve `vmagent`.
 
 
 ## Features
 
-* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
-  See [Quick Start](#quick-start) for details.
+* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter). See [Quick Start](#quick-start) for details.
+* Can read data from Kafka. See [these docs](#reading-metrics-from-kafka).
+* Can write data to Kafka. See [these docs](#writing-metrics-to-kafka).
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Accepts data via all ingestion protocols supported by VictoriaMetrics:
-  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * DataDog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent).
+  * InfluxDB line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
   * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
   * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-native-format).
-  * Data in Prometheus exposition format. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
+  * Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
@@ -44,7 +46,7 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and configure the following flags to the `vmagent` binary in order to start scraping Prometheus targets:
 
-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`)
+* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`). The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file, so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` additional command-line flag, so `vmagent` will ignore unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
 * `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified multiple times to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
@@ -53,13 +55,13 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-If you only need to collect Influx data, then the following command is sufficient:
+If you only need to collect InfluxDB data, then the following command is sufficient:
 
 ```
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+Then send InfluxDB data to `http://vmagent-host:8429`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
 
 `vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags).
 
@@ -212,15 +214,16 @@ The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders w
 
 ## Loading scrape configs from multiple files
 
-`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory plus a `single_scrape_config.yml` file:
+`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url:
 
 ```yml
 scrape_config_files:
 - configs/*.yml
 - single_scrape_config.yml
+- https://config-server/scrape_config.yml
 ```
 
-Every referred file can contain arbitrary number of any [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
+Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
 
 ```yml
 - job_name: foo
@@ -234,6 +237,19 @@ Every referred file can contain arbitrary number of any [supported scrape config
 `vmagent` dynamically reloads these files on `SIGHUP` signal or on the request to `http://vmagent:8429/-/reload`.
 
 
+## Unsupported Prometheus config sections
+
+`vmagent` doesn't support the following sections in Prometheus config file passed to `-promscrape.config` command-line flag:
+
+* [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This section is substituted with various `-remoteWrite*` command-line flags. See [the full list of flags](#advanced-usage). The `remote_write` section isn't supported in order to reduce possible confusion when `vmagent` is used for accepting incoming metrics via push protocols such as InfluxDB, Graphite, OpenTSDB, DataDog, etc. In this case the `-promscrape.config` file isn't needed. See [these docs](#features) for details.
+* `remote_read`. This section isn't supported at all.
+* `rule_files` and `alerting`. These sections are supported by [vmalert](https://docs.victoriametrics.com/vmalert.html).
+
+The list of supported service discovery types is available [here](#how-to-collect-metrics-in-prometheus-format).
+
+Additionally `vmagent` doesn't support `refresh_interval` option at service discovery sections. This option is substituted with `-promscrape.*CheckInterval` command-line options, which are specific per each service discovery type. See [the full list of command-line flags for vmagent](#advanced-usage).
+
+
 ## Adding labels to metrics
 
 Labels can be added to metrics by the following mechanisms:
@@ -248,19 +264,51 @@ Labels can be added to metrics by the following mechanisms:
 
 ## Relabeling
 
-`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
-and also provides the following actions:
+VictoriaMetrics components (including `vmagent`) support Prometheus-compatible relabeling.
+They provide the following additional actions on top of actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config):
 
 * `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`.
 * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
 * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
 * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
+* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+
+The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
+
+```yaml
+- action: keep_metrics
+  regex: "metric_a|metric_b|foo_.+"
+```
+
+```yaml
+- action: keep_metrics
+  regex:
+  - "metric_a"
+  - "metric_b"
+  - "foo_.+"
+```
+
+VictoriaMetrics components support an optional `if` filter, which can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops targets, which don't match `foo{bar="baz"}` series selector:
+
+```yaml
+- action: keep
+  if: 'foo{bar="baz"}'
+```
+
+This is equivalent to less clear traditional relabeling rule:
+
+```yaml
+- action: keep
+  source_labels: [__name__, bar]
+  regex: 'foo;baz'
+```
 
 The relabeling can be defined in the following places:
 
 * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
-* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
 
 You can read more about relabeling in the following articles:
@@ -275,28 +323,43 @@ You can read more about relabeling in the following articles:
 
 ## Prometheus staleness markers
 
-Starting from [v1.64.0](https://docs.victoriametrics.com/CHANGELOG.html#v1640), `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) for scraped metrics when the scrape target is removed from the list of targets. Prometheus staleness markers aren't sent in [stream parsing mode](#stream-parsing-mode) or if `-promscrape.noStaleMarkers` command-line is set.
+`vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases:
+
+* If they are passed to `vmagent` via [Prometheus remote_write protocol](#prometheus-remote_write-proxy).
+* If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metric.
+* If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target.
+* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
+
+Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.
 
 
 ## Stream parsing mode
 
-By default `vmagent` reads the full response from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics. Stream parsing mode may be enabled either globally for all of the scrape targets by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
+By default `vmagent` reads the full response body from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics.
 
-  ```yml
-  scrape_configs:
-  - job_name: 'big-federate'
-    stream_parse: true
-    static_configs:
-    - targets:
-      - big-prometeus1
-      - big-prometeus2
-    honor_labels: true
-    metrics_path: /federate
-    params:
-      'match[]': ['{__name__!=""}']
-  ```
+Stream parsing mode is automatically enabled for scrape targets returning response bodies with sizes bigger than the `-promscrape.minResponseSizeForStreamParse` command-line flag value. Additionally, the stream parsing mode can be explicitly enabled in the following places:
 
-Note that `sample_limit` option doesn't prevent from data push to remote storage if stream parsing is enabled because the parsed data is pushed to remote storage as soon as it is parsed.
+- Via `-promscrape.streamParse` command-line flag. In this case all the scrape targets defined in the file pointed by `-promscrape.config` are scraped in stream parsing mode.
+- Via `stream_parse: true` option at `scrape_configs` section. In this case all the scrape targets defined in this section are scraped in stream parsing mode.
+- Via `__stream_parse__=true` label, which can be set via [relabeling](#relabeling) at `relabel_configs` section. In this case stream parsing mode is enabled for the corresponding scrape targets. Typical use case: to set the label via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets exposing big number of metrics.
+
+Examples:
+
+```yml
+scrape_configs:
+- job_name: 'big-federate'
+  stream_parse: true
+  static_configs:
+  - targets:
+    - big-prometeus1
+    - big-prometeus2
+  honor_labels: true
+  metrics_path: /federate
+  params:
+    'match[]': ['{__name__!=""}']
+```
+
+Note that `sample_limit` and `series_limit` options cannot be used in stream parsing mode because the parsed data is pushed to remote storage as soon as it is parsed.
 
 
 ## Scraping big number of targets
@@ -364,7 +427,13 @@ scrape_configs:
 
 ## Cardinality limiter
 
-By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced across all the scrape targets by specifying `-promscrape.seriesLimitPerTarget` command-line option. The limit also can be specified via `series_limit` option at `scrape_config` section. All the scraped metrics are dropped for time series exceeding the given limit. The exceeded limit can be [monitored](#monitoring) via `promscrape_series_limit_rows_dropped_total` metric, which shows the number of metrics dropped due to the exceeded limit.
+By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced in the following places:
+
+- Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually to all the scrape targets defined in the file pointed by `-promscrape.config`.
+- Via `series_limit` config option at `scrape_config` section. This limit is applied individually to all the scrape targets defined in the given `scrape_config`.
+- Via `__series_limit__` label, which can be set with [relabeling](#relabeling) at `relabel_configs` section. This limit is applied to the corresponding scrape targets. Typical use case: to set the limit via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets, which may expose too high number of time series.
+
+All the scraped metrics are dropped for time series exceeding the given limit. The exceeded limit can be [monitored](#monitoring) via `promscrape_series_limit_rows_dropped_total` metric.
 
 See also `sample_limit` option at [scrape_config section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
 
@@ -387,7 +456,7 @@ These limits are approximate, so `vmagent` can underflow/overflow the limit by a
 
 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. We recommend setting up regular scraping of this page
 either through `vmagent` itself or by Prometheus so that the exported metrics may be analyzed later.
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add a review to the dashboard.
 
 `vmagent` also exports the status for various targets at the following handlers:
@@ -410,10 +479,12 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
   as `vmagent` establishes at least a single TCP connection per target.
 
 * If `vmagent` uses too big amounts of memory, then the following options can help:
-  * Enabling stream parsing. See [these docs](#stream-parsing-mode).
+  * Disabling staleness tracking with `-promscrape.noStaleMarkers` option. See [these docs](#prometheus-staleness-markers).
+  * Enabling stream parsing mode if `vmagent` scrapes targets with millions of metrics per target. See [these docs](#stream-parsing-mode).
   * Reducing the number of output queues with `-remoteWrite.queues` command-line option.
   * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kuberntes if `vmagent` runs under these systems.
   * Reducing the number of CPU cores vmagent can use by passing `GOMAXPROCS=N` environment variable to `vmagent`, where `N` is the desired limit on CPU cores. Another option is to reduce CPU limits in Docker or Kubernetes if `vmagent` runs under these systems.
+  * Passing `-promscrape.dropOriginalLabels` command-line option to `vmagent`, so it drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page. This reduces memory usage when scraping big number of targets at the cost of reduced debuggability for improperly configured per-target relabeling.
 
 * When `vmagent` scrapes many unreliable targets, it can flood the error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
@@ -422,21 +493,13 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 * The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
   This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default the `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM. Therefore big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if a big number of scrape targets are dropped during relabeling.
 
-* If `vmagent` scrapes a big number of targets then the `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
-  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly. It is also recommended increasing `-remoteWrite.maxBlockSize` and `-remoteWrite.maxRowsPerBlock` command-line options in this case. This can improve data ingestion performance to the configured remote storage systems at the cost of higher memory usage.
 
-* If `vmagent` scrapes targets with millions of metrics per target (for example, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
-  we recommend enabling [stream parsing mode](#stream-parsing-mode) in order to reduce memory usage during scraping.
-
-* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly.
-
-* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`.
-  Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data
-  if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
+* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`. Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
 
 * `vmagent` drops data blocks if remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
-* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos.
+* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos, which typically emit `out of order sample` errors. The best solution is to use remote storage with [backfilling support](https://docs.victoriametrics.com/#backfilling).
 
 * `vmagent` buffers scraped data at the `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
@@ -483,6 +546,108 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
     regex: true
   ```
 
+## Kafka integration
+
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read and write metrics from / to Kafka:
+
+* [Reading metrics from Kafka](#reading-metrics-from-kafka)
+* [Writing metrics to Kafka](#writing-metrics-to-kafka)
+
+The enterprise version of vmagent is available for evaluation at [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page in `vmutils-*-enteprise.tar.gz` archives and in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+
+### Reading metrics from Kafka
+
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
+
+* `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). Messages in this format can be sent by vmagent - see [these docs](#writing-metrics-to-kafka).
+* `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
+* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
+* `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
+* `jsonline` - [JSON line format](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format).
+
+Every Kafka message may contain multiple lines in `influx`, `prometheus`, `graphite` and `jsonline` format delimited by `\n`.
+
+`vmagent` consumes messages from Kafka topics specified by `-kafka.consumer.topic` command-line flag. Multiple topics can be specified by passing multiple `-kafka.consumer.topic` command-line flags to `vmagent`.
+
+`vmagent` consumes messages from Kafka brokers specified by `-kafka.consumer.topic.brokers` command-line flag. Multiple brokers can be specified per each `-kafka.consumer.topic` by passing a list of brokers delimited by `;`. For example, `-kafka.consumer.topic.brokers=host1:9092;host2:9092`.
+
+The following command starts `vmagent`, which reads metrics in InfluxDB line protocol format from Kafka broker at `localhost:9092` from the topic `metrics-by-telegraf` and sends them to remote storage at `http://localhost:8428/api/v1/write`:
+
+```bash
+./bin/vmagent -remoteWrite.url=http://localhost:8428/api/v1/write \
+       -kafka.consumer.topic.brokers=localhost:9092 \
+       -kafka.consumer.topic.format=influx \
+       -kafka.consumer.topic=metrics-by-telegraf \
+       -kafka.consumer.topic.groupID=some-id
+```
+
+It is expected that [Telegraf](https://github.com/influxdata/telegraf) sends metrics to the `metrics-by-telegraf` topic with the following config:
+
+```yaml
+[[outputs.kafka]]
+brokers = ["localhost:9092"]
+topic = "influx"
+data_format = "influx"
+```
+
+
+#### Command-line flags for Kafka consumer
+
+These command-line flags are available only in [enterprise](https://victoriametrics.com/products/enterprise/) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+```
+  -kafka.consumer.topic array
+        Kafka topic names for data consumption.
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.password array
+        Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.username array
+        Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.brokers array
+        List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.defaultFormat string
+        Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+  -kafka.consumer.topic.format array
+        data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.groupID array
+        Defines group.id for topic
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.isGzipped array
+        Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+        Supports array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.options array
+        Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+        Supports an array of values separated by comma or specified via multiple flags.
+```
+
+### Writing metrics to Kafka
+
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
+
+Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+
+
+#### Kafka broker authorization and authentication
+
+Two types of auth are supported:
+
+* sasl with username and password:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SASL_SSL&sasl.mechanisms=PLAIN -remoteWrite.basicAuth.username=user -remoteWrite.basicAuth.password=password
+```
+
+* tls certificates:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SSL -remoteWrite.tlsCAFile=/opt/ca.pem -remoteWrite.tlsCertFile=/opt/cert.pem -remoteWrite.tlsKeyFile=/opt/key.pem
+```
+
 
 ## How to build from sources
 
@@ -491,7 +656,7 @@ We recommend using [binary releases](https://github.com/VictoriaMetrics/Victoria
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds the `vmagent` binary and puts it into the `bin` folder.
 
@@ -520,7 +685,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent-arm` or `make vmagent-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
    It builds `vmagent-arm` or `vmagent-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -535,18 +700,26 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 `vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
 
-* Memory profile can be collected with the following command:
+* Memory profile can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
 
 ```bash
-curl -s http://<vmagent-host>:8429/debug/pprof/heap > mem.pprof
+curl http://0.0.0.0:8429/debug/pprof/heap > mem.pprof
 ```
 
-* CPU profile can be collected with the following command:
+</div>
+
+* CPU profile can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
 
 ```bash
-curl -s http://<vmagent-host>:8429/debug/pprof/profile > cpu.pprof
+curl http://0.0.0.0:8429/debug/pprof/profile > cpu.pprof
 ```
 
+</div>
+
 The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
@@ -563,16 +736,23 @@ vmagent collects metrics data via popular data ingestion protocols and routes th
 
 See the docs at https://docs.victoriametrics.com/vmagent.html .
 
+  -configAuthKey string
+    	Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+  -datadog.maxInsertRequestSize size
+    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dryRun
-    	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
+    	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -graphiteListenAddr string
@@ -604,20 +784,48 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
     	Supports an array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize size
-    	The maximum size in bytes for a single Influx line during parsing
+    	The maximum size in bytes for a single InfluxDB line during parsing
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
+  -influxDBLabel string
+    	Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
   -influxListenAddr string
-    	TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
+    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
   -influxMeasurementFieldSeparator string
-    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol (default "_")
+    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
     	Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
   -influxSkipSingleField
-    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field
+    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
   -influxTrimTimestamp duration
-    	Trim timestamps for Influx line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
     	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
+  -kafka.consumer.topic array
+    	Kafka topic names for data consumption.
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.password array
+    	Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.username array
+    	Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.brokers array
+    	List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.defaultFormat string
+    	Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+  -kafka.consumer.topic.format array
+    	data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.groupID array
+    	Defines group.id for topic
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.isGzipped array
+    	Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+    	Supports array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.options array
+    	Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -643,7 +851,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
     	TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
   -opentsdbListenAddr string
@@ -656,7 +864,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -opentsdbhttpTrimTimestamp duration
     	Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.cluster.memberNum int
     	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
   -promscrape.cluster.membersCount int
@@ -664,11 +872,11 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.cluster.replicationFactor int
     	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
     	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
   -promscrape.config.strictParse
-    	Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped
+    	Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
   -promscrape.configCheckInterval duration
     	Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
   -promscrape.consul.waitTime duration
@@ -698,7 +906,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.eurekaSDCheckInterval duration
     	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
@@ -709,11 +917,17 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets int
     	The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+  -promscrape.maxResponseHeadersSize size
+    	The maximum size of http response headers from Prometheus scrape targets
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
+  -promscrape.minResponseSizeForStreamParse size
+    	The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
   -promscrape.noStaleMarkers
-    	Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+    	Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
     	Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
   -promscrape.seriesLimitPerTarget int
@@ -745,7 +959,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.maxBlockSize size
-    	The maximum size in bytes of unpacked request to send to remote storage. It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics
+    	The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
   -remoteWrite.maxDailySeries int
     	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
@@ -754,6 +968,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -remoteWrite.maxHourlySeries int
     	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+  -remoteWrite.maxRowsPerBlock int
+    	The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
   -remoteWrite.multitenantURL array
     	Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
     	Supports an array of values separated by comma or specified via multiple flags.
@@ -781,7 +997,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
     	Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
-    	Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+    	Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
   -remoteWrite.relabelDebug
     	Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
@@ -816,7 +1032,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
-    	Optional path to relabel config for the corresponding -remoteWrite.url
+    	Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelDebug array
     	Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
@@ -826,9 +1042,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,99 @@
+package datadog
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
+		for _, tag := range ss.Tags {
+			n := strings.IndexByte(tag, ':')
+			if n < 0 {
+				return fmt.Errorf("cannot find ':' in tag %q", tag)
+			}
+			name := tag[:n]
+			value := tag[n+1:]
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -21,9 +21,10 @@ import (
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
+	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )
 
 var (
@@ -35,9 +36,9 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(r io.Reader) error {
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 			return insertRows(nil, db, rows, nil)
 		})
 	})
@@ -80,7 +81,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			if tag.Key == "db" {
+			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
 			commonLabels = append(commonLabels, prompbmarshal.Label{
@@ -90,7 +91,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		}
 		if len(db) > 0 && !hasDBKey {
 			commonLabels = append(commonLabels, prompbmarshal.Label{
-				Name:  "db",
+				Name:  *dbLabel,
 				Value: db,
 			})
 		}

app/vmagent/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
@@ -41,16 +43,17 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
 var (
@@ -93,7 +96,9 @@ func main() {
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
+		})
 	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
@@ -153,10 +158,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/metrics", "available service metrics"},
-			{"/-/reload", "reload configuration"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}
@@ -221,13 +228,64 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/target_response":
+		promscrapeTargetResponseRequests.Inc()
+		if err := promscrape.WriteTargetResponse(w, r); err != nil {
+			promscrapeTargetResponseErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
@@ -327,6 +385,35 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -349,14 +436,26 @@ var (
 	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
 
 	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
+	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
+	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
+
+	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 

app/vmagent/prometheusimport/request_handler.go

@@ -1,6 +1,7 @@
 package prometheusimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -38,6 +39,15 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader with optional gzip format
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		}, nil)
+	})
+}
+
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmagent/promremotewrite/request_handler.go

@@ -1,6 +1,7 @@
 package promremotewrite
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -29,12 +30,21 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
 			return insertRows(at, tss, extraLabels)
 		})
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, nil)
+		})
+	})
+}
+
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmagent/remotewrite/client.go

@@ -67,7 +67,8 @@ type client struct {
 	fq             *persistentqueue.FastQueue
 	hc             *http.Client
 
-	authCfg *promauth.Config
+	sendBlock func(block []byte) bool
+	authCfg   *promauth.Config
 
 	rl rateLimiter
 
@@ -84,7 +85,7 @@ type client struct {
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
 		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
@@ -104,11 +105,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		if !strings.Contains(pURL, "://") {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
 		}
-		urlProxy, err := url.Parse(pURL)
+		pu, err := url.Parse(pURL)
 		if err != nil {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
 		}
-		tr.Proxy = http.ProxyURL(urlProxy)
+		tr.Proxy = http.ProxyURL(pu)
 	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
@@ -121,6 +122,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		},
 		stopCh: make(chan struct{}),
 	}
+	c.sendBlock = c.sendBlockHTTP
+	return c
+}
+
+func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
@@ -143,7 +149,6 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		}()
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
-	return c
 }
 
 func (c *client) MustStop() {
@@ -160,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if username != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
-			Password:     password,
+			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
 	}
@@ -174,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if clientSecretFile != "" || clientSecret != "" {
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     clientSecret,
+			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
@@ -237,9 +242,9 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlock returns false only if c.stopCh is closed.
+// sendBlockHTTP returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlock(block []byte) bool {
+func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
 	retryDuration := time.Second
 	retriesCount := 0
@@ -249,7 +254,7 @@ func (c *client) sendBlock(block []byte) bool {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -290,6 +295,17 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
+		body, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+		if err != nil {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
+		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149

app/vmagent/remotewrite/pendingseries.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -20,16 +21,10 @@ import (
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
-		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
+	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 )
 
-// the maximum number of rows to send per each block.
-const maxRowsPerBlock = 10000
-
-// the maximum number of labels to send per each block.
-const maxLabelsPerBlock = 10 * maxRowsPerBlock
-
 type pendingSeries struct {
 	mu sync.Mutex
 	wr writeRequest
@@ -153,10 +148,13 @@ func (wr *writeRequest) adjustSampleValues() {
 
 func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
-		if len(wr.samples) >= maxRowsPerBlock || len(wr.labels) >= maxLabelsPerBlock {
+		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss
@@ -213,7 +211,22 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		writeRequestBufPool.Put(bb)
 	}
 
-	// Too big block. Recursively split it into smaller parts.
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
+		samples := wr.Timeseries[0].Samples
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return
+		}
+		n := len(samples) / 2
+		wr.Timeseries[0].Samples = samples[:n]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples[n:]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples
+		return
+	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]

app/vmagent/remotewrite/relabel.go

@@ -15,12 +15,14 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
-	relabelDebug       = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
 )

app/vmagent/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -170,28 +171,32 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
 
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / maxRowsPerBlock / 100
-	if maxInmemoryBlocks > 400 {
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
 		// There is no much sense in keeping higher number of blocks in memory,
 		// since this means that the producer outperforms consumer and the queue
 		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 400
+		maxInmemoryBlocks = 100 * *queues
 	}
 	if maxInmemoryBlocks < 2 {
 		maxInmemoryBlocks = 2
 	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
-	for i, remoteWriteURL := range urls {
+	for i, remoteWriteURLRaw := range urls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
 		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if at != nil {
 			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
-			remoteWriteURL = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL, at.AccountID, at.ProjectID)
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
 			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
 		}
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
 	return rwctxs
 }
@@ -269,6 +274,9 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -278,7 +286,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			samplesCount += len(tss[i].Samples)
 			labelsCount += len(tss[i].Labels)
 			i++
-			if samplesCount >= maxRowsPerBlock || labelsCount >= maxLabelsPerBlock {
+			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
 			}
 		}
@@ -296,11 +304,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if len(tssBlock) > 0 {
-			for _, rwctx := range rwctxs {
-				rwctx.Push(tssBlock)
-			}
-		}
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
 		if rctx != nil {
 			rctx.reset()
 		}
@@ -310,6 +314,23 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	}
 }
 
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+	if len(tssBlock) == 0 {
+		// Nothing to push
+		return
+	}
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	for _, rwctx := range rwctxs {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			rwctx.Push(tssBlock)
+		}(rwctx)
+	}
+	wg.Wait()
+}
+
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
 func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
 	if !*sortLabels {
@@ -369,6 +390,8 @@ var labelsHashBufPool bytesutil.ByteBufferPool
 func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
+		// Do not use logger.WithThrottler() here, since this will increase CPU usage
+		// because every call to logSkippedSeries will result to a call to labelsToString.
 		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
 	default:
 	}
@@ -403,17 +426,29 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
-	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+	// strip query params, otherwise changing params resets pq
+	pqURL := *remoteWriteURL
+	pqURL.RawQuery = ""
+	pqURL.Fragment = ""
+	h := xxhash.Sum64([]byte(pqURL.String()))
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
+	var c *client
+	switch remoteWriteURL.Scheme {
+	case "http", "https":
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+	default:
+		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+	}
+	c.init(argIdx, *queues, sanitizedURL)
+
 	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
 	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
@@ -432,7 +467,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 

app/vmagent/vmimport/request_handler.go

@@ -1,6 +1,7 @@
 package vmimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -31,12 +32,22 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
 			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		})
+	})
+}
+
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmalert/Makefile

@@ -27,6 +27,15 @@ vmalert-ppc64le-prod:
 vmalert-386-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-386
 
+vmalert-darwin-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
+
+vmalert-darwin-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
+
+vmalert-windows-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
+
 package-vmalert:
 	APP_NAME=vmalert $(MAKE) package-via-docker
 
@@ -70,6 +79,13 @@ run-vmalert: vmalert
 		-evaluationInterval=3s \
 		-rule.configCheckInterval=10s
 
+run-vmalert-sd: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-configCheckInterval=10s
+
 replay-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
@@ -102,6 +118,3 @@ vmalert-pure:
 
 vmalert-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch
-
-vmalert-windows-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64

app/vmalert/README.md

@@ -2,7 +2,11 @@
 
 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured address. It is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
+rules against configured `-datasource.url`. For sending alerting notifications
+vmalert relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag.
+Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+protocol and require `-remoteWrite.url` to be configured.
+Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
 ## Features
@@ -11,20 +15,19 @@ implementation and aims to be compatible with its syntax.
  support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
  support;
-* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-aplha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
 * Keeps the alerts [state on restarts](#alerts-state-on-restarts);
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight without extra dependencies.
 
 ## Limitations
-* `vmalert` execute queries against remote datasource which has reliability risks because of network.
-It is recommended to configure alerts thresholds and rules expressions with understanding that network request
-may fail;
-* by default, rules execution is sequential within one group, but persisting of execution results to remote
-storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
-recording rule is reused in next one;
-* `vmalert` has no UI, just an API for getting groups and rules statuses.
+* `vmalert` execute queries against remote datasource which has reliability risks because of the network.
+It is recommended to configure alerts thresholds and rules expressions with the understanding that network 
+requests may fail;
+* by default, rules execution is sequential within one group, but persistence of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
+recording rule is reused in the next one;
 
 ## QuickStart
 
@@ -34,29 +37,37 @@ git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
 ```
-The build binary will be placed to `VictoriaMetrics/bin` folder.
+The build binary will be placed in `VictoriaMetrics/bin` folder.
 
 To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable VictoriaMetrics instance for rules execution;
-* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts and sending notifications.
+* datasource address - reachable MetricsQL endpoint to run queries against;
+* notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
+aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via 
+[config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-compatible storage address for storing recording rules results and alerts state in for of timeseries.
+  compatible storage to persist rules and alerts state info;
+* remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
 ```
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
-    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
-    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules
+    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
     -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
     -external.label=cluster=east-1 \         # External label to be applied for each rule
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
-See the fill list of configuration flags in [configuration](#configuration) section.
+Note there's a separate `remoteRead.url` to allow writing results of
+alerting/recording rules into a different storage than the initial data that's
+queried. This allows using `vmalert` to aggregate data from a short-term,
+high-frequency, high-cardinality storage into a long-term storage with
+decreased cardinality and a bigger interval between samples.
+
+See the full list of configuration flags in [configuration](#configuration) section.
 
 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
 to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts.
@@ -86,15 +97,27 @@ name: <string>
 [ concurrency: <integer> | default = 1 ]
 
 # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
-# By default "prometheus" rule type is used.
+# By default "prometheus" type is used.
 [ type: <string> ]
 
-# Optional list of label filters applied to every rule's
-# request withing a group. Is compatible only with VM datasource.
-# See more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+# Warning: DEPRECATED
+# Please use `params` instead:
+#   params:
+#     extra_label: ["job=nodeexporter", "env=prod"]
 extra_filter_labels:
   [ <labelname>: <labelvalue> ... ]
 
+# Optional list of HTTP URL parameters
+# applied for all rules requests within a group
+# For example:
+#  params:
+#    nocache: ["1"]                # disable caching for vmselect
+#    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+# see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+params:
+  [ <string>: [<string>, ...]]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
@@ -114,14 +137,14 @@ expression and then act according to the Rule type.
 
 There are two types of Rules:
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allows to define alert conditions via `expr` field and to send notifications
+Alerting rules allow defining alert conditions via `expr` field and to send notifications to
 [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allows to define `expr` which result will be than backfilled to configured
+Recording rules allow defining `expr` which result will be then backfilled to configured
 `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
 expensive expressions and save their result as a new set of time series.
 
-`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.
 
 #### Alerting rules
@@ -131,17 +154,13 @@ The syntax for alerting rule is the following:
 # The name of the alert. Must be a valid metric name.
 alert: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default PromQL/MetricsQL expression is used. If type="graphite", then the expression
+# By default PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
 # Alerts are considered firing once they have been returned for this long.
-# Alerts which have not yet fired for long enough are considered pending.
+# Alerts which have not yet been fired for long enough are considered pending.
 # If param is omitted or set to 0 then alerts will be immediately considered
 # as firing once they return.
 [ for: <duration> | default = 0s ]
@@ -167,12 +186,8 @@ The syntax for recording rules is following:
 # The name of the time series to output to. Must be a valid metric name.
 record: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If type="graphite", then the expression
+# By default MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
@@ -190,19 +205,19 @@ For recording rules to work `-remoteWrite.url` must be specified.
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
 into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and may be queried from VM just as any other time series.
-The state stored to the configured address on every rule evaluation.
+These are regular time series and maybe queried from VM just as any other time series.
+The state is stored to the configured address on every rule evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
 from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
-Both flags are required for the proper state restoring. Restore process may fail if time series are missing
+Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
 or received state doesn't match current `vmalert` rules configuration.
 
 
 ### Multitenancy
 
-There are the following approaches for alerting and recording rules across
+There are the following approaches exist for alerting and recording rules across
 [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy):
 
 * To run a separate `vmalert` instance per each tenant.
@@ -215,7 +230,7 @@ There are the following approaches for alerting and recording rules across
   rules to `AccountID=123`.
 
 * To specify `tenant` parameter per each alerting and recording group if
-  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used
+  [enterprise version of vmalert](https://victoriametrics.com/products/enterprise/) is used
   with `-clusterMode` command-line flag. For example:
 
 ```yaml
@@ -233,16 +248,126 @@ groups:
 
 If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
 contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
-`vmselect` automatically adds the specified tenant to urls per each recording rule in this case.
+`vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
+
+If `-clusterMode` is enabled and the `tenant` in a particular group is missing, then the tenant value
+is obtained from `-defaultTenant.prometheus` or `-defaultTenant.graphite` depending on the `type` of the group.
 
 The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
+### Topology examples
 
-### WEB
+The following sections are showing how `vmalert` may be used and configured 
+for different scenarios. 
+
+Please note, not all flags in examples are required: 
+* `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
+you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+* `-notifier.url` is optional and is needed only if you have alerting rules.
+
+#### Single-node VictoriaMetrics
+
+The simplest configuration where one single-node VM server is used for
+rules execution, storing recording rules results and alerts state.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                    # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093          # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert single" width="500" src="vmalert_single.png">
+
+
+#### Cluster VictoriaMetrics
+
+In [cluster mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html)
+VictoriaMetrics has separate components for writing and reading path:
+`vminsert` and `vmselect` components respectively. `vmselect` is used for executing rules expressions
+and `vminsert` is used to persist recording rules results and alerts state.
+Cluster mode could have multiple `vminsert` and `vmselect` components. 
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                                # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for executing rules expressions
+    -remoteWrite.url=http://vminsert:8480/insert/0/prometheus   # vminsert addr to persist alerts state and recording rules results
+    -remoteRead.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093                      # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert cluster" src="vmalert_cluster.png">
+
+In case when you want to spread the load on these components - add balancers before them and configure
+`vmalert` with balancer's addresses. Please, see more about VM's cluster architecture
+[here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview).
+
+#### HA vmalert
+
+For HA user can run multiple identically configured `vmalert` instances.
+It means all of them will execute the same rules, write state and results to
+the same destinations, and send alert notifications to multiple configured
+Alertmanagers.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml \                   # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager1:9093 \       # Multiple AlertManager addresses to send alerts when they trigger
+    -notifier.url=http://alertmanagerN:9093         # The same alert will be sent to all configured notifiers
+```
+
+<img alt="vmalert ha" width="800px" src="vmalert_ha.png">
+
+To avoid recording rules results and alerts state duplication in VictoriaMetrics server
+don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+
+Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
+all `vmalert`s are having the same config.
+
+Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
+for Alertmanagers for better reliability.
+
+This example uses single-node VM server for the sake of simplicity. 
+Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. 
+
+
+#### Downsampling and aggregation via vmalert
+
+The following example shows how to build a topology where `vmalert` will process data from one cluster
+and write results into another. Such clusters may be called as "hot" (low retention,
+high-speed disks, used for operative monitoring) and "cold" (long term retention,
+slower/cheaper disks, low resolution data). With help of `vmalert`, user can setup
+recording rules to process raw data from "hot" cluster (by applying additional transformations
+or reducing resolution) and push results to "cold" cluster.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus    # vminsert addr to persist recording rules results
+```
+
+<img alt="vmalert multi cluster" src="vmalert_multicluster.png">
+
+Please note, [replay](#rules-backfilling) feature may be used for transforming historical data.
+
+Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only recording rules are used.
+
+See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
+
+
+### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
 * `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status" ` - get alert status by ID.
@@ -257,12 +382,12 @@ vmalert sends requests to `<-datasource.url>/render?format=json` during evaluati
 if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
 implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
 When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
-to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on query type.
+to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
 
 ## Rules backfilling
 
 vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert
-can read the same rules configuration as normally, evaluate them on the given time range and backfill
+can read the same rules configuration as normal, evaluate them on the given time range and backfill
 results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
 data source for backfilling.
 
@@ -308,19 +433,19 @@ max range per request:  8h20m0s
 In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
 executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
 to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
-of the configured `-datasource.url`. Returned data then processed according to the rule type and
-backfilled to `-remoteWrite.url` via [Remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
+of the configured `-datasource.url`. Returned data is then processed according to the rule type and
+backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
 Vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
 to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
 
 #### Recording rules
 
-Result of recording rules `replay` should match with results of normal rules evaluation.
+The result of recording rules `replay` should match with results of normal rules evaluation.
 
 #### Alerting rules
 
-Result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
+The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
 To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
 ```
 ALERTS{alertname="your_alertname", alertstate="firing"}
@@ -333,7 +458,7 @@ There are following non-required `replay` flags:
 
 * `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
 In two words, it affects the max time range for every `/query_range` request. The higher the value,
-the less requests will be issued during `replay`.
+the fewer requests will be issued during `replay`.
 * `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
 per rule before giving up.
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
@@ -351,34 +476,58 @@ See full description for these flags in `./vmalert --help`.
 
 ## Monitoring
 
-`vmalert` exports various metrics in Prometheus exposition format at `http://vmalert-host:8880/metrics` page. 
-We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported 
+`vmalert` exports various metrics in Prometheus exposition format at `http://vmalert-host:8880/metrics` page.
+We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
 metrics may be analyzed later.
 
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview.
-If you have suggestions for improvements or have found a bug - please open an issue on github or add 
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
+If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.
 
 
 ## Configuration
 
+### Flags
+
 Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.
 
 The shortlist of configuration flags is the following:
 ```
+  -clusterMode
+    	If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+  -configCheckInterval duration
+    	Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
   -datasource.appendTypePrefix
     	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
   -datasource.basicAuth.password string
     	Optional basic auth password for -datasource.url
+  -datasource.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.bearerToken string
+    	Optional bearer auth token to use for -datasource.url.
+  -datasource.bearerTokenFile string
+    	Optional path to bearer token file to use for -datasource.url.
   -datasource.lookback duration
     	Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
   -datasource.maxIdleConnections int
     	Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
+  -datasource.oauth2.clientID string
+    	Optional OAuth2 clientID to use for -datasource.url. 
+  -datasource.oauth2.clientSecret string
+    	Optional OAuth2 clientSecret to use for -datasource.url.
+  -datasource.oauth2.clientSecretFile string
+    	Optional OAuth2 clientSecretFile to use for -datasource.url. 
+  -datasource.oauth2.scopes string
+    	Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
+  -datasource.oauth2.tokenUrl string
+    	Optional OAuth2 tokenURL to use for -datasource.url.
   -datasource.queryStep duration
     	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+  -datasource.queryTimeAlignment
+    	Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
     	Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
   -datasource.tlsCAFile string
@@ -393,8 +542,12 @@ The shortlist of configuration flags is the following:
     	Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
     	VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+  -defaultTenant.graphite string
+    	Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+  -defaultTenant.prometheus string
+    	Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -disableAlertgroupLabel
-    	Whether to disable adding group's name as label to generated alerts and time series.
+    	Whether to disable adding group's Name as label to generated alerts and time series.
   -dryRun -rule
     	Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
@@ -403,13 +556,15 @@ The shortlist of configuration flags is the following:
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -evaluationInterval duration
     	How often to evaluate the rules (default 1m0s)
   -external.alert.source string
     	External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
     	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
   -external.label array
-    	Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
+    	Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
     	Supports an array of values separated by comma or specified via multiple flags.
   -external.url string
     	External URL is used as alert's source for sent alerts to the notifier
@@ -453,13 +608,41 @@ The shortlist of configuration flags is the following:
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
     	Optional basic auth password for -notifier.url
     	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.basicAuth.passwordFile array
+    	Optional path to basic auth password file for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.basicAuth.username array
     	Optional basic auth username for -notifier.url
     	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.bearerToken array
+    	Optional bearer token for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.bearerTokenFile array
+    	Optional path to bearer token file for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.config string
+    	Path to configuration file for notifiers
+  -notifier.oauth2.clientID array
+    	Optional OAuth2 clientID to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.clientSecret array
+    	Optional OAuth2 clientSecret to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.clientSecretFile array
+    	Optional OAuth2 clientSecretFile to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.scopes array
+    	Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.tokenUrl array
+    	Optional OAuth2 tokenURL to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.suppressDuplicateTargetErrors
+    	Whether to suppress 'duplicate target' errors during discovery
   -notifier.tlsCAFile array
     	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
     	Supports an array of values separated by comma or specified via multiple flags.
@@ -476,18 +659,44 @@ The shortlist of configuration flags is the following:
     	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
     	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+    	Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
     	Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -promscrape.consul.waitTime duration
+    	Wait time used by Consul service discovery. Default value is used if not set
+  -promscrape.consulSDCheckInterval duration
+    	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+  -promscrape.discovery.concurrency int
+    	The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
+  -promscrape.discovery.concurrentWaitTime duration
+    	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
   -remoteRead.basicAuth.password string
     	Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteRead.url
   -remoteRead.basicAuth.username string
     	Optional basic auth username for -remoteRead.url
+  -remoteRead.bearerToken string
+    	Optional bearer auth token to use for -remoteRead.url.
+  -remoteRead.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteRead.url.
+  -remoteRead.disablePathAppend
+    	Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.
   -remoteRead.ignoreRestoreErrors
     	Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
   -remoteRead.lookback duration
     	Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteRead.oauth2.clientID string
+    	Optional OAuth2 clientID to use for -remoteRead.url.
+  -remoteRead.oauth2.clientSecret string
+    	Optional OAuth2 clientSecret to use for -remoteRead.url.
+  -remoteRead.oauth2.clientSecretFile string
+    	Optional OAuth2 clientSecretFile to use for -remoteRead.url.
+  -remoteRead.oauth2.scopes string
+    	Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
+  -remoteRead.oauth2.tokenUrl string
+    	Optional OAuth2 tokenURL to use for -remoteRead.url. 
   -remoteRead.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
@@ -499,11 +708,17 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
     	Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-    	Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+    	Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
   -remoteWrite.basicAuth.password string
     	Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteWrite.url
   -remoteWrite.basicAuth.username string
     	Optional basic auth username for -remoteWrite.url
+  -remoteWrite.bearerToken string
+    	Optional bearer auth token to use for -remoteWrite.url.
+  -remoteWrite.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteWrite.url.
   -remoteWrite.concurrency int
     	Defines number of writers for concurrent writing into remote querier (default 1)
   -remoteWrite.disablePathAppend
@@ -514,6 +729,16 @@ The shortlist of configuration flags is the following:
     	Defines defines max number of timeseries to be flushed at once (default 1000)
   -remoteWrite.maxQueueSize int
     	Defines the max number of pending datapoints to remote write endpoint (default 100000)
+  -remoteWrite.oauth2.clientID string
+    	Optional OAuth2 clientID to use for -remoteWrite.url.
+  -remoteWrite.oauth2.clientSecret string
+    	Optional OAuth2 clientSecret to use for -remoteWrite.url.
+  -remoteWrite.oauth2.clientSecretFile string
+    	Optional OAuth2 clientSecretFile to use for -remoteWrite.url.
+  -remoteWrite.oauth2.scopes string
+    	Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
+  -remoteWrite.oauth2.tokenUrl string
+    	Optional OAuth2 tokenURL to use for -notifier.url.
   -remoteWrite.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
@@ -546,7 +771,11 @@ The shortlist of configuration flags is the following:
     	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
     	Supports an array of values separated by comma or specified via multiple flags.
   -rule.configCheckInterval duration
-    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
+  -rule.maxResolveDuration duration
+    	Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
+  -rule.resendDelay duration
+    	Minimum amount of time to wait before resending an alert to notifier
   -rule.validateExpressions
     	Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
@@ -554,19 +783,121 @@ The shortlist of configuration flags is the following:
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```
 
+### Hot config reload
 `vmalert` supports "hot" config reload via the following methods:
 * send SIGHUP signal to `vmalert` process;
 * send GET request to `/-/reload` endpoint;
-* configure `-rule.configCheckInterval` flag for periodic reload
+* configure `-configCheckInterval` flag for periodic reload
 on config change.
 
+### URL params
+
+To set additional URL params for `datasource.url`, `remoteWrite.url` or `remoteRead.url`
+just add them in address: `-datasource.url=http://localhost:8428?nocache=1`.
+
+To set additional URL params for specific [group of rules](#Groups) modify
+the `params` group:
+```yaml
+groups:
+  - name: TestGroup
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
+```
+Please note, `params` are used only for executing rules expressions (requests to `datasource.url`).
+If there would be a conflict between URL params set in `datasource.url` flag and params in group definition
+the latter will have higher priority.
+
+### Notifier configuration file
+
+Notifier also supports configuration via file specified with flag `notifier.config`:
+```
+./bin/vmalert -rule=app/vmalert/config/testdata/rules.good.rules \
+		-datasource.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
+```
+
+The configuration file allows to configure static notifiers or discover notifiers via 
+[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config).
+For example:
+```
+static_configs: 
+  - targets:
+      - localhost:9093
+      - localhost:9095
+
+consul_sd_configs:
+  - server: localhost:8500
+    services:
+      - alertmanager
+```
+
+The list of configured or discovered Notifiers can be explored via [UI](#Web).
+
+The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go)
+is the following:
+```
+# Per-target Notifier timeout when pushing alerts.
+[ timeout: <duration> | default = 10s ]
+
+# Prefix for the HTTP path alerts are pushed to.
+[ path_prefix: <path> | default = / ]
+
+# Configures the protocol scheme used for requests.
+[ scheme: <scheme> | default = http ]
+
+# Sets the `Authorization` header on every request with the
+# configured username and password.
+# password and password_file are mutually exclusive.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# Optional `Authorization` header configuration.
+authorization:
+  # Sets the authentication type.
+  [ type: <string> | default: Bearer ]
+  # Sets the credentials. It is mutually exclusive with
+  # `credentials_file`.
+  [ credentials: <secret> ]
+  # Sets the credentials to the credentials read from the configured file.
+  # It is mutually exclusive with `credentials`.
+  [ credentials_file: <filename> ]
+
+# Configures the scrape request's TLS settings.
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
+tls_config:
+  [ <tls_config> ]
+
+# List of labeled statically configured Notifiers.
+static_configs:
+  targets:
+    [ - '<host>' ]
+
+# List of Consul service discovery configurations.
+# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+consul_sd_configs:
+  [ - <consul_sd_config> ... ]
+
+# List of relabel configurations.
+# Supports the same relabeling features as the rest of VictoriaMetrics components.
+# See https://docs.victoriametrics.com/vmagent.html#relabeling
+relabel_configs:
+  [ - <relabel_config> ... ]
+
+```
+
+The configuration file can be [hot-reloaded](#hot-config-reload).
+
+
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.
@@ -582,7 +913,7 @@ It is recommended using
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
@@ -599,7 +930,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmalert/alerting.go

@@ -12,9 +12,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // AlertingRule is basic alert entity
@@ -38,6 +38,8 @@ type AlertingRule struct {
 	alerts map[uint64]*notifier.Alert
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -50,15 +52,15 @@ type AlertingRule struct {
 }
 
 type alertingRuleMetrics struct {
-	errors  *gauge
-	pending *gauge
-	active  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	pending *utils.Gauge
+	active  *utils.Gauge
+	samples *utils.Gauge
 }
 
 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
-		Type:         cfg.Type,
+		Type:         group.Type,
 		RuleID:       cfg.ID,
 		Name:         cfg.Alert,
 		Expr:         cfg.Expr,
@@ -69,16 +71,16 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
 		metrics: &alertingRuleMetrics{},
 	}
 
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
-	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -90,7 +92,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -102,7 +104,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -111,7 +113,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return 1
 		})
-	ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -122,10 +124,10 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 
 // Close unregisters rule metrics
 func (ar *AlertingRule) Close() {
-	metrics.UnregisterMetric(ar.metrics.active.name)
-	metrics.UnregisterMetric(ar.metrics.pending.name)
-	metrics.UnregisterMetric(ar.metrics.errors.name)
-	metrics.UnregisterMetric(ar.metrics.samples.name)
+	ar.metrics.active.Unregister()
+	ar.metrics.pending.Unregister()
+	ar.metrics.errors.Unregister()
+	ar.metrics.samples.Unregister()
 }
 
 // String implements Stringer interface
@@ -153,6 +155,13 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
+		// set additional labels to identify group and rule Name
+		if ar.Name != "" {
+			s.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			s.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(s, qFn, ar)
 		if err != nil {
@@ -163,7 +172,6 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			// so the hash key will be consistent on restore
 			s.SetLabel(k, v)
 		}
-
 		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
@@ -178,13 +186,11 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 
 		// if alert with For > 0
 		prevT := time.Time{}
-		//activeAt := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
-				//activeAt = at
 				a.Start = at
 			} else if at.Sub(a.Start) >= ar.For {
 				a.State = notifier.StateFiring
@@ -199,12 +205,14 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
 func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
 	qMetrics, err := ar.q.Query(ctx, ar.Expr)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
 	ar.lastExecError = err
-	ar.lastExecTime = time.Now()
 	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
@@ -221,6 +229,13 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			m.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			m.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(m, qFn, ar)
 		if err != nil {
@@ -352,11 +367,6 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notif
 		Start:   start,
 		Expr:    ar.Expr,
 	}
-	// label defined here to make override possible by
-	// time series labels.
-	if !*disableAlertGroupLabel && ar.GroupName != "" {
-		a.Labels[alertGroupNameLabel] = ar.GroupName
-	}
 	for _, l := range m.Labels {
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
@@ -380,31 +390,48 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// RuleAPI returns Rule representation in form
-// of APIAlertingRule
-func (ar *AlertingRule) RuleAPI() APIAlertingRule {
-	var lastErr string
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
 	if ar.lastExecError != nil {
-		lastErr = ar.lastExecError.Error()
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
 	}
-	return APIAlertingRule{
-		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", ar.ID()),
-		GroupID:     fmt.Sprintf("%d", ar.GroupID),
-		Type:        ar.Type.String(),
-		Name:        ar.Name,
-		Expression:  ar.Expr,
-		For:         ar.For.String(),
-		LastError:   lastErr,
-		LastSamples: ar.lastExecSamples,
-		LastExec:    ar.lastExecTime,
-		Labels:      ar.Labels,
-		Annotations: ar.Annotations,
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
 	}
+	return r
 }
 
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
 	ar.mu.RLock()
 	for _, a := range ar.alerts {
@@ -415,10 +442,11 @@ func (ar *AlertingRule) AlertsAPI() []*APIAlert {
 }
 
 func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
-	return &APIAlert{
+	aa := &APIAlert{
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", a.ID),
 		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
 
 		Name:        a.Name,
 		Expression:  ar.Expr,
@@ -426,8 +454,13 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Annotations: a.Annotations,
 		State:       a.State.String(),
 		ActiveAt:    a.Start,
-		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
 }
 
 const (
@@ -442,43 +475,42 @@ const (
 	alertStateLabel = "alertstate"
 
 	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
 	alertGroupNameLabel = "alertgroup"
 )
 
 // alertToTimeSeries converts the given alert with the given timestamp to timeseries
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	tss = append(tss, alertToTimeSeries(a, timestamp))
 	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
 	return tss
 }
 
-func alertToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertMetricName
-	labels[alertNameLabel] = name
 	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
 // alertForToTimeSeries returns a timeseries that represents
 // state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	labels[alertNameLabel] = name
 	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores the state of active alerts basing on previously written time series.
 // Restore restores only Start field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
@@ -506,25 +538,38 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	}
 
 	for _, m := range qMetrics {
-		labels := m.Labels
-		m.Labels = make([]datasource.Label, 0)
-		// drop all extra labels, so hash key will
-		// be identical to time series received in Exec
-		for _, l := range labels {
-			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
-				continue
-			}
-			m.Labels = append(m.Labels, l)
-		}
-
 		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = hash(m)
 		a.State = notifier.StatePending
+		a.Restored = true
 		ar.alerts[a.ID] = a
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
 	}
 	return nil
 }
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		switch a.State {
+		case notifier.StateFiring:
+			if time.Since(a.LastSent) < resendDelay {
+				continue
+			}
+			a.End = ts.Add(resolveDuration)
+			a.LastSent = ts
+			alerts = append(alerts, *a)
+		case notifier.StateInactive:
+			a.End = ts
+			a.LastSent = ts
+			alerts = append(alerts, *a)
+		}
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -27,7 +28,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant",
 				}),
 			},
 		},
@@ -41,7 +41,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant extra labels",
 					"job":           "foo",
 					"instance":      "bar",
 				}),
@@ -57,7 +56,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant labels override",
 				}),
 			},
 		},
@@ -68,13 +66,11 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "for",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
@@ -85,13 +81,11 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StatePending.String(),
-					alertNameLabel:  "for pending",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for pending",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
@@ -109,23 +103,27 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 
 func TestAlertingRule_Exec(t *testing.T) {
 	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
 	testCases := []struct {
 		rule      *AlertingRule
 		steps     [][]datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
+		expAlerts []testAlert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
 			[][]datasource.Metric{},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("empty labels", 0),
 			[][]datasource.Metric{
 				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 			},
-			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -133,8 +131,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -143,8 +141,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -154,8 +152,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -166,8 +164,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -179,7 +177,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -191,8 +189,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -204,10 +202,10 @@ func TestAlertingRule_Exec(t *testing.T) {
 					metricWithLabels(t, "name", "foo2"),
 				},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -220,9 +218,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
 			// 3: fire third alert, set second inactive, delete first one
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -230,8 +228,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -240,8 +238,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -252,7 +250,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset and delete pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
@@ -262,8 +260,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -275,8 +273,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -289,8 +287,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 	}
@@ -312,7 +310,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
 				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
 			}
-			for key, exp := range tc.expAlerts {
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := ta.labels
+				labels = append(labels, alertNameLabel)
+				labels = append(labels, tc.rule.Name)
+				h := hash(metricWithLabels(t, labels...))
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
 				got, ok := tc.rule.alerts[key]
 				if !ok {
 					t.Fatalf("expected to have key %d", key)
@@ -468,6 +474,11 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			var j int
 			for _, series := range tc.data {
 				for _, timestamp := range series.Timestamps {
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
 					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
 					j++
 				}
@@ -496,7 +507,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 				),
 			},
 			map[uint64]*notifier.Alert{
@@ -509,7 +519,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
+					alertNameLabel, "metric labels",
 					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
@@ -517,6 +527,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(metricWithLabels(t,
+					alertNameLabel, "metric labels",
+					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				)): {State: notifier.StatePending,
@@ -528,7 +540,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
 					// extra labels set by rule
@@ -645,18 +656,20 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "foo")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "foo",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "bar")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "bar",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
 					},
 				},
 			},
@@ -679,20 +692,22 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "foo")): {
 					Labels: map[string]string{
-						"instance": "foo",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "foo",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "foo" for region east`,
 						"description": `It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "bar")): {
 					Labels: map[string]string{
-						"instance": "bar",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "bar",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "bar" for region east`,
@@ -701,6 +716,44 @@ func TestAlertingRule_Template(t *testing.T) {
 				},
 			},
 		},
+		{
+			&AlertingRule{
+				Name:      "ExtraTemplating",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"name":     "alert_{{ $labels.alertname }}",
+					"group":    "group_{{ $labels.alertgroup }}",
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
+					"name", "alert_ExtraTemplating",
+					alertGroupNameLabel, "Testing",
+					"group", "group_Testing",
+					"instance", "foo")): {
+					Labels: map[string]string{
+						alertNameLabel:      "ExtraTemplating",
+						"name":              "alert_ExtraTemplating",
+						alertGroupNameLabel: "Testing",
+						"group":             "group_Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
+						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+					},
+				},
+			},
+		},
 	}
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {
@@ -729,6 +782,76 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 }
 
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+}
+
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)

app/vmalert/config/config.go

@@ -5,17 +5,19 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
+	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // Group contains list of Rules grouped into
@@ -24,12 +26,13 @@ type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
 	Name        string             `yaml:"name"`
-	Interval    utils.PromDuration `yaml:"interval,omitempty"`
+	Interval    promutils.Duration `yaml:"interval"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+	// DEPRECATED: use Params field instead
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
@@ -37,6 +40,8 @@ type Group struct {
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -56,12 +61,20 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if g.Type.Get() == "" {
 		g.Type.Set(datasource.NewPrometheusType())
 	}
-	// update rules with empty type.
-	for i, r := range g.Rules {
-		if r.Type.Get() == "" {
-			r.Type.Set(g.Type)
-			r.ID = HashRule(r)
-			g.Rules[i] = r
+
+	// backward compatibility with deprecated `ExtraFilterLabels` param
+	if len(g.ExtraFilterLabels) > 0 {
+		if g.Params == nil {
+			g.Params = url.Values{}
+		}
+		// Sort extraFilters for consistent order for query args across runs.
+		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
+		for k, v := range g.ExtraFilterLabels {
+			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(extraFilters)
+		for _, extraFilter := range extraFilters {
+			g.Params.Add("extra_label", extraFilter)
 		}
 	}
 
@@ -76,9 +89,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
-	if len(g.Rules) == 0 {
-		return fmt.Errorf("group %q can't contain no rules", g.Name)
-	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -97,9 +107,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
-			if r.Type.Get() != "" {
-				exprValidator = r.Type.ValidateExpr
-			}
 			if err := exprValidator(r.Expr); err != nil {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
@@ -120,11 +127,10 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Type        datasource.Type    `yaml:"type,omitempty"`
 	Record      string             `yaml:"record,omitempty"`
 	Alert       string             `yaml:"alert,omitempty"`
 	Expr        string             `yaml:"expr"`
-	For         utils.PromDuration `yaml:"for"`
+	For         promutils.Duration `yaml:"for"`
 	Labels      map[string]string  `yaml:"labels,omitempty"`
 	Annotations map[string]string  `yaml:"annotations,omitempty"`
 
@@ -162,7 +168,6 @@ func HashRule(r Rule) uint64 {
 		h.Write([]byte("alerting"))
 		h.Write([]byte(r.Alert))
 	}
-	h.Write([]byte(r.Type.Get()))
 	kv := sortMap(r.Labels)
 	for _, i := range kv {
 		h.Write([]byte(i.key))
@@ -194,6 +199,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
+	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -213,6 +219,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
+			if len(g.ExtraFilterLabels) > 0 {
+				isExtraFilterLabelsUsed = true
+			}
 			groups = append(groups, g)
 		}
 	}
@@ -222,6 +231,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
+	if isExtraFilterLabelsUsed {
+		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
+	}
 	return groups, nil
 }
 

app/vmalert/config/config_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestMain(m *testing.M) {
@@ -95,10 +96,6 @@ func TestGroup_Validate(t *testing.T) {
 			group:  &Group{},
 			expErr: "group name must be set",
 		},
-		{
-			group:  &Group{Name: "test"},
-			expErr: "contain no rules",
-		},
 		{
 			group: &Group{Name: "test",
 				Rules: []Rule{
@@ -263,11 +260,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -279,11 +275,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -347,7 +342,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: utils.NewPromDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -436,7 +431,7 @@ rules:
 `)
 	})
 
-	t.Run("Ok, `for` must change cs", func(t *testing.T) {
+	t.Run("`for` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup
 rules:
@@ -445,10 +440,118 @@ rules:
     for: 5m
 `, `
 name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`interval` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+interval: 2s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+interval: 4s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`concurrency` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+concurrency: 2
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+concurrency: 16
 rules:
   - alert: ExampleAlertWithFor
     expr: sum by(job) (up == 1)
 `)
 	})
 
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+}
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+
+	t.Run("extra labels", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+  env: prod
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+
+	t.Run("extra labels and params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+params:
+  nocache: ["1"]
+  extra_label: ["env=prod"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
+	})
 }

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -1,15 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 2s
-    concurrency: 2
-    type: prometheus
-    labels:
-      cluster: main
-    rules:
-      - alert: up
-        expr: up == 0
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules-update1-good.rules

@@ -1,12 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 30s
-    type: graphite
-    rules:
-      - alert: up
-        expr: filterSeries(time('host.2',20),'>','0')
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/rules0-good.rules

@@ -1,5 +1,8 @@
 groups:
   - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
     rules:
       - alert: VMRows
         for: 10s

app/vmalert/config/testdata/rules2-good.rules

@@ -2,8 +2,11 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
-    extra_filter_labels:
+    extra_filter_labels: # deprecated param, use `params` instead
         job: victoriametrics
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by(instance) > 1
@@ -22,6 +25,7 @@ groups:
           dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}'
         annotations:
           description: Job {{ $labels.job }} is up!
+          external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }}
           summary: All instances up {{ range query "up" }}
             {{ . | label "instance" }}
             {{ end }}

app/vmalert/config/testdata/rules3-good.rules

@@ -21,10 +21,3 @@ groups:
         annotations:
           summary: Too high connection number for {{$labels.instance}}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-      - alert: HostDown
-        type: graphite
-        expr: filterSeries(sumSeries(host.receiver.interface.up),'last','=', 0)
-        for: 3m
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules4-good.rules

@@ -0,0 +1,8 @@
+groups:
+  - name: TestEmptyRules
+    interval: 2s
+    concurrency: 2
+    rules:
+
+  - name: TestNoRules
+    type: prometheus

app/vmalert/config/testdata/rules_interval_good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupTest
+    interval: 1s
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/datasource/datasource.go

@@ -2,6 +2,7 @@ package datasource
 
 import (
 	"context"
+	"net/url"
 	"time"
 )
 
@@ -20,8 +21,7 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     *Type
 	EvaluationInterval time.Duration
-	// see https://docs.victoriametrics.com/#prometheus-querying-api-enhancements
-	ExtraLabels map[string]string
+	QueryParams        url.Values
 }
 
 // Metric is the basic entity which should be return by datasource

app/vmalert/datasource/init.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -12,9 +13,14 @@ import (
 var (
 	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
 		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
-	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+
+	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
+
+	bearerToken     = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
@@ -22,10 +28,18 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
+	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
+	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+
 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
 		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
 		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
@@ -37,9 +51,9 @@ type Param struct {
 }
 
 // Init creates a Querier from provided flag values.
-// Provided extraParams will be added as GET params to
+// Provided extraParams will be added as GET params for
 // each request.
-func Init(extraParams []Param) (QuerierBuilder, error) {
+func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
@@ -49,18 +63,28 @@ func Init(extraParams []Param) (QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
 
+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
 	if *roundDigits > 0 {
-		extraParams = append(extraParams, Param{
-			Key:   "round_digits",
-			Value: fmt.Sprintf("%d", *roundDigits),
-		})
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
+	}
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
 	return &VMStorage{
 		c:                &http.Client{Transport: tr},
-		basicAuthUser:    *basicAuthUsername,
-		basicAuthPass:    *basicAuthPassword,
+		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,

app/vmalert/datasource/type.go

@@ -7,9 +7,6 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-const graphiteType = "graphite"
-const prometheusType = "prometheus"
-
 // Type represents data source type
 type Type struct {
 	name string
@@ -17,12 +14,16 @@ type Type struct {
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
-	return Type{name: prometheusType}
+	return Type{
+		name: "prometheus",
+	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
-	return Type{name: graphiteType}
+	return Type{
+		name: "graphite",
+	}
 }
 
 // NewRawType returns datasource type from raw string
@@ -44,19 +45,19 @@ func (t *Type) Set(d Type) {
 // String implements String interface with default value.
 func (t Type) String() string {
 	if t.name == "" {
-		return prometheusType
+		return "prometheus"
 	}
 	return t.name
 }
 
 // ValidateExpr validates query expression with datasource ql.
 func (t *Type) ValidateExpr(expr string) error {
-	switch t.name {
-	case graphiteType:
+	switch t.String() {
+	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
 			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
 		}
-	case "", prometheusType:
+	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
@@ -72,12 +73,13 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
+	if s == "" {
+		s = "prometheus"
+	}
 	switch s {
-	case "":
-		s = prometheusType
-	case graphiteType, prometheusType:
+	case "graphite", "prometheus":
 	default:
-		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, prometheusType, graphiteType)
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
 	t.name = s
 	return nil

app/vmalert/datasource/vm.go

@@ -5,37 +5,39 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
+	authCfg          *promauth.Config
 	datasourceURL    string
-	basicAuthUser    string
-	basicAuthPass    string
 	appendTypePrefix bool
 	lookBack         time.Duration
 	queryStep        time.Duration
 
 	dataSourceType     Type
 	evaluationInterval time.Duration
-	extraLabels        []string
-	extraParams        []Param
+	extraParams        url.Values
+	disablePathAppend  bool
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                s.c,
-		datasourceURL:    s.datasourceURL,
-		basicAuthUser:    s.basicAuthUser,
-		basicAuthPass:    s.basicAuthPass,
-		lookBack:         s.lookBack,
-		queryStep:        s.queryStep,
-		appendTypePrefix: s.appendTypePrefix,
-		dataSourceType:   s.dataSourceType,
+		c:                 s.c,
+		authCfg:           s.authCfg,
+		datasourceURL:     s.datasourceURL,
+		lookBack:          s.lookBack,
+		queryStep:         s.queryStep,
+		appendTypePrefix:  s.appendTypePrefix,
+		dataSourceType:    s.dataSourceType,
+		disablePathAppend: s.disablePathAppend,
 	}
 }
 
@@ -45,9 +47,7 @@ func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 		s.dataSourceType = *params.DataSourceType
 	}
 	s.evaluationInterval = params.EvaluationInterval
-	for k, v := range params.ExtraLabels {
-		s.extraLabels = append(s.extraLabels, fmt.Sprintf("%s=%s", k, v))
-	}
+	s.extraParams = params.QueryParams
 	return s
 }
 
@@ -57,16 +57,16 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
 	return &VMStorage{
-		c:                c,
-		basicAuthUser:    basicAuthUser,
-		basicAuthPass:    basicAuthPass,
-		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix: appendTypePrefix,
-		lookBack:         lookBack,
-		queryStep:        queryStep,
-		dataSourceType:   NewPrometheusType(),
+		c:                 c,
+		authCfg:           authCfg,
+		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix:  appendTypePrefix,
+		lookBack:          lookBack,
+		queryStep:         queryStep,
+		dataSourceType:    NewPrometheusType(),
+		disablePathAppend: disablePathAppend,
 	}
 }
 
@@ -78,10 +78,10 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}
 
 	ts := time.Now()
-	switch s.dataSourceType.name {
-	case "", prometheusType:
+	switch s.dataSourceType.String() {
+	case "prometheus":
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case graphiteType:
+	case "graphite":
 		s.setGraphiteReqParams(req, query, ts)
 	default:
 		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
@@ -96,7 +96,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}()
 
 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		parseFn = parseGraphiteResponse
 	}
 	return parseFn(req, resp)
@@ -106,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
 	}
 	req, err := s.newRequestPOST()
@@ -133,12 +133,12 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return resp, nil
 }
@@ -148,9 +148,11 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
-	if s.basicAuthPass != "" {
-		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	req.Header.Set("Content-Type", "application/json")
+	if s.authCfg != nil {
+		if auth := s.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_graphite_api.go

@@ -38,7 +38,7 @@ func (r graphiteResponse) metrics() []Metric {
 func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &graphiteResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	return r.metrics(), nil
 }
@@ -54,6 +54,14 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("format", "json")
 	q.Set("target", query)
 	from := "-5min"

app/vmalert/datasource/vm_prom_api.go

@@ -82,10 +82,10 @@ const (
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &promResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	if r.Status == statusError {
-		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
@@ -118,12 +118,14 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusInstantPath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusInstantPath
+	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
 	}
-	if s.evaluationInterval > 0 {
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
@@ -136,7 +138,9 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusRangePath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusRangePath
+	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))
 	q.Add("end", fmt.Sprintf("%d", end.Unix()))
@@ -146,20 +150,24 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 
 func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("query", query)
-	if s.evaluationInterval > 0 {
-		// set step as evaluationInterval by default
-		q.Set("step", s.evaluationInterval.String())
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if s.queryStep > 0 {
-		// override step with user-specified value
-		q.Set("step", s.queryStep.String())
-	}
-	for _, l := range s.extraLabels {
-		q.Add("extra_label", l)
-	}
-	for _, p := range s.extraParams {
-		q.Add(p.Key, p.Value)
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
 }

app/vmalert/datasource/vm_test.go

@@ -5,19 +5,27 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
 	ctx           = context.Background()
 	basicAuthName = "foo"
 	basicAuthPass = "bar"
-	query         = "vm_rows"
-	queryRender   = "constantLine(10)"
+	baCfg         = &promauth.BasicAuthConfig{
+		Username: basicAuthName,
+		Password: promauth.NewSecret(basicAuthPass),
+	}
+	query       = "vm_rows"
+	queryRender = "constantLine(10)"
 )
 
 func TestVMInstantQuery(t *testing.T) {
@@ -73,7 +81,11 @@ func TestVMInstantQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
@@ -179,12 +191,16 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
 
-	_, err := pq.QueryRange(ctx, query, time.Now(), time.Time{})
+	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
 
 	_, err = pq.QueryRange(ctx, query, time.Time{}, time.Now())
@@ -216,6 +232,10 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
 	testCases := []struct {
@@ -234,6 +254,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus path with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus prefix",
 			false,
@@ -245,6 +276,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus prefix with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"prometheus range path",
 			true,
@@ -255,6 +298,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range path with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus range prefix",
 			true,
@@ -266,6 +320,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range prefix with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"graphite path",
 			false,
@@ -308,10 +374,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth",
 			false,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -321,10 +384,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth range",
 			true,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -377,15 +437,28 @@ func TestRequestParams(t *testing.T) {
 				queryStep: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, time.Minute, timestamp.Unix())
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"round digits",
+			"step to seconds",
 			false,
 			&VMStorage{
-				extraParams: []Param{{"round_digits", "10"}},
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
 			},
 			func(t *testing.T, r *http.Request) {
 				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
@@ -393,45 +466,32 @@ func TestRequestParams(t *testing.T) {
 			},
 		},
 		{
-			"extra labels",
-			false,
-			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-		{
-			"extra labels range",
+			"prometheus extra params range",
 			true,
 			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&start=%d",
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
 					timestamp.Unix(), query, timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"extra params",
+			"graphite extra params",
 			false,
 			&VMStorage{
-				extraParams: []Param{
-					{Key: "nocache", Value: "1"},
-					{Key: "max_lookback", Value: "1h"},
+				dataSourceType: NewGraphiteType(),
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("max_lookback=1h&nocache=1&query=%s&time=%d", query, timestamp.Unix())
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
@@ -443,14 +503,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.name {
-			case "", prometheusType:
+			switch tc.vm.dataSourceType.String() {
+			case "prometheus":
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case graphiteType:
+			case "graphite":
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -458,10 +518,63 @@ func TestRequestParams(t *testing.T) {
 	}
 }
 
+func TestAuthConfig(t *testing.T) {
+	var testCases = []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
+}
+
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {
-		t.Errorf("expected to get %q; got %q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 

app/vmalert/group.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"net/url"
 	"sync"
 	"time"
 
@@ -18,17 +19,18 @@ import (
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu          sync.RWMutex
-	Name        string
-	File        string
-	Rules       []Rule
-	Type        datasource.Type
-	Interval    time.Duration
-	Concurrency int
-	Checksum    string
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           datasource.Type
+	Interval       time.Duration
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
 
-	ExtraFilterLabels map[string]string
-	Labels            map[string]string
+	Labels map[string]string
+	Params url.Values
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -40,15 +42,15 @@ type Group struct {
 }
 
 type groupMetrics struct {
-	iterationTotal    *counter
-	iterationDuration *summary
+	iterationTotal    *utils.Counter
+	iterationDuration *utils.Summary
 }
 
 func newGroupMetrics(name, file string) *groupMetrics {
 	m := &groupMetrics{}
 	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
-	m.iterationTotal = getOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = getOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	return m
 }
 
@@ -71,14 +73,14 @@ func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[s
 
 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
-		Type:              cfg.Type,
-		Name:              cfg.Name,
-		File:              cfg.File,
-		Interval:          cfg.Interval.Duration(),
-		Concurrency:       cfg.Concurrency,
-		Checksum:          cfg.Checksum,
-		ExtraFilterLabels: cfg.ExtraFilterLabels,
-		Labels:            cfg.Labels,
+		Type:        cfg.Type,
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval.Duration(),
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		Params:      cfg.Params,
+		Labels:      cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -121,8 +123,11 @@ func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
 }
 
 // ID return unique group ID that consists of
-// rules file and group name
+// rules file and group Name
 func (g *Group) ID() uint64 {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -190,9 +195,12 @@ func (g *Group) updateWith(newGroup *Group) error {
 	for _, nr := range rulesRegistry {
 		newRules = append(newRules, nr)
 	}
+	// note that g.Interval is not updated here
+	// so the value can be compared later in
+	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
-	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Params = newGroup.Params
 	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
@@ -206,8 +214,8 @@ func (g *Group) close() {
 	close(g.doneCh)
 	<-g.finishedCh
 
-	metrics.UnregisterMetric(g.metrics.iterationDuration.name)
-	metrics.UnregisterMetric(g.metrics.iterationTotal.name)
+	g.metrics.iterationDuration.Unregister()
+	g.metrics.iterationTotal.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -215,12 +223,12 @@ func (g *Group) close() {
 
 var skipRandSleepOnGroupStart bool
 
-func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
 
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
 		if randSleep < sleepOffset {
 			randSleep += uint64(g.Interval)
@@ -239,16 +247,7 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 	}
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{rw: rw}
-	for _, nt := range nts {
-		ent := eNotifier{
-			Notifier:         nt,
-			alertsSent:       getOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", nt.Addr())),
-			alertsSendErrors: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", nt.Addr())),
-		}
-		e.notifiers = append(e.notifiers, ent)
-	}
-
+	e := &executor{rw: rw, notifiers: nts}
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -277,36 +276,45 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 		case <-t.C:
 			g.metrics.iterationTotal.Inc()
 			iterationStart := time.Now()
-
-			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
-			for err := range errs {
-				if err != nil {
-					logger.Errorf("group %q: %s", g.Name, err)
+			if len(g.Rules) > 0 {
+				errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, getResolveDuration(g.Interval))
+				for err := range errs {
+					if err != nil {
+						logger.Errorf("group %q: %s", g.Name, err)
+					}
 				}
+				g.LastEvaluation = iterationStart
 			}
-
 			g.metrics.iterationDuration.UpdateDuration(iterationStart)
 		}
 	}
 }
 
+// getResolveDuration returns the duration after which firing alert
+// can be considered as resolved.
+func getResolveDuration(groupInterval time.Duration) time.Duration {
+	delta := *resendDelay
+	if groupInterval > delta {
+		delta = groupInterval
+	}
+	resolveDuration := delta * 4
+	if *maxResolveDuration > 0 && resolveDuration > *maxResolveDuration {
+		resolveDuration = *maxResolveDuration
+	}
+	return resolveDuration
+}
+
 type executor struct {
-	notifiers []eNotifier
+	notifiers func() []notifier.Notifier
 	rw        *remotewrite.Client
 }
 
-type eNotifier struct {
-	notifier.Notifier
-	alertsSent       *counter
-	alertsSendErrors *counter
-}
-
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, interval)
+			res <- e.exec(ctx, rule, resolveDuration)
 		}
 		close(res)
 		return res
@@ -319,7 +327,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, interval)
+				res <- e.exec(ctx, r, resolveDuration)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -337,11 +345,13 @@ var (
 	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
 
 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
 	execTotal.Inc()
 
+	now := time.Now()
 	tss, err := rule.Exec(ctx)
 	if err != nil {
 		execErrors.Inc()
@@ -350,6 +360,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration)
 
 	if len(tss) > 0 && e.rw != nil {
 		for _, ts := range tss {
+			remoteWriteTotal.Inc()
 			if err := e.rw.Push(ts); err != nil {
 				remoteWriteErrors.Inc()
 				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
@@ -361,32 +372,16 @@ func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration)
 	if !ok {
 		return nil
 	}
-	var alerts []notifier.Alert
-	for _, a := range ar.alerts {
-		switch a.State {
-		case notifier.StateFiring:
-			// set End to execStart + 3 intervals
-			// so notifier can resolve it automatically if `vmalert`
-			// won't be able to send resolve for some reason
-			a.End = time.Now().Add(3 * interval)
-			alerts = append(alerts, *a)
-		case notifier.StateInactive:
-			// set End to execStart to notify
-			// that it was just resolved
-			a.End = time.Now()
-			alerts = append(alerts, *a)
-		}
-	}
+
+	alerts := ar.alertsToSend(now, resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
 
 	errGr := new(utils.ErrGroup)
-	for _, nt := range e.notifiers {
-		nt.alertsSent.Add(len(alerts))
+	for _, nt := range e.notifiers() {
 		if err := nt.Send(ctx, alerts); err != nil {
-			nt.alertsSendErrors.Inc()
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
+			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
 		}
 	}
 	return errGr.Err()

app/vmalert/group_test.go

@@ -2,14 +2,14 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func init() {
@@ -34,7 +34,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -46,7 +46,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -107,17 +107,6 @@ func TestUpdateWith(t *testing.T) {
 				{Record: "foo5"},
 			},
 		},
-		{
-			"update datasource type",
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewPrometheusType()},
-				{Alert: "foo3", Type: datasource.NewGraphiteType()},
-			},
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewGraphiteType()},
-				{Alert: "foo10", Type: datasource.NewPrometheusType()},
-			},
-		},
 	}
 
 	for _, tc := range testCases {
@@ -169,10 +158,11 @@ func TestGroupStart(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
-	const evalInterval = time.Millisecond
+
 	fs := &fakeQuerier{}
 	fn := &fakeNotifier{}
 
+	const evalInterval = time.Millisecond
 	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
 	g.Concurrency = 2
 
@@ -191,7 +181,14 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert1.Labels["label"] = "bar"
 	alert1.Labels["host"] = inst1
-	alert1.ID = hash(m1)
+	// add service labels
+	alert1.Labels[alertNameLabel] = alert1.Name
+	alert1.Labels[alertGroupNameLabel] = g.Name
+	var labels1 []string
+	for k, v := range alert1.Labels {
+		labels1 = append(labels1, k, v)
+	}
+	alert1.ID = hash(metricWithLabels(t, labels1...))
 
 	alert2, err := r.newAlert(m2, time.Now(), nil)
 	if err != nil {
@@ -203,13 +200,20 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert2.Labels["label"] = "bar"
 	alert2.Labels["host"] = inst2
-	alert2.ID = hash(m2)
+	// add service labels
+	alert2.Labels[alertNameLabel] = alert2.Name
+	alert2.Labels[alertGroupNameLabel] = g.Name
+	var labels2 []string
+	for k, v := range alert2.Labels {
+		labels2 = append(labels2, k, v)
+	}
+	alert2.ID = hash(metricWithLabels(t, labels2...))
 
 	finished := make(chan struct{})
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), []notifier.Notifier{fn}, nil)
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil)
 		close(finished)
 	}()
 
@@ -220,6 +224,12 @@ func TestGroupStart(t *testing.T) {
 	expectedAlerts := []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
+	gotAlertsNum := fn.getCounter()
+	if gotAlertsNum < len(expectedAlerts)*2 {
+		t.Fatalf("expected to receive at least %d alerts; got %d instead",
+			len(expectedAlerts)*2, gotAlertsNum)
+	}
+
 	// reset previous data
 	fs.reset()
 	// and set only one datapoint for response
@@ -235,3 +245,38 @@ func TestGroupStart(t *testing.T) {
 	g.close()
 	<-finished
 }
+
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		resendDelay   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
+	}
+
+	defaultResolveDuration := *maxResolveDuration
+	defaultResendDelay := *resendDelay
+	defer func() {
+		*maxResolveDuration = defaultResolveDuration
+		*resendDelay = defaultResendDelay
+	}()
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			*maxResolveDuration = tc.maxDuration
+			*resendDelay = tc.resendDelay
+			got := getResolveDuration(tc.groupInterval)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
+}

app/vmalert/helpers_test.go

@@ -61,16 +61,26 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric,
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert
+	// records number of received alerts in total
+	counter int
 }
 
+func (*fakeNotifier) Close()       {}
 func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
+	fn.counter += len(alerts)
 	fn.alerts = alerts
 	return nil
 }
 
+func (fn *fakeNotifier) getCounter() int {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.counter
+}
+
 func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	fn.Lock()
 	defer fn.Unlock()
@@ -205,7 +215,8 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 			}*/
 		}
 		if len(expTS.Labels) != len(gotTS.Labels) {
-			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+			return fmt.Errorf("expected number of labels %d (%v); got %d (%v)",
+				len(expTS.Labels), expTS.Labels, len(gotTS.Labels), gotTS.Labels)
 		}
 		for i, exp := range expTS.Labels {
 			got := gotTS.Labels[i]

app/vmalert/main.go

@@ -35,28 +35,37 @@ absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
-		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes")
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
 	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
+		"which is by default equal to 3 evaluation intervals of the parent group.")
+	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
-	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+
+	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
 
-	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's name as label to generated alerts and time series.")
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
+var alertURLGeneratorFn notifier.AlertURLGenerator
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -77,14 +86,24 @@ func main() {
 		}
 		return
 	}
+
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("failed to init `external.url`: %s", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("failed to init `external.alert.source`: %s", err)
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
 		}
-		eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-		if err != nil {
-			logger.Fatalf("failed to init `external.url`: %s", err)
+		if rw == nil {
+			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
 		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
@@ -93,8 +112,7 @@ func main() {
 		}
 		// prevent queries from caching and boundaries aligning
 		// when querying VictoriaMetrics datasource.
-		noCache := datasource.Param{Key: "nocache", Value: "1"}
-		q, err := datasource.Init([]datasource.Param{noCache})
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
 		if err != nil {
 			logger.Fatalf("failed to init datasource: %s", err)
 		}
@@ -116,11 +134,16 @@ func main() {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
 
+	// Register SIGHUP handler for config re-read just before manager.start call.
+	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
 	if err := manager.start(ctx, groupsCfg); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
 
-	go configReload(ctx, manager, groupsCfg)
+	go configReload(ctx, manager, groupsCfg, sighupCh)
 
 	rh := &requestHandler{m: manager}
 	go httpserver.Serve(*httpListenAddr, rh.handler)
@@ -146,25 +169,28 @@ func newManager(ctx context.Context) (*manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.url`: %w", err)
+
+	labels := make(map[string]string, 0)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
 	}
-	notifier.InitTemplateFunc(eu)
-	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.alert.source`: %w", err)
-	}
-	nts, err := notifier.Init(aug)
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
-
 	manager := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         map[string]string{},
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -178,16 +204,6 @@ func newManager(ctx context.Context) (*manager, error) {
 	}
 	manager.rr = rr
 
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		manager.labels[s[:n]] = s[n+1:]
-	}
 	return manager, nil
 }
 
@@ -244,15 +260,15 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .
 	flagutil.Usage(s)
 }
 
-func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
-	// Register SIGHUP handler for config re-read just before manager.start call.
-	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-	sighupCh := procutil.NewSighupChan()
-
+func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
 	var configCheckCh <-chan time.Time
-	if *rulesCheckInterval > 0 {
-		ticker := time.NewTicker(*rulesCheckInterval)
+	checkInterval := *configCheckInterval
+	if checkInterval == 0 && *rulesCheckInterval > 0 {
+		logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
+		checkInterval = *rulesCheckInterval
+	}
+	if checkInterval > 0 {
+		ticker := time.NewTicker(checkInterval)
 		configCheckCh = ticker.C
 		defer ticker.Stop()
 	}
@@ -269,6 +285,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
+		if err := notifier.Reload(); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to reload notifier config: %s", err)
+			continue
+		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
@@ -283,13 +305,13 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 			// config didn't change - skip it
 			continue
 		}
-		groupsCfg = newGroupsCfg
-		if err := m.update(ctx, groupsCfg, false); err != nil {
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)

app/vmalert/main_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
@@ -94,14 +95,21 @@ groups:
 	*rulesCheckInterval = 200 * time.Millisecond
 	*rulePath = []string{f.Name()}
 	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 
 	m := &manager{
 		querierBuilder: &fakeQuerier{},
 		groups:         make(map[uint64]*Group),
 		labels:         map[string]string{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		rw:             &remotewrite.Client{},
 	}
-	go configReload(ctx, m, nil)
+
+	syncCh := make(chan struct{})
+	sighupCh := procutil.NewSighupChan()
+	go func() {
+		configReload(ctx, m, nil, sighupCh)
+		close(syncCh)
+	}()
 
 	lenLocked := func(m *manager) int {
 		m.groupsMu.RLock()
@@ -138,6 +146,9 @@ groups:
 	if groupsLen != 1 { // should remain unchanged
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
+
+	cancel()
+	<-syncCh
 }
 
 func writeToFile(t *testing.T, file, b string) {

app/vmalert/manager.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"sort"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -15,7 +17,7 @@ import (
 // manager controls group states
 type manager struct {
 	querierBuilder datasource.QuerierBuilder
-	notifiers      []notifier.Notifier
+	notifiers      func() []notifier.Notifier
 
 	rw *remotewrite.Client
 	// remote read builder.
@@ -85,12 +87,31 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 }
 
 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
+	var rrPresent, arPresent bool
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
+		for _, r := range cfg.Rules {
+			if rrPresent && arPresent {
+				continue
+			}
+			if r.Record != "" {
+				rrPresent = true
+			}
+			if r.Alert != "" {
+				arPresent = true
+			}
+		}
 		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	if rrPresent && m.rw == nil {
+		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
+	}
+	if arPresent && m.notifiers == nil {
+		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
+	}
+
 	type updateItem struct {
 		old *Group
 		new *Group
@@ -142,21 +163,38 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:              g.Name,
-		Type:              g.Type.String(),
-		File:              g.File,
-		Interval:          g.Interval.String(),
-		Concurrency:       g.Concurrency,
-		ExtraFilterLabels: g.ExtraFilterLabels,
-		Labels:            g.Labels,
+		Name:           g.Name,
+		Type:           g.Type.String(),
+		File:           g.File,
+		Interval:       g.Interval.Seconds(),
+		LastEvaluation: g.LastEvaluation,
+		Concurrency:    g.Concurrency,
+		Params:         urlValuesToStrings(g.Params),
+		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
-		switch v := r.(type) {
-		case *AlertingRule:
-			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
-		case *RecordingRule:
-			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
-		}
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return ag
 }
+
+func urlValuesToStrings(values url.Values) []string {
+	if len(values) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		params := values[k]
+		for _, v := range params {
+			res = append(res, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return res
+}

app/vmalert/manager_test.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 )
 
 func TestMain(m *testing.M) {
@@ -38,7 +40,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: &fakeQuerier{},
-		notifiers:      []notifier.Notifier{&fakeNotifier{}},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -113,18 +115,6 @@ func TestManagerUpdate(t *testing.T) {
 			Name: "ExampleAlertAlwaysFiring",
 			Expr: "sum by(job) (up == 1)",
 		}
-		ExampleAlertGraphite = &AlertingRule{
-			Name: "up graphite",
-			Expr: "filterSeries(time('host.1',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
-		ExampleAlertGraphite2 = &AlertingRule{
-			Name: "up",
-			Expr: "filterSeries(time('host.2',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
 	)
 
 	testCases := []struct {
@@ -226,28 +216,15 @@ func TestManagerUpdate(t *testing.T) {
 				},
 			},
 		},
-		{
-			name:       "update prometheus to graphite type",
-			initPath:   "config/testdata/dir/rules-update0-good.rules",
-			updatePath: "config/testdata/dir/rules-update1-good.rules",
-			want: []*Group{
-				{
-					File:     "config/testdata/dir/rules-update1-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     datasource.NewGraphiteType(),
-					Name:     "TestUpdateGroup",
-					Rules: []Rule{
-						ExampleAlertGraphite2,
-						ExampleAlertGraphite,
-					},
-				},
-			},
-		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}}
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+			}
 
 			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
 			if err := m.update(ctx, cfgInit, false); err != nil {
@@ -276,6 +253,80 @@ func TestManagerUpdate(t *testing.T) {
 	}
 }
 
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        *remotewrite.Client
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&fakeNotifier{}},
+			nil,
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				rw:             tc.rw,
+			}
+			if tc.notifiers != nil {
+				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
+}
+
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
 	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)

app/vmalert/metrics.go

@@ -1,39 +0,0 @@
-package main
-
-import "github.com/VictoriaMetrics/metrics"
-
-type gauge struct {
-	name string
-	*metrics.Gauge
-}
-
-func getOrCreateGauge(name string, f func() float64) *gauge {
-	return &gauge{
-		name:  name,
-		Gauge: metrics.GetOrCreateGauge(name, f),
-	}
-}
-
-type counter struct {
-	name string
-	*metrics.Counter
-}
-
-func getOrCreateCounter(name string) *counter {
-	return &counter{
-		name:    name,
-		Counter: metrics.GetOrCreateCounter(name),
-	}
-}
-
-type summary struct {
-	name string
-	*metrics.Summary
-}
-
-func getOrCreateSummary(name string) *summary {
-	return &summary{
-		name:    name,
-		Summary: metrics.GetOrCreateSummary(name),
-	}
-}

app/vmalert/notifier/alert.go

@@ -14,17 +14,30 @@ import (
 // Alert the triggered alert
 // TODO: Looks like alert name isn't unique
 type Alert struct {
-	GroupID     uint64
-	Name        string
-	Labels      map[string]string
+	// GroupID contains the ID of the parent rules group
+	GroupID uint64
+	// Name represents Alert name
+	Name string
+	// Labels is the list of label-value pairs attached to the Alert
+	Labels map[string]string
+	// Annotations is the list of annotations generated on Alert evaluation
 	Annotations map[string]string
-	State       AlertState
-
-	Expr  string
+	// State represents the current state of the Alert
+	State AlertState
+	// Expr contains expression that was executed to generate the Alert
+	Expr string
+	// Start defines the moment of time when Alert has triggered
 	Start time.Time
-	End   time.Time
+	// End defines the moment of time when Alert supposed to expire
+	End time.Time
+	// LastSent defines the moment when Alert was sent last time
+	LastSent time.Time
+	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	ID    uint64
+	// ID is the unique identifer for the Alert
+	ID uint64
+	// Restored is true if Alert was restored after restart
+	Restored bool
 }
 
 // AlertState type indicates the Alert state
@@ -59,7 +72,13 @@ type AlertTplData struct {
 	Expr   string
 }
 
-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+}
 
 // ExecTemplate executes the Alert template for given
 // map of annotations.
@@ -89,13 +108,15 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
 	for key, text := range annotations {
 		buf.Reset()
 		builder.Reset()
-		builder.Grow(len(tplHeader) + len(text))
-		builder.WriteString(tplHeader)
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -105,7 +126,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	return r, eg.Err()
 }
 
-func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error {
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
 	t := template.New("").Funcs(funcs).Option("missingkey=zero")
 	tpl, err := t.Parse(text)
 	if err != nil {

app/vmalert/notifier/alert_test.go

@@ -1,12 +1,24 @@
 package notifier
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string, 0)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
 	testCases := []struct {
 		name        string
 		alert       *Alert
@@ -74,6 +86,26 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"desc":    "bar 1;garply 2;",
 			},
 		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
 	}
 
 	qFn := func(q string) ([]datasource.Metric, error) {

app/vmalert/notifier/alertmanager.go

@@ -7,17 +7,41 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	addr          string
-	alertURL      string
-	basicAuthUser string
-	basicAuthPass string
-	argFunc       AlertURLGenerator
-	client        *http.Client
+	addr    string
+	argFunc AlertURLGenerator
+	client  *http.Client
+	timeout time.Duration
+
+	authCfg *promauth.Config
+
+	metrics *metrics
+}
+
+type metrics struct {
+	alertsSent       *utils.Counter
+	alertsSendErrors *utils.Counter
+}
+
+func newMetrics(addr string) *metrics {
+	return &metrics{
+		alertsSent:       utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+	}
+}
+
+// Close is a destructor method for AlertManager
+func (am *AlertManager) Close() {
+	am.metrics.alertsSent.Unregister()
+	am.metrics.alertsSendErrors.Unregister()
 }
 
 // Addr returns address where alerts are sent.
@@ -25,17 +49,36 @@ func (am AlertManager) Addr() string { return am.addr }
 
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
+	am.metrics.alertsSent.Add(len(alerts))
+	err := am.send(ctx, alerts)
+	if err != nil {
+		am.metrics.alertsSendErrors.Add(len(alerts))
+	}
+	return err
+}
+
+func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc)
 
-	req, err := http.NewRequest("POST", am.alertURL, b)
+	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
+
+	if am.timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, am.timeout)
+		defer cancel()
+	}
+
 	req = req.WithContext(ctx)
-	if am.basicAuthPass != "" {
-		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
+
+	if am.authCfg != nil {
+		if auth := am.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -47,9 +90,9 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return fmt.Errorf("failed to read response from %q: %w", am.alertURL, err)
+			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
 		}
-		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body))
 	}
 	return nil
 }
@@ -60,14 +103,39 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
-	url := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
-	return &AlertManager{
-		addr:          alertManagerURL,
-		alertURL:      url,
-		argFunc:       fn,
-		client:        c,
-		basicAuthUser: user,
-		basicAuthPass: pass,
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+	tls := &promauth.TLSConfig{}
+	if authCfg.TLSConfig != nil {
+		tls = authCfg.TLSConfig
 	}
+	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+
+	ba := new(promauth.BasicAuthConfig)
+	oauth := new(promauth.OAuth2Config)
+	if authCfg.BasicAuth != nil {
+		ba = authCfg.BasicAuth
+	}
+	if authCfg.OAuth2 != nil {
+		oauth = authCfg.OAuth2
+	}
+
+	aCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
+	return &AlertManager{
+		addr:    alertManagerURL,
+		argFunc: fn,
+		authCfg: aCfg,
+		client:  &http.Client{Transport: tr},
+		timeout: timeout,
+		metrics: newMetrics(alertManagerURL),
+	}, nil
 }

app/vmalert/notifier/alertmanager_test.go

@@ -8,11 +8,16 @@ import (
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am := NewAlertManager(addr, "", "", nil, nil)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
 	if am.Addr() != addr {
 		t.Errorf("expected to have %q; got %q", addr, am.Addr())
 	}
@@ -75,9 +80,19 @@ func TestAlertManager_Send(t *testing.T) {
 	})
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewAlertManager(srv.URL, baUser, baPass, func(alert Alert) string {
+
+	aCfg := promauth.HTTPClientConfig{
+		BasicAuth: &promauth.BasicAuthConfig{
+			Username: baUser,
+			Password: promauth.NewSecret(baPass),
+		},
+	}
+	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, srv.Client())
+	}, aCfg, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil {
 		t.Error("expected connection error got nil")
 	}

app/vmalert/notifier/config.go

@@ -0,0 +1,182 @@
+package notifier
+
+import (
+	"crypto/md5"
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// Config contains list of supported configuration settings
+// for Notifier
+type Config struct {
+	// Scheme defines the HTTP scheme for Notifier address
+	Scheme string `yaml:"scheme,omitempty"`
+	// PathPrefix is added to URL path before adding alertManagerPath value
+	PathPrefix string `yaml:"path_prefix,omitempty"`
+
+	// ConsulSDConfigs contains list of settings for service discovery via Consul
+	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// StaticConfigs contains list of static targets
+	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
+
+	// HTTPClientConfig contains HTTP configuration for Notifier clients
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
+	// RelabelConfigs contains list of relabeling rules
+	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
+
+	// The timeout used when sending alerts.
+	Timeout promutils.Duration `yaml:"timeout,omitempty"`
+
+	// Checksum stores the hash of yaml definition for the config.
+	// May be used to detect any changes to the config file.
+	Checksum string
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+
+	// This is set to the directory from where the config has been loaded.
+	baseDir string
+
+	// stores already parsed RelabelConfigs object
+	parsedRelabelConfigs *promrelabel.ParsedConfigs
+}
+
+// StaticConfig contains list of static targets in the following form:
+// 	targets:
+//  	[ - '<host>' ]
+type StaticConfig struct {
+	Targets []string `yaml:"targets"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type config Config
+	if err := unmarshal((*config)(cfg)); err != nil {
+		return err
+	}
+	if cfg.Scheme == "" {
+		cfg.Scheme = "http"
+	}
+	if cfg.Timeout.Duration() == 0 {
+		cfg.Timeout = promutils.NewDuration(time.Second * 10)
+	}
+	rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse relabeling config: %w", err)
+	}
+	cfg.parsedRelabelConfigs = rCfg
+
+	b, err := yaml.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)
+	}
+	h := md5.New()
+	h.Write(b)
+	cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
+func parseConfig(path string) (*Config, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading config file: %w", err)
+	}
+	var cfg *Config
+	err = yaml.Unmarshal(data, &cfg)
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.XXX) > 0 {
+		var keys []string
+		for k := range cfg.XXX {
+			keys = append(keys, k)
+		}
+		return nil, fmt.Errorf("unknown fields in %s", strings.Join(keys, ", "))
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
+	}
+	cfg.baseDir = filepath.Dir(absPath)
+	return cfg, nil
+}
+
+func parseLabels(target string, metaLabels map[string]string, cfg *Config) (string, []prompbmarshal.Label, error) {
+	labels := mergeLabels(target, metaLabels, cfg)
+	labels = cfg.parsedRelabelConfigs.Apply(labels, 0, false)
+	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
+	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
+	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details.
+	labels = append([]prompbmarshal.Label{}, labels...)
+
+	if len(labels) == 0 {
+		return "", nil, nil
+	}
+	schemeRelabeled := promrelabel.GetLabelValueByName(labels, "__scheme__")
+	if len(schemeRelabeled) == 0 {
+		schemeRelabeled = "http"
+	}
+	addressRelabeled := promrelabel.GetLabelValueByName(labels, "__address__")
+	if len(addressRelabeled) == 0 {
+		return "", nil, nil
+	}
+	if strings.Contains(addressRelabeled, "/") {
+		return "", nil, nil
+	}
+	addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled)
+	alertsPathRelabeled := promrelabel.GetLabelValueByName(labels, "__alerts_path__")
+	if !strings.HasPrefix(alertsPathRelabeled, "/") {
+		alertsPathRelabeled = "/" + alertsPathRelabeled
+	}
+	u := fmt.Sprintf("%s://%s%s", schemeRelabeled, addressRelabeled, alertsPathRelabeled)
+	if _, err := url.Parse(u); err != nil {
+		return "", nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q, metrics_path=%q (%q): %w",
+			u, cfg.Scheme, schemeRelabeled, target, addressRelabeled, alertsPathRelabeled, err)
+	}
+	return u, labels, nil
+}
+
+func addMissingPort(scheme, target string) string {
+	if strings.Contains(target, ":") {
+		return target
+	}
+	if scheme == "https" {
+		target += ":443"
+	} else {
+		target += ":80"
+	}
+	return target
+}
+
+func mergeLabels(target string, metaLabels map[string]string, cfg *Config) []prompbmarshal.Label {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+	m := make(map[string]string)
+	m["__address__"] = target
+	m["__scheme__"] = cfg.Scheme
+	m["__alerts_path__"] = path.Join("/", cfg.PathPrefix, alertManagerPath)
+	for k, v := range metaLabels {
+		m[k] = v
+	}
+	result := make([]prompbmarshal.Label, 0, len(m))
+	for k, v := range m {
+		result = append(result, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	return result
+}

app/vmalert/notifier/config_test.go

@@ -0,0 +1,31 @@
+package notifier
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestConfigParseGood(t *testing.T) {
+	f := func(path string) {
+		_, err := parseConfig(path)
+		checkErr(t, err)
+	}
+	f("testdata/mixed.good.yaml")
+	f("testdata/consul.good.yaml")
+	f("testdata/static.good.yaml")
+}
+
+func TestConfigParseBad(t *testing.T) {
+	f := func(path, expErr string) {
+		_, err := parseConfig(path)
+		if err == nil {
+			t.Fatalf("expected to get non-nil err for config %q", path)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("testdata/unknownFields.bad.yaml", "unknown field")
+	f("non-existing-file", "error reading")
+}

app/vmalert/notifier/config_watcher.go

@@ -0,0 +1,235 @@
+package notifier
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+)
+
+// configWatcher supports dynamic reload of Notifier objects
+// from static configuration and service discovery.
+// Use newWatcher to create a new object.
+type configWatcher struct {
+	cfg   *Config
+	genFn AlertURLGenerator
+	wg    sync.WaitGroup
+
+	reloadCh chan struct{}
+	syncCh   chan struct{}
+
+	targetsMu sync.RWMutex
+	targets   map[TargetType][]Target
+}
+
+func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) {
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return nil, err
+	}
+	cw := &configWatcher{
+		cfg:       cfg,
+		wg:        sync.WaitGroup{},
+		reloadCh:  make(chan struct{}, 1),
+		syncCh:    make(chan struct{}),
+		genFn:     gen,
+		targetsMu: sync.RWMutex{},
+		targets:   make(map[TargetType][]Target),
+	}
+	return cw, cw.start()
+}
+
+func (cw *configWatcher) notifiers() []Notifier {
+	cw.targetsMu.RLock()
+	defer cw.targetsMu.RUnlock()
+
+	var notifiers []Notifier
+	for _, ns := range cw.targets {
+		for _, n := range ns {
+			notifiers = append(notifiers, n.Notifier)
+		}
+
+	}
+	return notifiers
+}
+
+func (cw *configWatcher) reload(path string) error {
+	select {
+	case cw.reloadCh <- struct{}{}:
+	default:
+		return nil
+	}
+
+	defer func() { <-cw.reloadCh }()
+
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return err
+	}
+	if cfg.Checksum == cw.cfg.Checksum {
+		return nil
+	}
+
+	// stop existing discovery
+	cw.mustStop()
+
+	// re-start cw with new config
+	cw.syncCh = make(chan struct{})
+	cw.cfg = cfg
+	return cw.start()
+}
+
+func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
+	targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+	for _, err := range errors {
+		return fmt.Errorf("failed to init notifier for %q: %s", typeK, err)
+	}
+
+	cw.setTargets(typeK, targets)
+
+	cw.wg.Add(1)
+	go func() {
+		defer cw.wg.Done()
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-cw.syncCh:
+				return
+			case <-ticker.C:
+			}
+			updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+			for _, err := range errors {
+				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+			}
+			cw.setTargets(typeK, updateTargets)
+		}
+	}()
+	return nil
+}
+
+func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
+	metaLabels, err := labelsFn()
+	if err != nil {
+		return nil, []error{fmt.Errorf("failed to get labels: %s", err)}
+	}
+	var targets []Target
+	var errors []error
+	duplicates := make(map[string]struct{})
+	for _, labels := range metaLabels {
+		target := labels["__address__"]
+		u, processedLabels, err := parseLabels(target, labels, cfg)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		if len(u) == 0 {
+			continue
+		}
+		if _, ok := duplicates[u]; ok { // check for duplicates
+			if !*suppressDuplicateTargetErrors {
+				logger.Errorf("skipping duplicate target with identical address %q; "+
+					"make sure service discovery and relabeling is set up properly; "+
+					"original labels: %s; resulting labels: %s",
+					u, labels, processedLabels)
+			}
+			continue
+		}
+		duplicates[u] = struct{}{}
+
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		targets = append(targets, Target{
+			Notifier: am,
+			Labels:   processedLabels,
+		})
+	}
+	return targets, errors
+}
+
+type getLabels func() ([]map[string]string, error)
+
+func (cw *configWatcher) start() error {
+	if len(cw.cfg.StaticConfigs) > 0 {
+		var targets []Target
+		for _, cfg := range cw.cfg.StaticConfigs {
+			for _, target := range cfg.Targets {
+				address, labels, err := parseLabels(target, nil, cw.cfg)
+				if err != nil {
+					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
+				}
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				if err != nil {
+					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
+				}
+				targets = append(targets, Target{
+					Notifier: notifier,
+					Labels:   labels,
+				})
+			}
+		}
+		cw.setTargets(TargetStatic, targets)
+	}
+
+	if len(cw.cfg.ConsulSDConfigs) > 0 {
+		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.ConsulSDConfigs {
+				sdc := &cw.cfg.ConsulSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start consulSD discovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func (cw *configWatcher) mustStop() {
+	close(cw.syncCh)
+	cw.wg.Wait()
+
+	cw.targetsMu.Lock()
+	for _, targets := range cw.targets {
+		for _, t := range targets {
+			t.Close()
+		}
+	}
+	cw.targets = make(map[TargetType][]Target)
+	cw.targetsMu.Unlock()
+
+	for i := range cw.cfg.ConsulSDConfigs {
+		cw.cfg.ConsulSDConfigs[i].MustStop()
+	}
+	cw.cfg = nil
+}
+
+func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
+	cw.targetsMu.Lock()
+	newT := make(map[string]Target)
+	for _, t := range targets {
+		newT[t.Addr()] = t
+	}
+	oldT := cw.targets[key]
+
+	for _, ot := range oldT {
+		if _, ok := newT[ot.Addr()]; !ok {
+			ot.Notifier.Close()
+		}
+	}
+	cw.targets[key] = targets
+	cw.targetsMu.Unlock()
+}

app/vmalert/notifier/config_watcher_test.go

@@ -0,0 +1,301 @@
+package notifier
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"testing"
+)
+
+func TestConfigWatcherReload(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f.Name()) }()
+
+	writeToFile(t, f.Name(), `
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9094
+`)
+	cw, err := newWatcher(f.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+	ns := cw.notifiers()
+	if len(ns) != 2 {
+		t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns)
+	}
+
+	f2, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f2.Name()) }()
+
+	writeToFile(t, f2.Name(), `
+static_configs:
+  - targets:
+      - 127.0.0.1:9093
+`)
+	checkErr(t, cw.reload(f2.Name()))
+
+	ns = cw.notifiers()
+	if len(ns) != 1 {
+		t.Fatalf("expected to have 1 notifier; got %d", len(ns))
+	}
+	expAddr := "http://127.0.0.1:9093/api/v2/alerts"
+	if ns[0].Addr() != expAddr {
+		t.Fatalf("expected to get %q; got %q instead", expAddr, ns[0].Addr())
+	}
+}
+
+func TestConfigWatcherStart(t *testing.T) {
+	consulSDServer := newFakeConsulServer()
+	defer consulSDServer.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+scheme: https
+path_prefix: proxy
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+`, consulSDServer.URL))
+
+	cw, err := newWatcher(consulSDFile.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	if len(cw.notifiers()) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers()))
+	}
+
+	expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1)
+	expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2)
+
+	n1, n2 := cw.notifiers()[0], cw.notifiers()[1]
+	if n1.Addr() != expAddr1 {
+		t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr())
+	}
+	if n2.Addr() != expAddr2 {
+		t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
+	}
+}
+
+// TestConfigWatcherReloadConcurrent supposed to test concurrent
+// execution of configuration update.
+// Should be executed with -race flag
+func TestConfigWatcherReloadConcurrent(t *testing.T) {
+	consulSDServer1 := newFakeConsulServer()
+	defer consulSDServer1.Close()
+	consulSDServer2 := newFakeConsulServer()
+	defer consulSDServer2.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	staticAndConsulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(staticAndConsulSDFile.Name()) }()
+
+	writeToFile(t, staticAndConsulSDFile.Name(), fmt.Sprintf(`
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	paths := []string{
+		staticAndConsulSDFile.Name(),
+		consulSDFile.Name(),
+		"testdata/static.good.yaml",
+		"unknownFields.bad.yaml",
+	}
+
+	cw, err := newWatcher(paths[0], nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	const workers = 500
+	const iterations = 10
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				_ = cw.reload(paths[rnd]) // update can fail and this is expected
+				_ = cw.notifiers()
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	checkErr(t, ioutil.WriteFile(file, []byte(b), 0644))
+}
+
+func checkErr(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+}
+
+const (
+	fakeConsulService1 = "127.0.0.1:9093"
+	fakeConsulService2 = "127.0.0.1:9095"
+)
+
+func newFakeConsulServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
+	})
+	mux.HandleFunc("/v1/catalog/services", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`{
+    "alertmanager": [
+        "alertmanager",
+        "__scheme__=http"
+    ]
+}`))
+	})
+	mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`
+[
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am1",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "__scheme__=http"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9093,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 16,
+            "ModifyIndex": 16
+        }
+    },
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am2",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "bad-node"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9095,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 15,
+            "ModifyIndex": 15
+        }
+    }
+]`))
+	})
+
+	return httptest.NewServer(mux)
+}

app/vmalert/notifier/init.go

@@ -1,17 +1,28 @@
 package notifier
 
 import (
+	"flag"
 	"fmt"
-	"net/http"
+	"strings"
+	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
-	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
-	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
+	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
+
+	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+
+	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
+
+	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
 
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
 	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
@@ -20,26 +31,158 @@ var (
 		"By default system CA is used")
 	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")
+
+	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )
 
-// Init creates a Notifier object based on provided flags.
-func Init(gen AlertURLGenerator) ([]Notifier, error) {
-	if len(*addrs) == 0 {
-		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
+// cw holds a configWatcher for configPath configuration file
+// configWatcher provides a list of Notifier objects discovered
+// from static config or via service discovery.
+// cw is not nil only if configPath is provided.
+var cw *configWatcher
+
+// Reload checks the changes in configPath configuration file
+// and applies changes if any.
+func Reload() error {
+	if cw == nil {
+		return nil
+	}
+	return cw.reload(*configPath)
+}
+
+var staticNotifiersFn func() []Notifier
+
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
+// Init returns a function for retrieving actual list of Notifier objects.
+// Init works in two mods:
+//   * configuration via flags (for backward compatibility). Is always static
+//     and don't support live reloads.
+//   * configuration via file. Supports live reloads and service discovery.
+// Init returns an error if both mods are used.
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	if externalLabels != nil || externalURL != "" {
+		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
 	}
 
+	externalURL = extURL
+	externalLabels = extLabels
+
+	if *configPath == "" && len(*addrs) == 0 {
+		return nil, nil
+	}
+	if *configPath != "" && len(*addrs) > 0 {
+		return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
+	}
+
+	if len(*addrs) > 0 {
+		notifiers, err := notifiersFromFlags(gen)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create notifier from flag values: %s", err)
+		}
+		staticNotifiersFn = func() []Notifier {
+			return notifiers
+		}
+		return staticNotifiersFn, nil
+	}
+
+	var err error
+	cw, err = newWatcher(*configPath, gen)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init config watcher: %s", err)
+	}
+	return cw.notifiers, nil
+}
+
+func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 	var notifiers []Notifier
 	for i, addr := range *addrs {
-		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)
-		ca, serverName := tlsCAFile.GetOptionalArg(i), tlsServerName.GetOptionalArg(i)
-		tr, err := utils.Transport(addr, cert, key, ca, serverName, tlsInsecureSkipVerify.GetOptionalArg(i))
-		if err != nil {
-			return nil, fmt.Errorf("failed to create transport: %w", err)
+		authCfg := promauth.HTTPClientConfig{
+			TLSConfig: &promauth.TLSConfig{
+				CAFile:             tlsCAFile.GetOptionalArg(i),
+				CertFile:           tlsCertFile.GetOptionalArg(i),
+				KeyFile:            tlsKeyFile.GetOptionalArg(i),
+				ServerName:         tlsServerName.GetOptionalArg(i),
+				InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i),
+			},
+			BasicAuth: &promauth.BasicAuthConfig{
+				Username:     basicAuthUsername.GetOptionalArg(i),
+				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
+				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
+			},
+			BearerToken:     promauth.NewSecret(bearerToken.GetOptionalArg(i)),
+			BearerTokenFile: bearerTokenFile.GetOptionalArg(i),
+			OAuth2: &promauth.OAuth2Config{
+				ClientID:         oauth2ClientID.GetOptionalArg(i),
+				ClientSecret:     promauth.NewSecret(oauth2ClientSecret.GetOptionalArg(i)),
+				ClientSecretFile: oauth2ClientSecretFile.GetOptionalArg(i),
+				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
+				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
+			},
+		}
+
+		addr = strings.TrimSuffix(addr, "/")
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		if err != nil {
+			return nil, err
 		}
-		user, pass := basicAuthUsername.GetOptionalArg(i), basicAuthPassword.GetOptionalArg(i)
-		am := NewAlertManager(addr, user, pass, gen, &http.Client{Transport: tr})
 		notifiers = append(notifiers, am)
 	}
-
 	return notifiers, nil
 }
+
+// Target represents a Notifier and optional
+// list of labels added during discovery.
+type Target struct {
+	Notifier
+	Labels []prompbmarshal.Label
+}
+
+// TargetType defines how the Target was discovered
+type TargetType string
+
+const (
+	// TargetStatic is for targets configured statically
+	TargetStatic TargetType = "static"
+	// TargetConsul is for targets discovered via Consul
+	TargetConsul TargetType = "consulSD"
+)
+
+// GetTargets returns list of static or discovered targets
+// via notifier configuration.
+func GetTargets() map[TargetType][]Target {
+	var targets = make(map[TargetType][]Target)
+
+	if staticNotifiersFn != nil {
+		for _, ns := range staticNotifiersFn() {
+			targets[TargetStatic] = append(targets[TargetStatic], Target{
+				Notifier: ns,
+			})
+		}
+	}
+
+	if cw != nil {
+		cw.targetsMu.RLock()
+		for key, ns := range cw.targets {
+			targets[key] = append(targets[key], ns...)
+		}
+		cw.targetsMu.RUnlock()
+	}
+	return targets
+}

app/vmalert/notifier/notifier.go

@@ -10,4 +10,6 @@ type Notifier interface {
 	Send(ctx context.Context, alerts []Alert) error
 	// Addr returns address where alerts are sent.
 	Addr() string
+	// Close is a destructor for the Notifier
+	Close()
 }

app/vmalert/notifier/template_func.go

@@ -17,8 +17,10 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net"
 	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"time"
 
@@ -26,6 +28,7 @@ import (
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // metric is private copy of datasource.Metric,
@@ -61,6 +64,7 @@ var tmplFunc textTpl.FuncMap
 
 // InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
 	tmplFunc = textTpl.FuncMap{
 		/* Strings */
 
@@ -91,6 +95,24 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// alias for https://golang.org/pkg/strings/#ToLower
 		"toLower": strings.ToLower,
 
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
 		/* Numbers */
 
 		// humanize converts given number to a human readable format
@@ -261,6 +283,14 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return m.Labels[label]
 		},
 
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
 		// value returns the value of the given metric.
 		// usually used alongside with `query` template function.
 		"value": func(m metric) float64 {

app/vmalert/notifier/testdata/consul.good.yaml

@@ -0,0 +1,13 @@
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -0,0 +1,18 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__

app/vmalert/notifier/testdata/static.good.yaml

@@ -0,0 +1,4 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095

app/vmalert/notifier/testdata/unknownFields.bad.yaml

@@ -0,0 +1,5 @@
+scheme: https
+unknown: field
+static_configs:
+  - targets:
+      - localhost:9093

app/vmalert/recording.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // RecordingRule is a Rule that supposed
@@ -31,6 +31,8 @@ type RecordingRule struct {
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -43,8 +45,8 @@ type RecordingRule struct {
 }
 
 type recordingRuleMetrics struct {
-	errors  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	samples *utils.Gauge
 }
 
 // String implements Stringer interface
@@ -60,7 +62,7 @@ func (rr *RecordingRule) ID() uint64 {
 
 func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
 	rr := &RecordingRule{
-		Type:    cfg.Type,
+		Type:    group.Type,
 		RuleID:  cfg.ID,
 		Name:    cfg.Record,
 		Expr:    cfg.Expr,
@@ -68,14 +70,14 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 	}
 
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
-	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
+	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -84,7 +86,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 			}
 			return 1
 		})
-	rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -95,8 +97,8 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 
 // Close unregisters rule metrics
 func (rr *RecordingRule) Close() {
-	metrics.UnregisterMetric(rr.metrics.errors.name)
-	metrics.UnregisterMetric(rr.metrics.samples.name)
+	rr.metrics.errors.Unregister()
+	rr.metrics.samples.Unregister()
 }
 
 // ExecRange executes recording rule on the given time range similarly to Exec.
@@ -123,11 +125,13 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 
 // Exec executes RecordingRule expression via the given Querier.
 func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
 	qMetrics, err := rr.q.Query(ctx, rr.Expr)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
 
-	rr.lastExecTime = time.Now()
+	rr.lastExecTime = start
+	rr.lastExecDuration = time.Since(start)
 	rr.lastExecError = err
 	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
@@ -193,23 +197,27 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	return nil
 }
 
-// RuleAPI returns Rule representation in form
-// of APIRecordingRule
-func (rr *RecordingRule) RuleAPI() APIRecordingRule {
-	var lastErr string
-	if rr.lastExecError != nil {
-		lastErr = rr.lastExecError.Error()
-	}
-	return APIRecordingRule{
+// ToAPI returns Rule's representation in form
+// of APIRule
+func (rr *RecordingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "recording",
+		DatasourceType: rr.Type.String(),
+		Name:           rr.Name,
+		Query:          rr.Expr,
+		Labels:         rr.Labels,
+		LastEvaluation: rr.lastExecTime,
+		EvaluationTime: rr.lastExecDuration.Seconds(),
+		Health:         "ok",
+		LastSamples:    rr.lastExecSamples,
 		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", rr.ID()),
-		GroupID:     fmt.Sprintf("%d", rr.GroupID),
-		Name:        rr.Name,
-		Type:        rr.Type.String(),
-		Expression:  rr.Expr,
-		LastError:   lastErr,
-		LastSamples: rr.lastExecSamples,
-		LastExec:    rr.lastExecTime,
-		Labels:      rr.Labels,
+		ID:      fmt.Sprintf("%d", rr.ID()),
+		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
+
+	if rr.lastExecError != nil {
+		r.LastError = rr.lastExecError.Error()
+		r.Health = "err"
+	}
+	return r
 }

app/vmalert/remoteread/init.go

@@ -12,9 +12,15 @@ import (
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
 		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428")
+		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
+
+	bearerToken     = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
 	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
@@ -22,6 +28,14 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
+
+	oauth2ClientID         = flag.String("remoteRead.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteRead.url.")
+	oauth2ClientSecret     = flag.String("remoteRead.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteRead.url.")
+	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
+	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
+	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
+
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
 )
 
 // Init creates a Querier from provided flag values.
@@ -34,6 +48,14 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
 }

app/vmalert/remotewrite/init.go

@@ -13,8 +13,13 @@ var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
-	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+
+	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
+
+	bearerToken     = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -28,6 +33,13 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
+
+	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url.")
+	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url.")
+	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
+	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
+	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
+
 	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 )
 
@@ -43,14 +55,21 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
 	return NewClient(ctx, Config{
 		Addr:              *addr,
+		AuthCfg:           authCfg,
 		Concurrency:       *concurrency,
 		MaxQueueSize:      *maxQueueSize,
 		MaxBatchSize:      *maxBatchSize,
 		FlushInterval:     *flushInterval,
-		BasicAuthUser:     *basicAuthUsername,
-		BasicAuthPass:     *basicAuthPassword,
 		DisablePathAppend: *disablePathAppend,
 		Transport:         t,
 	})

app/vmalert/remotewrite/remotewrite.go

@@ -6,14 +6,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 // Client is an asynchronous HTTP client for writing
@@ -21,8 +24,8 @@ import (
 type Client struct {
 	addr              string
 	c                 *http.Client
+	authCfg           *promauth.Config
 	input             chan prompbmarshal.TimeSeries
-	baUser, baPass    string
 	flushInterval     time.Duration
 	maxBatchSize      int
 	maxQueueSize      int
@@ -35,10 +38,8 @@ type Client struct {
 // Config is config for remote write.
 type Config struct {
 	// Addr of remote storage
-	Addr string
-
-	BasicAuthUser string
-	BasicAuthPass string
+	Addr    string
+	AuthCfg *promauth.Config
 
 	// Concurrency defines number of readers that
 	// concurrently read from the queue and flush data
@@ -92,14 +93,17 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Transport == nil {
 		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
 	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
 		addr:              strings.TrimSuffix(cfg.Addr, "/"),
-		baUser:            cfg.BasicAuthUser,
-		baPass:            cfg.BasicAuthPass,
+		authCfg:           cfg.AuthCfg,
 		flushInterval:     cfg.FlushInterval,
 		maxBatchSize:      cfg.MaxBatchSize,
 		maxQueueSize:      cfg.MaxQueueSize,
@@ -107,10 +111,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 		disablePathAppend: cfg.DisablePathAppend,
 	}
-	cc := defaultConcurrency
-	if cfg.Concurrency > 0 {
-		cc = cfg.Concurrency
-	}
+
 	for i := 0; i < cc; i++ {
 		c.run(ctx)
 	}
@@ -183,10 +184,11 @@ func (c *Client) run(ctx context.Context) {
 }
 
 var (
-	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -197,6 +199,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		return
 	}
 	defer prompbmarshal.ResetWriteRequest(wr)
+	defer bufferFlushDuration.UpdateDuration(time.Now())
 
 	data, err := wr.Marshal()
 	if err != nil {
@@ -232,22 +235,24 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
-	if c.baPass != "" {
-		req.SetBasicAuth(c.baUser, c.baPass)
+	if c.authCfg != nil {
+		if auth := c.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	if !c.disablePathAppend {
-		req.URL.Path += writePath
+		req.URL.Path = path.Join(req.URL.Path, writePath)
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
-			req.URL, err, len(data), r.Size())
+			req.URL.Redacted(), err, len(data), r.Size())
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
-			resp.StatusCode, req.URL, body)
+			resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return nil
 }

app/vmalert/replay_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 type fakeReplayQuerier struct {
@@ -83,7 +83,7 @@ func TestReplay(t *testing.T) {
 			to:    "2021-01-01T15:02:30.000Z",
 			maxDP: 60,
 			cfg: []config.Group{
-				{Interval: utils.NewPromDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 			},
 			qb: &fakeReplayQuerier{
 				registry: map[string]map[string]struct{}{

app/vmalert/rule.go

@@ -21,6 +21,8 @@ type Rule interface {
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error
+	// ToAPI converts Rule into APIRule
+	ToAPI() APIRule
 	// Close performs the shutdown procedures for rule
 	// such as metrics unregister
 	Close()

app/vmalert/tpl/footer.qtpl

@@ -0,0 +1,36 @@
+{% func Footer() %}
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+{% endfunc %}

app/vmalert/tpl/footer.qtpl.go

@@ -0,0 +1,86 @@
+// Code generated by qtc from "footer.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/footer.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/footer.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+func StreamFooter(qw422016 *qt422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:1
+	qw422016.N().S(`
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+`)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func WriteFooter(qq422016 qtio422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:36
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	StreamFooter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func Footer() string {
+//line app/vmalert/tpl/footer.qtpl:36
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/footer.qtpl:36
+	WriteFooter(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	return qs422016
+//line app/vmalert/tpl/footer.qtpl:36
+}

app/vmalert/tpl/header.qtpl

@@ -0,0 +1,43 @@
+{% func Header(title string, pages []NavItem) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    {%= PrintNavItems(title, pages) %}
+    <main class="px-2">
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -0,0 +1,107 @@
+// Code generated by qtc from "header.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/header.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/header.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:1
+	qw422016.N().S(`
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert`)
+//line app/vmalert/tpl/header.qtpl:5
+	if title != "" {
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.N().S(` - `)
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.E().S(title)
+//line app/vmalert/tpl/header.qtpl:5
+	}
+//line app/vmalert/tpl/header.qtpl:5
+	qw422016.N().S(`</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    `)
+//line app/vmalert/tpl/header.qtpl:41
+	StreamPrintNavItems(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:41
+	qw422016.N().S(`
+    <main class="px-2">
+`)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:43
+	StreamHeader(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func Header(title string, pages []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:43
+	WriteHeader(qb422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:43
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:43
+}

app/vmalert/tpl/nav.qtpl

@@ -0,0 +1,25 @@
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func PrintNavItems(current string, items []NavItem) %}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}
+
+

app/vmalert/tpl/nav.qtpl.go

@@ -0,0 +1,96 @@
+// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/nav.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/nav.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/nav.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/nav.qtpl:2
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/nav.qtpl:8
+func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:8
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/nav.qtpl:13
+	for _, item := range items {
+//line app/vmalert/tpl/nav.qtpl:13
+		qw422016.N().S(`
+                <li class="nav-item">
+                    <a class="nav-link`)
+//line app/vmalert/tpl/nav.qtpl:15
+		if current == item.Name {
+//line app/vmalert/tpl/nav.qtpl:15
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/nav.qtpl:15
+		}
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`" href="`)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.E().S(item.Url)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/nav.qtpl:19
+	}
+//line app/vmalert/tpl/nav.qtpl:19
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	StreamPrintNavItems(qw422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func PrintNavItems(current string, items []NavItem) string {
+//line app/vmalert/tpl/nav.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/nav.qtpl:23
+	WritePrintNavItems(qb422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	return qs422016
+//line app/vmalert/tpl/nav.qtpl:23
+}

app/vmalert/utils/auth.go

@@ -0,0 +1,60 @@
+package utils
+
+import (
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+// AuthConfigOptions options which helps build promauth.Config
+type AuthConfigOptions func(config *promauth.HTTPClientConfig)
+
+// AuthConfig returns promauth.Config based on the given params
+func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
+	authCfg := &promauth.HTTPClientConfig{}
+	for _, option := range filterOptions {
+		option(authCfg)
+	}
+
+	return authCfg.NewConfig(".")
+}
+
+// WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
+func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if username != "" || password != "" || passwordFile != "" {
+			config.BasicAuth = &promauth.BasicAuthConfig{
+				Username:     username,
+				Password:     promauth.NewSecret(password),
+				PasswordFile: passwordFile,
+			}
+		}
+	}
+}
+
+// WithBearer returns AuthConfigOptions and set BearerToken or BearerTokenFile based on given params
+func WithBearer(token, tokenFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if token != "" {
+			config.BearerToken = promauth.NewSecret(token)
+		}
+		if tokenFile != "" {
+			config.BearerTokenFile = tokenFile
+		}
+	}
+}
+
+// WithOAuth returns AuthConfigOptions and set OAuth params based on given params
+func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if clientSecretFile != "" || clientSecret != "" {
+			config.OAuth2 = &promauth.OAuth2Config{
+				ClientID:         clientID,
+				ClientSecret:     promauth.NewSecret(clientSecret),
+				ClientSecretFile: clientSecretFile,
+				TokenURL:         tokenURL,
+				Scopes:           strings.Split(scopes, ";"),
+			}
+		}
+	}
+}

app/vmalert/utils/metrics.go

@@ -0,0 +1,54 @@
+package utils
+
+import "github.com/VictoriaMetrics/metrics"
+
+type namedMetric struct {
+	Name string
+}
+
+// Unregister removes the metric by name from default registry
+func (nm namedMetric) Unregister() {
+	metrics.UnregisterMetric(nm.Name)
+}
+
+// Gauge is a metrics.Gauge with Name
+type Gauge struct {
+	namedMetric
+	*metrics.Gauge
+}
+
+// GetOrCreateGauge creates a new Gauge with the given name
+func GetOrCreateGauge(name string, f func() float64) *Gauge {
+	return &Gauge{
+		namedMetric: namedMetric{Name: name},
+		Gauge:       metrics.GetOrCreateGauge(name, f),
+	}
+}
+
+// Counter is a metrics.Counter with Name
+type Counter struct {
+	namedMetric
+	*metrics.Counter
+}
+
+// GetOrCreateCounter creates a new Counter with the given name
+func GetOrCreateCounter(name string) *Counter {
+	return &Counter{
+		namedMetric: namedMetric{Name: name},
+		Counter:     metrics.GetOrCreateCounter(name),
+	}
+}
+
+// Summary is a metrics.Summary with Name
+type Summary struct {
+	namedMetric
+	*metrics.Summary
+}
+
+// GetOrCreateSummary creates a new Summary with the given name
+func GetOrCreateSummary(name string) *Summary {
+	return &Summary{
+		namedMetric: namedMetric{Name: name},
+		Summary:     metrics.GetOrCreateSummary(name),
+	}
+}

app/vmalert/utils/prom_duration.go

@@ -1,43 +0,0 @@
-package utils
-
-import (
-	"time"
-
-	"github.com/VictoriaMetrics/metricsql"
-)
-
-// PromDuration is Prometheus duration.
-type PromDuration struct {
-	milliseconds int64
-}
-
-// NewPromDuration returns PromDuration for given d.
-func NewPromDuration(d time.Duration) PromDuration {
-	return PromDuration{
-		milliseconds: d.Milliseconds(),
-	}
-}
-
-// MarshalYAML implements yaml.Marshaler interface.
-func (pd PromDuration) MarshalYAML() (interface{}, error) {
-	return pd.Duration().String(), nil
-}
-
-// UnmarshalYAML implements yaml.Unmarshaler interface.
-func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var s string
-	if err := unmarshal(&s); err != nil {
-		return err
-	}
-	ms, err := metricsql.DurationValue(s, 0)
-	if err != nil {
-		return err
-	}
-	pd.milliseconds = ms
-	return nil
-}
-
-// Duration returns duration for pd.
-func (pd *PromDuration) Duration() time.Duration {
-	return time.Duration(pd.milliseconds) * time.Millisecond
-}

app/vmalert/vmalert_cluster.excalidraw

@@ -0,0 +1,525 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 794,
+      "versionNonce": 1855937036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 659,
+      "versionNonce": 247957684,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347948032,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1670,
+      "versionNonce": 2021681972,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 750.3317260742188,
+      "y": 226.5509033203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348054411
+    },
+    {
+      "type": "text",
+      "version": 1311,
+      "versionNonce": 1283453068,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 759.2862243652344,
+      "y": 238.68240356445312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert:8480",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 897,
+      "versionNonce": 1983434892,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 61.744873046875,
+      "y": 224.9600830078125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638348077724
+    },
+    {
+      "type": "text",
+      "version": 719,
+      "versionNonce": 457402292,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 70.69937133789062,
+      "y": 237.33523559570312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect:8481",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1098,
+      "versionNonce": 1480603788,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 371.7434387207031,
+      "y": 398.50787353515625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 864,
+      "versionNonce": 1115813900,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 393.73809814453125,
+      "y": 410.5976257324219,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348009180,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2405,
+      "versionNonce": 959767732,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 556.6860961914062,
+      "y": 252.2582773408825,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 184.9195556640625,
+      "height": 1.6022679018915937,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.0344528515859526,
+        "gap": 10.57574462890625
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.039393828258510157,
+        "gap": 8.72607421875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.9195556640625,
+          -1.6022679018915937
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1173,
+      "versionNonce": 1248255756,
+      "isDeleted": false,
+      "id": "wRO0q9xKPHc8e8XPPsQWh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.0439244722469,
+      "y": 246.80533728741074,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 157.86774649373126,
+      "height": 0.1417938392881979,
+      "seed": 656189364,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.13736472619498497,
+        "gap": 16.306295254315614
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.013200835330936087,
+        "gap": 14.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -157.86774649373126,
+          0.1417938392881979
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 557,
+      "versionNonce": 995289780,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 552.4987182617188,
+      "y": 212.27996826171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 76,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348059525,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist alerts state\n\n\nand recording rules",
+      "baseline": 72,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 803,
+      "versionNonce": 1576507444,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 290.34130859375,
+      "y": 210.56927490234375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1471,
+      "versionNonce": 1361321140,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 484.18669893674246,
+      "y": 302.3424013553929,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 1.0484739253853945,
+      "height": 84.72775855671654,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348083348,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.010768924644894236,
+        "gap": 14.650750964767894
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.051051952959743775,
+        "gap": 11.437713623046818
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0484739253853945,
+          84.72775855671654
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 576,
+      "versionNonce": 2112088460,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 530.5612182617188,
+      "y": 318.60687255859375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348023735,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 1835119500,
+      "isDeleted": false,
+      "id": "ff5OkfgmkKLifS13_TFj3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 291.37474060058594,
+      "y": 261.4861297607422,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 264004620,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "restore state",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_ha.excalidraw

@@ -0,0 +1,911 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 906,
+      "versionNonce": 448716468,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 302.2676696777344,
+      "y": 275.59356689453125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "OTKoeHmKtqCxFArDbY-sP"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 762,
+      "versionNonce": 223660724,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 318.0538024902344,
+      "y": 296.6778106689453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2"
+      ],
+      "updated": 1638355102070,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert1",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 2067,
+      "versionNonce": 817347852,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 329.6426086425781,
+      "y": 90.3275146484375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355058507
+    },
+    {
+      "type": "text",
+      "version": 1717,
+      "versionNonce": 2040797452,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 349.01177978515625,
+      "y": 102.45901489257812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1553,
+      "versionNonce": 39381044,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 173.67257690429688,
+      "y": 470.5100402832031,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "OTKoeHmKtqCxFArDbY-sP",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 1309,
+      "versionNonce": 1112872844,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 189.667236328125,
+      "y": 482.59979248046875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager1:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 835,
+      "versionNonce": 2036483596,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 525.83837890625,
+      "y": 147.33470153808594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 777,
+      "versionNonce": 990468492,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 617.0690307617188,
+      "y": 376.9822692871094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638355227582,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1112,
+      "versionNonce": 999136652,
+      "isDeleted": false,
+      "id": "mIu-d0lmShCxzMLD5iA_p",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 492.29505920410156,
+      "y": 272.3052215576172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 756416780,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "S_dOHQrhGmu8SFJzobJK7",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "Ar-hcDLlzVSoTPs2MaywO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 970,
+      "versionNonce": 430760500,
+      "isDeleted": false,
+      "id": "ZqIR6SaLNDQl8s0zZbdnE",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 508.08119201660156,
+      "y": 293.38946533203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 1477404084,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355105020,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalertN",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "qZFezRGU4Chwxgvb2451t",
+      "type": "line",
+      "x": 449.93653869628906,
+      "y": 306.30010986328125,
+      "width": 19.48321533203125,
+      "height": 0.3865966796875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1833426828,
+      "version": 69,
+      "versionNonce": 871948300,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          19.48321533203125,
+          0.3865966796875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": null,
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": null
+    },
+    {
+      "type": "rectangle",
+      "version": 1649,
+      "versionNonce": 93981708,
+      "isDeleted": false,
+      "id": "gNHiZJKo0ap69ALDobtZ-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518.5596466064453,
+      "y": 471.4539489746094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 454422412,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "Ar-hcDLlzVSoTPs2MaywO",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 1412,
+      "versionNonce": 75038348,
+      "isDeleted": false,
+      "id": "O-zgjZBvt4RI1PrkBNQnb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 534.3148040771484,
+      "y": 483.543701171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1026268980,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanagerN:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "m9_BptFOFxbV2sS_xJDu2",
+      "type": "arrow",
+      "x": 378.57185562792466,
+      "y": 262.1596984863281,
+      "width": 79.96146539019026,
+      "height": 111.53411865234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 141224884,
+      "version": 521,
+      "versionNonce": 283254540,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          79.96146539019026,
+          -111.53411865234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.23996018586441184,
+        "gap": 13.433868408203125
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.14207100087207922,
+        "gap": 15.550811767578125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "S_dOHQrhGmu8SFJzobJK7",
+      "type": "arrow",
+      "x": 558.1990515577129,
+      "y": 263.4271240234375,
+      "width": 88.18940317574186,
+      "height": 114.15780639648438,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1004789812,
+      "version": 314,
+      "versionNonce": 1259171724,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355105019,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -88.18940317574186,
+          -114.15780639648438
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4315094049079832,
+        "gap": 8.878097534179688
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": 0.14840834302306677,
+        "gap": 14.194549560546875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "fsGFp4NW4JlrCdF0HR3uA",
+      "type": "arrow",
+      "x": 356.6613630516658,
+      "y": 354.95416259765625,
+      "width": 278.74351860741064,
+      "height": 106.90020751953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 712384692,
+      "version": 334,
+      "versionNonce": 266493324,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          278.74351860741064,
+          106.90020751953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.7721210277890177,
+        "gap": 7.228485107421875
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4493598001028791,
+        "gap": 9.599578857421875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "OTKoeHmKtqCxFArDbY-sP",
+      "type": "arrow",
+      "x": 361.02563221226757,
+      "y": 356.2252197265625,
+      "width": 95.3594006000182,
+      "height": 105.52130126953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1992252596,
+      "version": 466,
+      "versionNonce": 472553100,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355221749,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -95.3594006000182,
+          105.52130126953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.39325294425055324,
+        "gap": 8.499542236328125
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.400636314282374,
+        "gap": 8.763519287109375
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "XhPgLRBk-YhWAFcSQi9TJ",
+      "type": "arrow",
+      "x": 561.7189961327852,
+      "y": 349.238037109375,
+      "width": 266.271510370216,
+      "height": 112.383544921875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 588869260,
+      "version": 453,
+      "versionNonce": 584723212,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355150131,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -266.271510370216,
+          112.383544921875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": -0.7084007026732048,
+        "gap": 4.8007049560546875
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.4180430111580123,
+        "gap": 8.888458251953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "Ar-hcDLlzVSoTPs2MaywO",
+      "type": "arrow",
+      "x": 557.5999880535809,
+      "y": 352.71795654296875,
+      "width": 112.87813113656136,
+      "height": 106.3255615234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1508782476,
+      "version": 331,
+      "versionNonce": 1137561908,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355153683,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          112.87813113656136,
+          106.3255615234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4358123206211808,
+        "gap": 8.280624389648438
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4783744286829653,
+        "gap": 12.410430908203125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "WqHnv9_g3SdkLZuy4WSHa",
+      "type": "text",
+      "x": 186.5891876220703,
+      "y": 523.145263671875,
+      "width": 272,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1310147468,
+      "version": 281,
+      "versionNonce": 949157044,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "text": "Alertmanagers in cluster mode",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "id": "yA0kgvdF71wZbHJ7Cg2p8",
+      "type": "rectangle",
+      "x": 159.41685485839844,
+      "y": 440.7666015625,
+      "width": 625.9590148925781,
+      "height": 110.13494873046878,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1465986996,
+      "version": 366,
+      "versionNonce": 196040844,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939
+    },
+    {
+      "id": "D6fkQH1E_MFbCuL697ArO",
+      "type": "arrow",
+      "x": 422.4853057861328,
+      "y": 494.8779132311229,
+      "width": 90.5517578125,
+      "height": 0.12427004064892344,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 2025208204,
+      "version": 316,
+      "versionNonce": 2144102156,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          90.5517578125,
+          0.12427004064892344
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.08064204025505255,
+        "gap": 8.706283569335938
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": -0.05976220061952895,
+        "gap": 5.5225830078125
+      },
+      "startArrowhead": "arrow",
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 463,
+      "versionNonce": 2118914484,
+      "isDeleted": false,
+      "id": "p4EDvSKPqZzfM_SReybeq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 274.6086883544922,
+      "y": 56.77886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 394,
+      "height": 19,
+      "seed": 1214462348,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355074872,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VictoriaMetrics with deduplication enabled",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 712,
+      "versionNonce": 737833612,
+      "isDeleted": false,
+      "id": "YfNFeOucMo8BJk2P2JRex",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 229.77923583984375,
+      "y": 227.84378051757812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 448.5542297363281,
+      "height": 134.06329345703122,
+      "seed": 932207756,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355133244
+    },
+    {
+      "type": "text",
+      "version": 427,
+      "versionNonce": 1458527796,
+      "isDeleted": false,
+      "id": "oldM7Q_aJtpWd2jXQ0iKf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 238.9442901611328,
+      "y": 230.862548828125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 225,
+      "height": 38,
+      "seed": 2131899572,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355259358,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "vmalerts with identical \nconfigurations",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_multicluster.excalidraw

@@ -0,0 +1,915 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 791,
+      "versionNonce": 48874036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "pD9DcILMxa6GaR1U5YyMO",
+        "HPEwr85wL4IedW0AgdArp",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 658,
+      "versionNonce": 1653816076,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347796775,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 802,
+      "versionNonce": 995326644,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.05322265625,
+      "y": 228.65371704101562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 624,
+      "versionNonce": 707755700,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 640.7635803222656,
+      "y": 241.02886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701075,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 802,
+      "versionNonce": 1974403340,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 401.241943359375,
+      "y": 342.4627990722656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 178,
+      "height": 38,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347863144,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute aggregating\nrecording rules",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "jrFZeFNsxlss7IsEZpA-J",
+      "type": "rectangle",
+      "x": 594.1509857177734,
+      "y": 192.09194946289062,
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 48379060,
+      "version": 665,
+      "versionNonce": 617456652,
+      "isDeleted": false,
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347763716
+    },
+    {
+      "id": "6ibhLp94HJFIdfP5HCEv8",
+      "type": "text",
+      "x": 609.7205657958984,
+      "y": 199.81723022460938,
+      "width": 225,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1053164852,
+      "version": 256,
+      "versionNonce": 538595124,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347761092,
+      "text": "VM cluster with raw data",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "type": "rectangle",
+      "version": 914,
+      "versionNonce": 1576666164,
+      "isDeleted": false,
+      "id": "R5v-yZCVJ97BkJqr0Qb7H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 807.5664215087891,
+      "y": 287.8628387451172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1794212620,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "bZXA8PH9gYu-clotqJ4f7",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 725,
+      "versionNonce": 267455500,
+      "isDeleted": false,
+      "id": "pWRC_smX7TuOI8_8UrA4H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 840.0209197998047,
+      "y": 300.2379913330078,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 421856180,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 843,
+      "versionNonce": 1244193972,
+      "isDeleted": false,
+      "id": "EqROOfYulSPsZm7ovxfQN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 604.9091339111328,
+      "y": 345.2847442626953,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 2043521972,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347718537
+    },
+    {
+      "type": "text",
+      "version": 676,
+      "versionNonce": 143370892,
+      "isDeleted": false,
+      "id": "ddQH1nnmT7HbKW7Xmv4zx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 642.6199798583984,
+      "y": 357.90354919433594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 335223180,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "bZXA8PH9gYu-clotqJ4f7",
+      "type": "arrow",
+      "x": 785.2667708463345,
+      "y": 367.2342882272049,
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1153029388,
+      "version": 420,
+      "versionNonce": 1726025228,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347704644,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "EqROOfYulSPsZm7ovxfQN",
+        "focus": 0.5247890891198189,
+        "gap": 8.36404562660789
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": -0.6931056940383433,
+        "gap": 9.943033572650961
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "arrow",
+      "version": 528,
+      "versionNonce": 1908259468,
+      "isDeleted": false,
+      "id": "MGdu6GDIPNBAaEUr0Gt-a",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 781.4456641707259,
+      "y": 248.777857603323,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1695061516,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347737174,
+      "startBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 883,
+      "versionNonce": 845352076,
+      "isDeleted": false,
+      "id": "4pW8hvBu3bo1eMtFvg_gS",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 601.6520690917969,
+      "y": 473.7677536010742,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1678097076,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 703,
+      "versionNonce": 519371956,
+      "isDeleted": false,
+      "id": "w_i2hO06oLa0bWbyAfFzU",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 639.3624267578125,
+      "y": 486.14290618896484,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 340753036,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 745,
+      "versionNonce": 879581324,
+      "isDeleted": false,
+      "id": "U5U-67wL5fPwvzlxOAF5A",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 592.7498321533203,
+      "y": 437.2059860229492,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "seed": 912540724,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347776748
+    },
+    {
+      "type": "text",
+      "version": 345,
+      "versionNonce": 1628116148,
+      "isDeleted": false,
+      "id": "0IGVrvICMVeZp2RCaqTeP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 608.3194122314453,
+      "y": 444.93126678466797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 291,
+      "height": 19,
+      "seed": 68700428,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347784373,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VM cluster with aggregated data",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 996,
+      "versionNonce": 2112461580,
+      "isDeleted": false,
+      "id": "9QMf0HXPE1W4M9S-zMJVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 806.1652679443359,
+      "y": 532.9768753051758,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 523607476,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "9WBH34em4CdU2OwzqYIl5",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 804,
+      "versionNonce": 1660832052,
+      "isDeleted": false,
+      "id": "WOrOD5vn6EtMUQRJomt3-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 838.6197662353516,
+      "y": 545.3520278930664,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 1321420684,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 925,
+      "versionNonce": 2052807604,
+      "isDeleted": false,
+      "id": "4dtJZpXEUxSK3biwFG7vd",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.5079803466797,
+      "y": 590.3987808227539,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1738524468,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "9WBH34em4CdU2OwzqYIl5",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 756,
+      "versionNonce": 2040967820,
+      "isDeleted": false,
+      "id": "rAbyooo-X08-86qjoK0WR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 641.2188262939453,
+      "y": 603.0175857543945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 948598284,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "9WBH34em4CdU2OwzqYIl5"
+      ],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 660,
+      "versionNonce": 1560678196,
+      "isDeleted": false,
+      "id": "9WBH34em4CdU2OwzqYIl5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.8656172818814,
+      "y": 612.3483247872634,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "seed": 1141424308,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": 0.5247890891198189,
+        "gap": 8.364045626608004
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": -0.6931056940383404,
+        "gap": 9.943033572650847
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 768,
+      "versionNonce": 1653264948,
+      "isDeleted": false,
+      "id": "5W90eDBjtfZvkSuQTG0Iw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.0445106062728,
+      "y": 493.8918941633816,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1852298380,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4pW8hvBu3bo1eMtFvg_gS",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "type": "arrow",
+      "x": 423.70701599121094,
+      "y": 428.34434509277344,
+      "width": 179.54901123046875,
+      "height": 182.9937744140625,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 389863732,
+      "version": 47,
+      "versionNonce": 1364399028,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347805500,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          179.54901123046875,
+          -182.9937744140625
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "type": "arrow",
+      "x": 424.7585906982422,
+      "y": 441.12806701660156,
+      "width": 174.29449462890625,
+      "height": 170.56607055664062,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 981082124,
+      "version": 92,
+      "versionNonce": 1261546252,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347812431,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          174.29449462890625,
+          170.56607055664062
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": -0.8207814548026305,
+        "gap": 4.45489501953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 775,
+      "versionNonce": 95530764,
+      "isDeleted": false,
+      "id": "o-yIJ_WZzVubhbVODvhcC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 407.27012634277344,
+      "y": 489.33091735839844,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 775116812,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347824876,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_single.excalidraw

@@ -0,0 +1,354 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 795,
+      "versionNonce": 1195460364,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 660,
+      "versionNonce": 1034125236,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1690,
+      "versionNonce": 236788620,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 344.9837951660156,
+      "y": 64.64248657226562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348176044
+    },
+    {
+      "type": "text",
+      "version": 1331,
+      "versionNonce": 945335476,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 364.35296630859375,
+      "y": 76.77398681640625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1099,
+      "versionNonce": 671872012,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 360.6495666503906,
+      "y": 382.2536315917969,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 865,
+      "versionNonce": 635839156,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 382.64422607421875,
+      "y": 394.3433837890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2444,
+      "versionNonce": 361351692,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.0465703465111,
+      "y": 204.98379516601562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.7962088410123442,
+      "height": 86.86798095703125,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "gap": 10.57574462890625,
+        "focus": 0.0344528515859526
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "gap": 8.72607421875,
+        "focus": -0.039393828258510157
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.7962088410123442,
+          -86.86798095703125
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 640,
+      "versionNonce": 1892951180,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 500.609619140625,
+      "y": 134.11544799804688,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348163424,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1472,
+      "versionNonce": 768565516,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.5245361328125,
+      "y": 300.5478957313172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.3521007656264601,
+      "height": 70.26802223743277,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "startBinding": {
+        "elementId": "E9Run6wCm2chQ6JHrmc_y",
+        "focus": 1.1925203824459496,
+        "gap": 11.77154541015625
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.0441077573536454,
+        "gap": 11.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.3521007656264601,
+          70.26802223743277
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 577,
+      "versionNonce": 1646003636,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 498.29608154296875,
+      "y": 298.6573791503906,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/web.go

@@ -4,40 +4,84 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var (
+	once     = sync.Once{}
+	apiLinks [][2]string
+	navItems []tpl.NavItem
+)
+
+func initLinks() {
+	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
+	apiLinks = [][2]string{
+		{path.Join(pathPrefix, "api/v1/rules"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
+		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
+		{path.Join(pathPrefix, "flags"), "command-line flags"},
+		{path.Join(pathPrefix, "metrics"), "list of application metrics"},
+		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
+	}
+	navItems = []tpl.NavItem{
+		{Name: "vmalert", Url: path.Join(pathPrefix, "/")},
+		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
+		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
+		{Name: "Notifiers", Url: path.Join(pathPrefix, "notifiers")},
+		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
+	}
+}
+
 type requestHandler struct {
 	m *manager
 }
 
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	once.Do(func() {
+		initLinks()
+	})
+
+	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
+
 	switch r.URL.Path {
 	case "/":
 		if r.Method != "GET" {
 			return false
 		}
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/api/v1/groups", "list all loaded groups and rules"},
-			{"/api/v1/alerts", "list all active alerts"},
-			{"/api/v1/groupID/alertID/status", "get alert status by ID"},
-			{"/metrics", "list of application metrics"},
-			{"/-/reload", "reload configuration"},
-		})
+		WriteWelcome(w)
 		return true
-	case "/api/v1/groups":
+	case "/alerts":
+		WriteListAlerts(w, pathPrefix, rh.groupAlerts())
+		return true
+	case "/groups":
+		WriteListGroups(w, rh.groups())
+		return true
+	case "/notifiers":
+		WriteListTargets(w, notifier.GetTargets())
+		return true
+	case "/api/v1/rules":
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
@@ -46,7 +90,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -58,39 +102,57 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		if !strings.HasSuffix(r.URL.Path, "/status") {
 			return false
 		}
-		// /api/v1/<groupName>/<alertID>/status
-		data, err := rh.alert(r.URL.Path)
+		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.Write(data)
+
+		// /api/v1/<groupID>/<alertID>/status
+		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
+			data, err := json.Marshal(alert)
+			if err != nil {
+				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+				return true
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.Write(data)
+			return true
+		}
+
+		// <groupID>/<alertID>/status
+		WriteAlert(w, pathPrefix, alert)
 		return true
 	}
 }
 
 type listGroupsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Groups []APIGroup `json:"groups"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) groups() []APIGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	lr := listGroupsResponse{Status: "success"}
+	var groups []APIGroup
 	for _, g := range rh.m.groups {
-		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+		groups = append(groups, g.toAPI())
 	}
 
 	// sort list of alerts for deterministic output
-	sort.Slice(lr.Data.Groups, func(i, j int) bool {
-		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	sort.Slice(groups, func(i, j int) bool {
+		return groups[i].Name < groups[j].Name
 	})
 
+	return groups
+}
+
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	lr := listGroupsResponse{Status: "success"}
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{
@@ -102,10 +164,34 @@ func (rh *requestHandler) listGroups() ([]byte, error) {
 }
 
 type listAlertsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Alerts []*APIAlert `json:"alerts"`
 	} `json:"data"`
-	Status string `json:"status"`
+}
+
+func (rh *requestHandler) groupAlerts() []GroupAlerts {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	var groupAlerts []GroupAlerts
+	for _, g := range rh.m.groups {
+		var alerts []*APIAlert
+		for _, r := range g.Rules {
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			alerts = append(alerts, a.AlertsToAPI()...)
+		}
+		if len(alerts) > 0 {
+			groupAlerts = append(groupAlerts, GroupAlerts{
+				Group:  g.toAPI(),
+				Alerts: alerts,
+			})
+		}
+	}
+	return groupAlerts
 }
 
 func (rh *requestHandler) listAlerts() ([]byte, error) {
@@ -119,7 +205,7 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 			if !ok {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
 		}
 	}
 
@@ -138,18 +224,17 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
-func (rh *requestHandler) alert(path string) ([]byte, error) {
+func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", 3)
 	if len(parts) != 3 {
 		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
 			StatusCode: http.StatusBadRequest,
 		}
 	}
-
 	groupID, err := uint64FromPath(parts[0])
 	if err != nil {
 		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
@@ -162,7 +247,7 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 	if err != nil {
 		return nil, errResponse(err, http.StatusNotFound)
 	}
-	return json.Marshal(resp)
+	return resp, nil
 }
 
 func uint64FromPath(path string) (uint64, error) {

app/vmalert/web.qtpl

@@ -0,0 +1,346 @@
+{% package main %}
+
+{% import (
+    "time"
+    "sort"
+    "path"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+) %}
+
+
+{% func Welcome() %}
+    {%= tpl.Header("vmalert", navItems) %}
+    <p>
+        API:<br>
+        {% for _, p := range apiLinks  %}
+            {%code
+                p, doc := p[0], p[1]
+            %}
+        	<a href="{%s p %}">{%s p %}</a> - {%s doc %}<br/>
+        {% endfor %}
+    </p>
+    {%= tpl.Footer() %}
+{% endfunc %}
+
+{% func ListGroups(groups []APIGroup) %}
+    {%= tpl.Header("Groups", navItems) %}
+    {%  if len(groups) > 0 %}
+        {%code
+            rOk := make(map[string]int)
+            rNotOk := make(map[string]int)
+            for _, g := range groups {
+                for _, r := range g.Rules {
+                    if r.LastError != "" {
+                        rNotOk[g.Name]++
+                    } else {
+                        rOk[g.Name]++
+                    }
+                }
+            }
+        %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+        {% for _, g := range groups  %}
+              <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
+                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
+                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+                {% if len(g.Params) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra params
+                    {% for _, param := range g.Params %}
+                            <span class="float-left badge bg-primary">{%s param %}</span>
+                    {% endfor %}
+                    </div>
+                {% endif %}
+            </div>
+            <div class="collapse" id="rules-{%s g.ID %}">
+                <table class="table table-striped table-hover table-sm">
+                    <thead>
+                        <tr>
+                            <th scope="col">Rule</th>
+                            <th scope="col" title="Shows if rule's execution ended with error">Error</th>
+                            <th scope="col" title="How many samples were produced by the rule">Samples</th>
+                            <th scope="col" title="How many seconds ago rule was executed">Updated</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                    {% for _, r := range g.Rules %}
+                        <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
+                            <td>
+                                {% if r.Type == "alerting" %}
+                                <b>alert:</b> (for: {%v r.Duration %})
+                                {% else %}
+                                <b>record:</b> {%s r.Name %}
+                                {% endif %}
+                                <br>
+                                <code><pre>{%s r.Query %}</pre></code><br>
+                                {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range r.Labels %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                {% endfor %}
+                            </td>
+                            <td><div class="error-cell">{%s r.LastError %}</div></td>
+                            <td>{%d r.LastSamples %}</td>
+                            <td>{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
+                        </tr>
+                    {% endfor %}
+                 </tbody>
+                </table>
+            </div>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+
+{% func ListAlerts(pathPrefix string, groupAlerts []GroupAlerts) %}
+    {%= tpl.Header("Alerts", navItems) %}
+    {% if len(groupAlerts) > 0 %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         {% for _, ga := range groupAlerts %}
+            {%code g := ga.Group %}
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
+                <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
+                <br>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+            </div>
+            {%code
+                var keys []string
+                alertsByRule := make(map[string][]*APIAlert)
+                for _, alert := range ga.Alerts {
+                    if len(alertsByRule[alert.RuleID]) < 1 {
+                        keys = append(keys, alert.RuleID)
+                    }
+                    alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
+                }
+                sort.Strings(keys)
+            %}
+            <div class="collapse" id="rules-{%s g.ID %}">
+                {% for _, ruleID := range keys %}
+                    {%code
+                        defaultAR := alertsByRule[ruleID][0]
+                        var labelKeys []string
+                        for k := range defaultAR.Labels {
+                            labelKeys = append(labelKeys, k)
+                        }
+                        sort.Strings(labelKeys)
+                    %}
+                    <br>
+                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                    <br>
+                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr>
+                                <th scope="col">Labels</th>
+                                <th scope="col">State</th>
+                                <th scope="col">Active at</th>
+                                <th scope="col">Value</th>
+                                <th scope="col">Link</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                        {% for _, ar := range alertsByRule[ruleID] %}
+                            <tr>
+                                <td>
+                                    {% for _, k := range labelKeys %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
+                                    {% endfor %}
+                                </td>
+                                <td>{%= badgeState(ar.State) %}</td>
+                                <td>
+                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                </td>
+                                <td>{%s ar.Value %}</td>
+                                <td>
+                                    <a href="{%s path.Join(pathPrefix, g.ID, ar.ID, "status") %}">Details</a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                     </tbody>
+                    </table>
+                {% endfor %}
+            </div>
+            <br>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func ListTargets(targets map[notifier.TargetType][]notifier.Target) %}
+    {%= tpl.Header("Notifiers", navItems) %}
+    {% if len(targets) > 0 %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+
+         {%code
+             var keys []string
+             for key := range targets {
+                 keys = append(keys, string(key))
+             }
+             sort.Strings(keys)
+         %}
+
+         {% for i := range keys  %}
+           {%code typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
+                count := len(ns)
+            %}
+           <div class="group-heading data-bs-target="rules-{%s typeK %}">
+             <span class="anchor" id="notifiers-{%s typeK %}"></span>
+             <a href="#notifiers-{%s typeK %}">{%s typeK %} ({%d count %})</a>
+         </div>
+         <div class="collapse show" id="notifiers-{%s typeK %}">
+             <table class="table table-striped table-hover table-sm">
+                 <thead>
+                     <tr>
+                         <th scope="col">Labels</th>
+                         <th scope="col">Address</th>
+                     </tr>
+                 </thead>
+                 <tbody>
+                 {% for _, n := range ns %}
+                     <tr>
+                         <td>
+                              {% for _, l := range n.Labels %}
+                                      <span class="ms-1 badge bg-primary">{%s l.Name %}={%s l.Value %}</span>
+                              {% endfor %}
+                          </td>
+                         <td>{%s n.Notifier.Addr() %}</td>
+                     </tr>
+                 {% endfor %}
+              </tbody>
+             </table>
+         </div>
+     {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func Alert(pathPrefix string, alert *APIAlert) %}
+    {%= tpl.Header("", navItems) %}
+    {%code
+        var labelKeys []string
+        for k := range alert.Labels {
+            labelKeys = append(labelKeys, k)
+        }
+        sort.Strings(labelKeys)
+
+        var annotationKeys []string
+        for k := range alert.Annotations {
+            annotationKeys = append(annotationKeys, k)
+        }
+        sort.Strings(annotationKeys)
+    %}
+    <div class="display-6 pb-3 mb-3">{%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Active at
+        </div>
+        <div class="col">
+          {%s alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+        </div>
+      </div>
+      </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Expr
+        </div>
+        <div class="col">
+          <code><pre>{%s alert.Expression %}</pre></code>
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Labels
+        </div>
+        <div class="col">
+           {% for _, k := range labelKeys %}
+                <span class="m-1 badge bg-primary">{%s k %}={%s alert.Labels[k] %}</span>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Annotations
+        </div>
+        <div class="col">
+           {% for _, k := range annotationKeys %}
+                <b>{%s k %}:</b><br>
+                <p>{%s alert.Annotations[k] %}</p>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Group
+        </div>
+        <div class="col">
+           <a target="_blank" href="{%s path.Join(pathPrefix,"groups") %}#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
+        </div>
+      </div>
+    </div>
+     <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Source link
+        </div>
+        <div class="col">
+           <a target="_blank" href="{%s alert.SourceLink %}">Link</a>
+        </div>
+      </div>
+    </div>
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func badgeState(state string) %}
+{%code
+    badgeClass := "bg-warning text-dark"
+    if state == "firing" {
+        badgeClass = "bg-danger"
+    }
+%}
+<span class="badge {%s badgeClass %}">{%s state %}</span>
+{% endfunc %}
+
+{% func badgeRestored() %}
+<span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
+{% endfunc %}

app/vmalert/web_test.go

@@ -54,9 +54,9 @@ func TestHandler(t *testing.T) {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 	})
-	t.Run("/api/v1/groups", func(t *testing.T) {
+	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/groups", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}

app/vmalert/web_types.go

@@ -4,58 +4,105 @@ import (
 	"time"
 )
 
-// APIAlert represents an notifier.AlertingRule state
+// APIAlert represents a notifier.AlertingRule state
 // for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIAlert struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
 	State       string            `json:"state"`
+	Name        string            `json:"name"`
 	Value       string            `json:"value"`
-	Labels      map[string]string `json:"labels"`
+	Labels      map[string]string `json:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations"`
 	ActiveAt    time.Time         `json:"activeAt"`
+
+	// Additional fields
+
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// RuleID is an unique Rule's ID within a group
+	RuleID string `json:"rule_id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+	// Expression contains the PromQL/MetricsQL expression
+	// for Rule's evaluation
+	Expression string `json:"expression"`
+	// SourceLink contains a link to a system which should show
+	// why Alert was generated
+	SourceLink string `json:"source"`
+	// Restored shows whether Alert's state was restored on restart
+	Restored bool `json:"restored"`
 }
 
 // APIGroup represents Group for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIGroup struct {
-	Name              string             `json:"name"`
-	Type              string             `json:"type"`
-	ID                string             `json:"id"`
-	File              string             `json:"file"`
-	Interval          string             `json:"interval"`
-	Concurrency       int                `json:"concurrency"`
-	ExtraFilterLabels map[string]string  `json:"extra_filter_labels"`
-	Labels            map[string]string  `json:"labels,omitempty"`
-	AlertingRules     []APIAlertingRule  `json:"alerting_rules"`
-	RecordingRules    []APIRecordingRule `json:"recording_rules"`
+	// Name is the group name as present in the config
+	Name string `json:"name"`
+	// Rules contains both recording and alerting rules
+	Rules []APIRule `json:"rules"`
+	// Interval is the Group's evaluation interval in float seconds as present in the file.
+	Interval float64 `json:"interval"`
+	// LastEvaluation is the timestamp of the last time the Group was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
+
+	// Additional fields
+
+	// Type shows the datasource type (prometheus or graphite) of the Group
+	Type string `json:"type"`
+	// ID is a unique Group ID
+	ID string `json:"id"`
+	// File contains a path to the file with Group's config
+	File string `json:"file"`
+	// Concurrency shows how many rules may be evaluated simultaneously
+	Concurrency int `json:"concurrency"`
+	// Params contains HTTP URL parameters added to each Rule's request
+	Params []string `json:"params,omitempty"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
-// APIAlertingRule represents AlertingRule for WEB view
-type APIAlertingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	For         string            `json:"for"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
-	Annotations map[string]string `json:"annotations"`
+// GroupAlerts represents a group of alerts for WEB view
+type GroupAlerts struct {
+	Group  APIGroup
+	Alerts []*APIAlert
 }
 
-// APIRecordingRule represents RecordingRule for WEB view
-type APIRecordingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
+// APIRule represents a Rule for WEB view
+// see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
+type APIRule struct {
+	// State must be one of these under following scenarios
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state.
+	//  "firing": at least 1 alert in the rule in firing state.
+	//  "inactive": no alert in the rule in firing or pending state.
+	State string `json:"state"`
+	Name  string `json:"name"`
+	// Query represents Rule's `expression` field
+	Query string `json:"query"`
+	// Duration represents Rule's `for` field
+	Duration    float64           `json:"duration"`
+	Labels      map[string]string `json:"labels,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// LastError contains the error faced while executing the rule.
+	LastError string `json:"lastError"`
+	// EvaluationTime is the time taken to completely evaluate the rule in float seconds.
+	EvaluationTime float64 `json:"evaluationTime"`
+	// LastEvaluation is the timestamp of the last time the rule was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
+	// Alerts  is the list of all the alerts in this rule that are currently pending or firing
+	Alerts []*APIAlert `json:"alerts,omitempty"`
+	// Health is the health of rule evaluation.
+	// It MUST be one of "ok", "err", "unknown"
+	Health string `json:"health"`
+	// Type of the rule: recording or alerting
+	Type string `json:"type"`
+
+	// Additional fields
+
+	// Type of the rule: recording or alerting
+	DatasourceType string `json:"datasourceType"`
+	LastSamples    int    `json:"lastSamples"`
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
 }

app/vmauth/Makefile

@@ -27,6 +27,15 @@ vmauth-ppc64le-prod:
 vmauth-386-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-386
 
+vmauth-darwin-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-amd64
+
+vmauth-darwin-arm64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-arm64
+
+vmauth-windows-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-windows-amd64
+
 package-vmauth:
 	APP_NAME=vmauth $(MAKE) package-via-docker
 
@@ -80,6 +89,3 @@ vmauth-pure:
 
 vmauth-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmauth $(MAKE) app-local-windows-with-goarch
-
-vmauth-windows-amd64-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-windows-amd64

app/vmauth/README.md

@@ -1,9 +1,9 @@
 # vmauth
 
 `vmauth` is a simple auth proxy, router and [load balancer](#load-balancing) for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
-It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) and `Bearer token` is supported),
+It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), `Bearer token` and [InfluxDB authorization](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1897) is supported),
 matches them against configs pointed by [-auth.config](#auth-config) command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
-
+The `-auth.config` can point to either local file or to http url.
 
 ## Quick start
 
@@ -26,27 +26,34 @@ Pass `-help` to `vmauth` in order to see all the supported command-line flags wi
 Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML,
 accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.com/vmgateway.html).
 
-
 ## Load balancing
 
 Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
-
 ## Auth config
 
 `-auth.config` is represented in the following simple `yml` format:
 
 ```yml
-
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.
 
 users:
-  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
+  # Requests with the 'Authorization: Bearer XXXX' and 'Authorization: Token XXXX'
+  # header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
   # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be proxied to http://localhost:8428 .
@@ -89,7 +96,6 @@ users:
   - "http://vminsert1:8480/insert/42/prometheus"
   - "http://vminsert2:8480/insert/42/prometheus"
 
-
   # A single user for querying and inserting data:
   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
   #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@@ -97,7 +103,8 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
   url_map:
   - src_paths:
@@ -109,14 +116,17 @@ users:
     - "http://vmselect2:8481/select/42/prometheus"
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"
 ```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
-
 ## Security
 
+It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.
+
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
 
 ```
@@ -132,21 +142,27 @@ Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termi
 
 It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
 
-
 ## Monitoring
 
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
+`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+
+```yml
+users:
+- username: "secret_user"
+  name: "foobar"
+  # other config options here
+```
 
 ## How to build from sources
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.
 
-
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth` binary and puts it into the `bin` folder.
 
@@ -169,28 +185,34 @@ by setting it via `<ROOT_IMAGE>` environment variable. For example, the followin
 ROOT_IMAGE=scratch make package-vmauth
 ```
 
-
 ## Profiling
 
 `vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
 
-* Memory profile. It can be collected with the following command:
+* Memory profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
 
 ```bash
-curl -s http://<vmauth-host>:8427/debug/pprof/heap > mem.pprof
+curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof
 ```
 
-* CPU profile. It can be collected with the following command:
+</div>
+
+* CPU profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
 
 ```bash
-curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
+curl http://0.0.0.0:8427/debug/pprof/profile > cpu.pprof
 ```
 
+</div>
+
 The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
 
-
 ## Advanced usage
 
 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
@@ -203,13 +225,15 @@ vmauth authenticates and authorizes incoming requests and proxies them to Victor
 See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
-    	Path to auth config. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+    	Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
@@ -230,6 +254,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
     	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
     	TCP address to listen for http connections (default ":8427")
+  -logInvalidAuthTokens
+    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -252,17 +278,17 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -reloadAuthKey string
     	Auth key for /-/reload http endpoint. It must be passed as authKey=...
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
     	Show VictoriaMetrics version
 ```

app/vmauth/auth_config.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"regexp"
@@ -14,6 +13,7 @@ import (
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,30 +21,60 @@ import (
 )
 
 var (
-	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://docs.victoriametrics.com/vmauth.html "+
-		"for details on the format of this auth config")
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
 )
 
 // AuthConfig represents auth config.
 type AuthConfig struct {
-	Users []UserInfo `yaml:"users"`
+	Users []UserInfo `yaml:"users,omitempty"`
 }
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	BearerToken string     `yaml:"bearer_token"`
-	Username    string     `yaml:"username"`
-	Password    string     `yaml:"password"`
-	URLPrefix   *URLPrefix `yaml:"url_prefix"`
-	URLMap      []URLMap   `yaml:"url_map"`
+	Name        string     `yaml:"name,omitempty"`
+	BearerToken string     `yaml:"bearer_token,omitempty"`
+	Username    string     `yaml:"username,omitempty"`
+	Password    string     `yaml:"password,omitempty"`
+	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMap      []URLMap   `yaml:"url_map,omitempty"`
+	Headers     []Header   `yaml:"headers,omitempty"`
 
 	requests *metrics.Counter
 }
 
+// Header is `Name: Value` http header, which must be added to the proxied request.
+type Header struct {
+	Name  string
+	Value string
+}
+
+// UnmarshalYAML unmarshals h from f.
+func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
+	var s string
+	if err := f(&s); err != nil {
+		return err
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
+	}
+	h.Name = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
+}
+
+// MarshalYAML marshals h to yaml.
+func (h *Header) MarshalYAML() (interface{}, error) {
+	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
+	return s, nil
+}
+
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths"`
-	URLPrefix *URLPrefix `yaml:"url_prefix"`
+	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+	Headers   []Header   `yaml:"headers,omitempty"`
 }
 
 // SrcPath represents an src path
@@ -207,9 +237,9 @@ var authConfigWG sync.WaitGroup
 var stopCh chan struct{}
 
 func readAuthConfig(path string) (map[string]*UserInfo, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := fs.ReadFileOrHTTP(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read %q: %w", path, err)
+		return nil, err
 	}
 	m, err := parseAuthConfig(data)
 	if err != nil {
@@ -246,9 +276,12 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if byUsername[ui.Username] {
 			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
 		}
-		authToken := getAuthToken(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[authToken] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", authToken, ui.BearerToken, ui.Username)
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+		}
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
 		}
 		if ui.URLPrefix != nil {
 			if err := ui.URLPrefix.sanitize(); err != nil {
@@ -270,21 +303,41 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
 		if ui.BearerToken != "" {
+			name := "bearer_token"
+			if ui.Name != "" {
+				name = ui.Name
+			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
-			ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total{username="bearer_token"}`)
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byBearerToken[ui.BearerToken] = true
 		}
 		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+			name := ui.Username
+			if ui.Name != "" {
+				name = ui.Name
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byUsername[ui.Username] = true
 		}
-		byAuthToken[authToken] = ui
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
+func getAuthTokens(bearerToken, username, password string) (string, string) {
+	if bearerToken != "" {
+		// Accept the bearerToken as Basic Auth username with empty password
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
+	}
+	at := getAuthToken("", username, password)
+	return at, at
+}
+
 func getAuthToken(bearerToken, username, password string) string {
 	if bearerToken != "" {
 		return "Bearer " + bearerToken

app/vmauth/auth_config_test.go

@@ -69,6 +69,14 @@ users:
   - [foo]
 `)
 
+	// Invalid headers
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  headers: foobar
+`)
+
 	// empty url_prefix
 	f(`
 users:
@@ -156,6 +164,27 @@ users:
   - src_paths: ['fo[obar']
     url_prefix: http://foobar
 `)
+
+	// Invalid headers in url_map (missing ':')
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+    - foobar
+`)
+	// Invalid headers in url_map (dictionary instead of array)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+      aaa: bbb
+`)
 }
 
 func TestParseAuthConfigSuccess(t *testing.T) {
@@ -231,6 +260,9 @@ users:
     url_prefix: http://vmselect/select/0/prometheus
   - src_paths: ["/api/v1/write"]
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
+    headers:
+    - "foo: bar"
+    - "xxx: y"
 `, map[string]*UserInfo{
 		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
@@ -245,6 +277,42 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMap: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
 				},
 			},
 		},

app/vmauth/example_config.yml

@@ -1,12 +1,22 @@
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.
 
 users:
-  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
+  # Requests with the 'Authorization: Bearer XXXX' and 'Authorization: Token XXXX'
+  # header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
   # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be proxied to http://localhost:8428 .
@@ -49,7 +59,6 @@ users:
   - "http://vminsert1:8480/insert/42/prometheus"
   - "http://vminsert2:8480/insert/42/prometheus"
 
-
   # A single user for querying and inserting data:
   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
   #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@@ -57,7 +66,8 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
   url_map:
   - src_paths:
@@ -69,3 +79,5 @@ users:
     - "http://vmselect2:8481/select/42/prometheus"
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"

app/vmauth/main.go

@@ -2,10 +2,13 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -21,6 +24,8 @@ var (
 	httpListenAddr         = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
 	reloadAuthKey          = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	logInvalidAuthTokens   = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
+		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 )
 
 func main() {
@@ -68,19 +73,33 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
+	if strings.HasPrefix(authToken, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
+	}
 	ac := authConfig.Load().(map[string]*UserInfo)
 	ui := ac[authToken]
 	if ui == nil {
-		httpserver.Errorf(w, r, "cannot find the provided auth token %q in config", authToken)
+		invalidAuthTokenRequests.Inc()
+		if *logInvalidAuthTokens {
+			httpserver.Errorf(w, r, "cannot find the provided auth token %q in config", authToken)
+		} else {
+			errStr := fmt.Sprintf("cannot find the provided auth token %q in config", authToken)
+			http.Error(w, errStr, http.StatusBadRequest)
+		}
 		return true
 	}
 	ui.requests.Inc()
-	targetURL, err := createTargetURL(ui, r.URL)
+	targetURL, headers, err := createTargetURL(ui, r.URL)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
 		return true
 	}
 	r.Header.Set("vm-target-url", targetURL.String())
+	for _, h := range headers {
+		r.Header.Set(h.Name, h.Value)
+	}
 	proxyRequest(w, r)
 	return true
 }
@@ -96,31 +115,51 @@ func proxyRequest(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	reverseProxy.ServeHTTP(w, r)
+	getReverseProxy().ServeHTTP(w, r)
 }
 
-var configReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+var (
+	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
+	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
+	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+)
 
-var reverseProxy = &httputil.ReverseProxy{
-	Director: func(r *http.Request) {
-		targetURL := r.Header.Get("vm-target-url")
-		target, err := url.Parse(targetURL)
-		if err != nil {
-			logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
-		}
-		r.URL = target
-	},
-	Transport: func() *http.Transport {
-		tr := http.DefaultTransport.(*http.Transport).Clone()
-		// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
-		tr.DisableCompression = true
-		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
-		tr.ForceAttemptHTTP2 = false
-		tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
-		return tr
-	}(),
-	FlushInterval: time.Second,
-	ErrorLog:      logger.StdErrorLogger(),
+var (
+	reverseProxy     *httputil.ReverseProxy
+	reverseProxyOnce sync.Once
+)
+
+func getReverseProxy() *httputil.ReverseProxy {
+	reverseProxyOnce.Do(initReverseProxy)
+	return reverseProxy
+}
+
+// initReverseProxy must be called after flag.Parse(), since it uses command-line flags.
+func initReverseProxy() {
+	reverseProxy = &httputil.ReverseProxy{
+		Director: func(r *http.Request) {
+			targetURL := r.Header.Get("vm-target-url")
+			target, err := url.Parse(targetURL)
+			if err != nil {
+				logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
+			}
+			r.URL = target
+		},
+		Transport: func() *http.Transport {
+			tr := http.DefaultTransport.(*http.Transport).Clone()
+			// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+			tr.DisableCompression = true
+			// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+			tr.ForceAttemptHTTP2 = false
+			tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
+			if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+				tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+			}
+			return tr
+		}(),
+		FlushInterval: time.Second,
+		ErrorLog:      logger.StdErrorLogger(),
+	}
 }
 
 func usage() {