docs/CHANGELOG.md: cut v1.74.0

vmui: update packages (#2264 )
* update package.json * update package-lock.json * app/vmselect: `make vmui-update` Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
2026-06-08 11:23:53 +03:00 · 2022-03-03 19:30:41 +02:00 · 2022-03-03 18:49:13 +02:00 · 2022-03-03 18:34:06 +02:00 · 2022-03-03 18:28:46 +02:00 · 2022-03-03 18:27:12 +02:00
1217 changed files with 86855 additions and 334338 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -4,3 +4,4 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
+.vscode
--- a/.github/workflows/check-licenses.yml
+++ b/.github/workflows/check-licenses.yml
@@ -14,7 +14,7 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@main
        with:
-          go-version: 1.16
+          go-version: 1.17
        id: go
      - name: Code checkout
        uses: actions/checkout@master
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -16,7 +16,7 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@main
        with:
-          go-version: 1.16
+          go-version: 1.17
        id: go
      - name: Code checkout
        uses: actions/checkout@master
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 *.pprof
 /bin
 .idea
+.vscode
 *.test
 *.swp
 /gocache-for-docker
--- a/2
+++ b/2
@@ -175,7 +175,7 @@

   END OF TERMS AND CONDITIONS

-   Copyright 2019-2021 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
--- a/92
+++ b/92
@@ -24,6 +24,8 @@ all: \

 include app/*/Makefile
 include deployment/*/Makefile
+include snap/local/Makefile
+

 clean:
 	rm -rf bin/*
@@ -84,9 +86,6 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64

-release-snap:
-	snapcraft
-	snapcraft upload "victoriametrics_$(PKG_TAG)_multi.snap" --release beta,edge,candidate

 publish-release:
 	git checkout $(TAG) && $(MAKE) release publish && \
@@ -101,66 +100,82 @@ release: \
 release-victoria-metrics: \
 	release-victoria-metrics-amd64 \
 	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64
+	release-victoria-metrics-arm64 \
+	release-victoria-metrics-darwin-amd64 \
+	release-victoria-metrics-darwin-arm64

 release-victoria-metrics-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic

 release-victoria-metrics-arm:
-	GOARCH=arm $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm $(MAKE) release-victoria-metrics-generic

 release-victoria-metrics-arm64:
-	GOARCH=arm64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic

-release-victoria-metrics-generic: victoria-metrics-$(GOARCH)-prod
+release-victoria-metrics-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt

 release-vmutils: \
 	release-vmutils-amd64 \
 	release-vmutils-arm64 \
 	release-vmutils-arm \
+	release-vmutils-darwin-amd64  \
+	release-vmutils-darwin-arm64 \
 	release-vmutils-windows-amd64

 release-vmutils-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-generic
+	OSARCH=amd64 $(MAKE) release-vmutils-generic

 release-vmutils-arm64:
-	GOARCH=arm64 $(MAKE) release-vmutils-generic
+	OSARCH=arm64 $(MAKE) release-vmutils-generic

 release-vmutils-arm:
-	GOARCH=arm $(MAKE) release-vmutils-generic
+	OSARCH=arm $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic

 release-vmutils-windows-amd64:
 	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic

 release-vmutils-generic: \
-	vmagent-$(GOARCH)-prod \
-	vmalert-$(GOARCH)-prod \
-	vmauth-$(GOARCH)-prod \
-	vmbackup-$(GOARCH)-prod \
-	vmrestore-$(GOARCH)-prod \
-	vmctl-$(GOARCH)-prod
+	vmagent-$(OSARCH)-prod \
+	vmalert-$(OSARCH)-prod \
+	vmauth-$(OSARCH)-prod \
+	vmbackup-$(OSARCH)-prod \
+	vmrestore-$(OSARCH)-prod \
+	vmctl-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > vmutils-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt

 release-vmutils-windows-generic: \
 	vmagent-windows-$(GOARCH)-prod \
@@ -180,6 +195,7 @@ release-vmutils-windows-generic: \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt

+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)

@@ -267,7 +283,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m

 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.44.1

 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
--- a/README.md
+++ b/README.md
@@ -13,46 +13,13 @@
 VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.

 VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
-in [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), in [Snap packages](https://snapcraft.io/victoriametrics)
-and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics follow [these instructions](#how-to-start-victoriametrics).
+[Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap packages](https://snapcraft.io/victoriametrics)
+and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and follow [these instructions](#how-to-start-victoriametrics).
 Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.

 Cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).

-[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics.
-See [features available in enterprise package](https://victoriametrics.com/enterprise.html).
-Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
-
-
-## Case studies and talks
-
-Case studies:
-
-* [AbiosGaming](https://docs.victoriametrics.com/CaseStudies.html#abiosgaming)
-* [adidas](https://docs.victoriametrics.com/CaseStudies.html#adidas)
-* [Adsterra](https://docs.victoriametrics.com/CaseStudies.html#adsterra)
-* [ARNES](https://docs.victoriametrics.com/CaseStudies.html#arnes)
-* [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
-* [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
-* [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
-* [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
-* [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
-* [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
-* [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
-* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
-* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
-* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
-* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
-* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
-* [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
-* [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
-* [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
-* [Wedos.com](https://docs.victoriametrics.com/CaseStudies.html#wedoscom)
-* [Wix.com](https://docs.victoriametrics.com/CaseStudies.html#wixcom)
-* [Zerodha](https://docs.victoriametrics.com/CaseStudies.html#zerodha)
-* [zhihu](https://docs.victoriametrics.com/CaseStudies.html#zhihu)
-
-See also [articles and slides about VictoriaMetrics from our users](https://docs.victoriametrics.com/Articles.html#third-party-articles-and-slides-about-victoriametrics)
+[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. See [features available in enterprise package](https://victoriametrics.com/products/enterprise/). Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).


 ## Prominent features
@@ -89,12 +56,43 @@ VictoriaMetrics has the following prominent features:
  * [Native binary format](#how-to-import-data-in-native-format).
 * It supports metrics' relabeling. See [these docs](#relabeling) for details.
 * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
-* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/enterprise.html).
+* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
 * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).

 See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html).


+## Case studies and talks
+
+Case studies:
+
+* [AbiosGaming](https://docs.victoriametrics.com/CaseStudies.html#abiosgaming)
+* [adidas](https://docs.victoriametrics.com/CaseStudies.html#adidas)
+* [Adsterra](https://docs.victoriametrics.com/CaseStudies.html#adsterra)
+* [ARNES](https://docs.victoriametrics.com/CaseStudies.html#arnes)
+* [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
+* [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
+* [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
+* [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
+* [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
+* [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
+* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
+* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
+* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
+* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
+* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
+* [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
+* [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
+* [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
+* [Wedos.com](https://docs.victoriametrics.com/CaseStudies.html#wedoscom)
+* [Wix.com](https://docs.victoriametrics.com/CaseStudies.html#wixcom)
+* [Zerodha](https://docs.victoriametrics.com/CaseStudies.html#zerodha)
+* [zhihu](https://docs.victoriametrics.com/CaseStudies.html#zhihu)
+
+See also [articles and slides about VictoriaMetrics from our users](https://docs.victoriametrics.com/Articles.html#third-party-articles-and-slides-about-victoriametrics)
+
+
 ## Operation

 ## How to start VictoriaMetrics
@@ -104,7 +102,7 @@ Just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/Vi
 The following command-line flags are used the most:

 * `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
-* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. See [these docs](#retention) for more details.
+* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. See [the Retention section](#retention) for more details.

 Other flags have good enough default values, so set them only if you really need this. Pass `-help` to see [all the available flags with description and default values](#list-of-command-line-flags).

@@ -276,6 +274,8 @@ VictoriaMetrics accepts data from [DataDog agent](https://docs.datadoghq.com/age

 Run DataDog agent with `DD_DD_URL=http://victoriametrics-host:8428/datadog` environment variable in order to write data to VictoriaMetrics at `victoriametrics-host` host. Another option is to set `dd_url` param at [DataDog agent configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) to `http://victoriametrics-host:8428/datadog`.

+VictoriaMetrics doesn't check `DD_API_KEY` param, so it can be set to arbitrary value.
+
 Example on how to send data to VictoriaMetrics via DataDog "submit metrics" API from command line:

 ```bash
@@ -332,7 +332,7 @@ and stream plain InfluxDB line protocol data to the configured TCP and/or UDP ad
 VictoriaMetrics performs the following transformations to the ingested InfluxDB data:

 * [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
-  unless `db` tag exists in the InfluxDB line.
+  unless `db` tag exists in the InfluxDB line. The `db` label name can be overriden via `-influxDBLabel` command-line flag.
 * Field names are mapped to time series names prefixed with `{measurement}{separator}` value, where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag. See also `-influxSkipSingleField` command-line flag. If `{measurement}` is empty or if `-influxSkipMeasurement` command-line flag is set, then time series names correspond to field names.
 * Field values are mapped to time series values.
 * Tags are mapped to Prometheus labels as-is.
@@ -418,14 +418,20 @@ The `/api/v1/export` endpoint should return the following response:
 Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:

 * [Graphite API](#graphite-api-usage)
-* [Prometheus querying API](#prometheus-querying-api-usage). VictoriaMetrics supports `__graphite__` pseudo-label for selecting time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster and it is easier to use when migrating from Graphite to VictoriaMetrics. VictoriaMetrics also supports [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function for extracting the given groups from Graphite metric name.
+* [Prometheus querying API](#prometheus-querying-api-usage). See also [selecting Graphite metrics](#selecting-graphite-metrics).
 * [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml)

+## Selecting Graphite metrics
+
+VictoriaMetrics supports `__graphite__` pseudo-label for selecting time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster and it is easier to use when migrating from Graphite to VictoriaMetrics. See [docs for Graphite paths and wildcards](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). VictoriaMetrics also supports [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function for extracting the given groups from Graphite metric name.
+
+The `__graphite__` pseudo-label supports e.g. alternate regexp filters such as `(value1|...|valueN)`. They are transparently converted to `{value1,...,valueN}` syntax [used in Graphite](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). This allows using [multi-value template variables in Grafana](https://grafana.com/docs/grafana/latest/variables/formatting-multi-value-variables/) inside `__graphite__` pseudo-label. For example, Grafana expands `{__graphite__=~"foo.($bar).baz"}` into `{__graphite__=~"foo.(x|y).baz"}` if `$bar` template variable contains `x` and `y` values. In this case the query is automatically converted into `{__graphite__=~"foo.{x,y}.baz"}` before execution.
+
 ## How to send data from OpenTSDB-compatible agents

 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
 and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
-The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/build/html/PushingData.html).
+The same protocol is used for [ingesting data in KairosDB](https://kairosdb.github.io/docs/PushingData.html).

 ### Sending data via `telnet put` protocol

@@ -517,9 +523,10 @@ All the Prometheus querying API handlers can be prepended with `/prometheus` pre
 ### Prometheus querying API enhancements

 VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used for enforcing additional label filters for queries. For example,
-`/api/v1/query_range?extra_label=user_id=123&query=<query>` would automatically add `{user_id="123"}` label filter to the given `<query>`. This functionality can be used
-for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting
-in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+`/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>` would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+
+VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
+`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.

 VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
 For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
@@ -556,17 +563,16 @@ VictoriaMetrics supports the following Graphite APIs, which are needed for [Grap

 All the Graphite handlers can be pre-pended with `/graphite` prefix. For example, both `/graphite/metrics/find` and `/metrics/find` should work.

-VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg for all the Graphite APIs. This arg can be used for limiting the scope of time series
-visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting in front of VictoriaMetrics.
+VictoriaMetrics accepts optional query args: `extra_label=<label_name>=<label_value>` and `extra_filters[]=series_selector` query args for all the Graphite APIs. These args can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+
 [Contact us](mailto:sales@victoriametrics.com) if you need assistance with such a proxy.

-VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html).
-For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster and it is easier to use when migrating from Graphite to VictoriaMetrics. See also [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function.
+VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). See [these docs](#selecting-graphite-metrics).


 ### Graphite Render API usage

-[VictoriaMetrics Enterprise](https://victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
+[VictoriaMetrics Enterprise](https://victoriametrics.com/products/enterprise/) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
 at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
 When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
 Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
@@ -612,6 +618,10 @@ Query history can be navigated by holding `Ctrl` (or `Cmd` on MacOS) and pressin

 When querying the [backfilled data](https://docs.victoriametrics.com/#backfilling), it may be useful disabling response cache by clicking `Enable cache` checkbox.

+VMUI automatically adjusts the interval between datapoints on the graph depending on the horizontal resolution and on the selected time range. The step value can be customized by clickhing `Override step value` checkbox.
+
+VMUI allows investigating correlations between two queries on the same graph. Just click `+Query` button, enter the second query in the newly appeared input field and press `Ctrl+Enter`. Results for both queries should be displayed simultaneously on the same graph. Every query has its own vertical scale, which is displayed on the left and the right side of the graph. Lines for the second query are dashed.
+
 See the [example VMUI at VictoriaMetrics playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/?g0.expr=100%20*%20sum(rate(process_cpu_seconds_total))%20by%20(job)&g0.range_input=1d).


@@ -624,7 +634,7 @@ to your needs or when testing bugfixes.

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `victoria-metrics` binary and puts it into the `bin` folder.

@@ -640,7 +650,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 ### Development ARM build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics-arm` or `make victoria-metrics-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `victoria-metrics-arm` or `victoria-metrics-arm64` binary respectively and puts it into the `bin` folder.

@@ -654,7 +664,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 `Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics-pure` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.

@@ -734,7 +744,7 @@ The delete API is intended mainly for the following cases:
 * One-off deleting of accidentally written invalid (or undesired) time series.
 * One-off deleting of user data due to [GDPR](https://en.wikipedia.org/wiki/General_Data_Protection_Regulation).

-It isn't recommended using delete API for the following cases, since it brings non-zero overhead:
+Using the delete API is not recommended in the following cases, since it brings a non-zero overhead:

 * Regular cleanups for unneeded data. Just prevent writing unneeded data into VictoriaMetrics.
  This can be done with [relabeling](#relabeling).
@@ -743,7 +753,7 @@ It isn't recommended using delete API for the following cases, since it brings n
  time series occupy disk space until the next merge operation, which can never occur when deleting too old data.
  [Forced merge](#forced-merge) may be used for freeing up disk space occupied by old data.

-It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
+It's better to use the `-retentionPeriod` command-line flag for efficient pruning of old data.


 ## Forced merge
@@ -765,33 +775,10 @@ when new data is ingested into it.

 VictoriaMetrics provides the following handlers for exporting data:

-* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
-  See [these docs](#how-to-export-data-in-native-format) for details.
 * `/api/v1/export` for exporing data in JSON line format. See [these docs](#how-to-export-data-in-json-line-format) for details.
 * `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
-
-
-### How to export data in native format
-
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
-where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
-for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
-
-On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
-
-```bash
-# count unique timeseries in database
-wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
-
-# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
-```
-
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
-
-The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
-The native export format may change in incompatible way between VictoriaMetrics releases, so the data exported from the release X
-can fail to be imported into VictoriaMetrics release Y.
+* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
+  See [these docs](#how-to-export-data-in-native-format) for details.


 ### How to export data in JSON line format
@@ -812,7 +799,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val

 Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
 Optional `reduce_mem_usage=1` arg may be added to the request for reducing memory usage when exporting big number of time series.
-In this case the output may contain multiple lines with distinct samples for the same time series.
+In this case the output may contain multiple lines with samples for the same time series.

 Pass `Accept-Encoding: gzip` HTTP header in the request to `/api/v1/export` in order to reduce network bandwidth during exporing big amounts
 of time series data. This enables gzip compression for the exported data. Example for exporting gzipped data:
@@ -825,6 +812,9 @@ The maximum duration for each request to `/api/v1/export` is limited by `-search

 Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).

+The [deduplication](#deduplication) is applied to the data exported via `/api/v1/export` by default. The deduplication
+isn't applied if `reduce_mem_usage=1` query arg is passed to the request.
+

 ### How to export CSV data

@@ -849,10 +839,37 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val

 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).

+The [deduplication](#deduplication) is applied for the data exported in CSV by default. It is possible to export raw data without de-duplication by passing `reduce_mem_usage=1` query arg to `/api/v1/export/csv`.
+
+
+### How to export data in native format
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
+where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
+
+On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
+
+```bash
+# count unique timeseries in database
+wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
+
+# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
+```
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
+The native export format may change in incompatible way between VictoriaMetrics releases, so the data exported from the release X
+can fail to be imported into VictoriaMetrics release Y.
+
+The [deduplication](#deduplication) isn't applied for the data exported in native format. It is expected that the de-duplication is performed during data import.
+

 ## How to import time series data

-Time series data can be imported via any supported ingestion protocol:
+Time series data can be imported into VictoriaMetrics via any supported ingestion protocol:

 * [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). See [these docs](#prometheus-setup) for details.
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
@@ -868,36 +885,6 @@ Time series data can be imported via any supported ingestion protocol:
 * `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.


-### How to import data in native format
-
-The specification of VictoriaMetrics' native format may yet change and is not formally documented yet. So currently we do not recommend that external clients attempt to pack their own metrics in native format file.
-
-If you have a native format file obtained via [/api/v1/export/native](#how-to-export-data-in-native-format) however this is the most efficient protocol for importing data in.
-
-```bash
-# Export the data from <source-victoriametrics>:
-curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
-
-# Import the data to <destination-victoriametrics>:
-curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
-```
-
-Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/native` for importing gzipped data:
-
-```bash
-# Export gzipped data from <source-victoriametrics>:
-curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin.gz
-
-# Import gzipped data to <destination-victoriametrics>:
-curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin.gz
-```
-
-Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
-For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
-
-Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
-
-
 ### How to import data in JSON line format

 Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
@@ -928,6 +915,26 @@ Note that it could be required to flush response cache after importing historica
 VictoriaMetrics parses input JSON lines one-by-one. It loads the whole JSON line in memory, then parses it and then saves the parsed samples into persistent storage. This means that VictoriaMetrics can occupy big amounts of RAM when importing too long JSON lines. The solution is to split too long JSON lines into smaller lines. It is OK if samples for a single time series are split among multiple JSON lines.


+### How to import data in native format
+
+The specification of VictoriaMetrics' native format may yet change and is not formally documented yet. So currently we do not recommend that external clients attempt to pack their own metrics in native format file.
+
+If you have a native format file obtained via [/api/v1/export/native](#how-to-export-data-in-native-format) however this is the most efficient protocol for importing data in.
+
+```bash
+# Export the data from <source-victoriametrics>:
+curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
+```
+
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
 ### How to import CSV data

 Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
@@ -1003,6 +1010,13 @@ It should return something like the following:
 {"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
 ```

+Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/prometheus` for importing gzipped data:
+
+```bash
+# Import gzipped data to <destination-victoriametrics>:
+curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/prometheus -T prometheus_data.gz
+```
+
 Extra labels may be added to all the imported metrics by passing `extra_label=name=value` query args.
 For example, `/api/v1/import/prometheus?extra_label=foo=bar` would add `{foo="bar"}` label to all the imported metrics.

@@ -1021,6 +1035,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc

 VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
 to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
+The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
 See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).

 Example contents for `-relabelConfig` file:
@@ -1054,7 +1069,7 @@ with scrape intervals exceeding `5m`.

 VictoriaMetrics uses lower amounts of CPU, RAM and storage space on production workloads compared to competing solutions (Prometheus, Thanos, Cortex, TimescaleDB, InfluxDB, QuestDB, M3DB) according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html).

-VictoriaMetrics capacity scales linearly with the available resources. The needed amounts of CPU and RAM highly depends on the workload - the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series), series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate), query types, query qps, etc. It is recommended setting up a test VictoriaMetrics for your production workload and iteratively scaling CPU and RAM resources until it becomes stable according to [troubleshooting docs](#troubleshooting). A single-node VictoriaMetrics works perfectly with the following production workload according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html):
+VictoriaMetrics capacity scales linearly with the available resources. The needed amounts of CPU and RAM highly depends on the workload - the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series), series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate), query types, query qps, etc. It is recommended setting up a test VictoriaMetrics for your production workload and iteratively scaling CPU and RAM resources until it becomes stable according to [troubleshooting docs](#troubleshooting). A single-node VictoriaMetrics works perfectly with the following production workload according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html):

 * Ingestion rate: 1.5+ million samples per second
 * Active time series: 50+ million
@@ -1070,7 +1085,7 @@ It is recommended leaving the following amounts of spare resources:

 * 50% of free RAM for reducing the probability of OOM (out of memory) crashes and slowdowns during temporary spikes in workload.
 * 50% of spare CPU for reducing the probability of slowdowns during temporary spikes in workload.
-* At least 30% of free storage space at the directory pointed by `-storageDataPath` command-line flag.
+* At least 30% of free storage space at the directory pointed by `-storageDataPath` command-line flag. See also `-storage.minFreeDiskSpaceBytes` command-line flag description [here](#list-of-command-line-flags).


 ## High availability
@@ -1116,9 +1131,9 @@ with the enabled de-duplication. See [this section](#deduplication) for details.

 ## Deduplication

-VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag
-is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points
-on the same time series if they fall within the same discrete 60s bucket.  The earliest data point will be kept.  In the case of equal timestamps, an arbitrary data point will be kept.
+VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points on the same time series if they fall within the same discrete 60s bucket.  The earliest data point will be kept. In the case of equal timestamps, an arbitrary data point will be kept. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2112#issuecomment-1032587618) for more details on how downsampling works.
+
+The `-dedup.minScrapeInterval=D` is equivalent to `-downsampling.period=0s:D` if [downsampling](#downsampling) is enabled. It is safe to use deduplication and downsampling simultaneously.

 The recommended value for `-dedup.minScrapeInterval` must equal to `scrape_interval` config from Prometheus configs. It is recommended to have a single `scrape_interval` across all the scrape targets. See [this article](https://www.robustperception.io/keep-it-simple-scrape_interval-id) for details.

@@ -1127,24 +1142,63 @@ write data to the same VictoriaMetrics instance. These vmagent or Prometheus ins
 `external_labels` section in their configs, so they write data to the same time series.


+## Storage
+
+VictoriaMetrics stores time series data in [MergeTree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)-like 
+data structures. On insert, VictoriaMetrics accumulates up to 1s of data and dumps it on disk to
+`<-storageDataPath>/data/small/YYYY_MM/` subdirectory forming a `part` with the following 
+name pattern: `rowsCount_blocksCount_minTimestamp_maxTimestamp`. Each part consists of two "columns":
+values and timestamps. These are sorted and compressed raw time series values. Additionally, part contains
+index files for searching for specific series in the values and timestamps files.
+
+`Parts` are periodically merged into the bigger parts. The resulting `part` is constructed 
+under `<-storageDataPath>/data/{small,big}/YYYY_MM/tmp` subdirectory. When the resulting `part` is complete, it is atomically moved from the `tmp` 
+to its own subdirectory, while the source parts are atomically removed. The end result is that the source 
+parts are substituted by a single resulting bigger `part` in the `<-storageDataPath>/data/{small,big}/YYYY_MM/` directory.
+Information about merging process is available in [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) 
+and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176) Grafana dashboards. 
+See more details in [monitoring docs](#monitoring).
+
+The `merge` process is usually named "compaction", because the resulting `part` size is usually smaller than 
+the sum of the source `parts`. There are following benefits of doing the merge process:
+* it improves query performance, since lower number of `parts` are inspected with each query;
+* it reduces the number of data files, since each `part`contains fixed number of files; 
+* better compression rate for the resulting part.
+
+Newly added `parts` either appear in the storage or fail to appear. 
+Storage never contains partially created parts. The same applies to merge process — `parts` are either fully 
+merged into a new `part` or fail to merge. There are no partially merged `parts` in MergeTree. 
+`Part` contents in MergeTree never change. Parts are immutable. They may be only deleted after the merge 
+to a bigger `part` or when the `part` contents goes outside the configured `-retentionPeriod`.
+
+See [this article](https://valyala.medium.com/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) for more details.
+
+See also [how to work with snapshots](#how-to-work-with-snapshots).
+
 ## Retention

-Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
-that the data will be stored for 3 months and then deleted.
-Data is split in per-month subdirectories inside `<-storageDataPath>/data/small` and `<-storageDataPath>/data/big` folders.
-Directories for months outside the configured retention are deleted on the first day of new month.
-In order to keep data according to `-retentionPeriod` max disk space usage is going to be `-retentionPeriod` + 1 month.
-For example if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
-It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod` is set to lower
-value than before then data outside the configured period will be eventually deleted.
+Retention is configured with the `-retentionPeriod` command-line flag, which takes a number followed by a time unit character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month is assumed. For instance, `-retentionPeriod=3` means that the data will be stored for 3 months and then deleted. The default retention period is one month.

-VictoriaMetrics supports retention smaller than 1 month. For example, `-retentionPeriod=5d` would set data retention for 5 days.
-Older data is eventually deleted during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
+Data partitions outside the configured retention are deleted on the first day of the new month.
+Each partition consists of one or more data parts with the following name pattern `rowsCount_blocksCount_minTimestamp_maxTimestamp`.
+Data parts outside of the configured retention are eventually deleted during 
+[background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).

+The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
+For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
+
+Please note, the time range covered by data part is not limited by retention period unit. Hence, data part may contain data
+for multiple days and will be deleted only when fully outside of the configured retention.
+
+It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod` is set to a lower
+value than before, then data outside the configured period will be eventually deleted.
+
+VictoriaMetrics does not support indefinite retention, but you can specify an arbitrarily high duration, e.g. `-retentionPeriod=100y`.

 ## Multiple retentions

-Just start multiple VictoriaMetrics instances with distinct values for the following flags:
+A single instance of VictoriaMetrics supports only a single retention, which can be configured via `-retentionPeriod` command-line flag. If you need multiple retentions, then you may start multiple VictoriaMetrics instances with distinct values for the following flags:

 * `-retentionPeriod`
 * `-storageDataPath`, so the data for each retention period is saved in a separate directory
@@ -1153,24 +1207,20 @@ Just start multiple VictoriaMetrics instances with distinct values for the follo
 Then set up [vmauth](https://docs.victoriametrics.com/vmauth.html) in front of VictoriaMetrics instances,
 so it could route requests from particular user to VictoriaMetrics with the desired retention.
 The same scheme could be implemented for multiple tenants in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+See [these docs](https://docs.victoriametrics.com/guides/guide-vmcluster-multiple-retention-setup.html) for multi-retention setup details.


 ## Downsampling

-There is no downsampling support at the moment, but:
+[VictoriaMetrics Enterprise](https://victoriametrics.com/products/enterprise/) supports multi-level downsampling with `-downsampling.period` command-line flag. For example:

-* VictoriaMetrics is optimized for querying big amounts of raw data. See benchmark results for heavy queries
-  in [this article](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
-  for details.
-* The downsampling doesn't improve query performance on a long time range if the time range contains big number of time series due to [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). The query performance depends on the number of unique time series on the selected time range, while downsampling doesn't reduce the number of unique time series in the database - it can reduce only the number of samples per each time series.
+* `-downsampling.period=30d:5m` instructs VictoriaMetrics to [deduplicate](#deduplication) samples older than 30 days with 5 minutes interval.

-These properties reduce the need of downsampling. We plan to implement downsampling in the future.
-See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
+* `-downsampling.period=30d:5m,180d:1h` instructs VictoriaMetrics to deduplicate samples older than 30 days with 5 minutes interval and to deduplicate samples older than 180 days with 1 hour interval.

-It is possible to (ab)use [-dedup.minScrapeInterval](#deduplication) for basic downsampling.
-For instance, if interval between the ingested data points is 15s, then `-dedup.minScrapeInterval=5m` will leave
-only a single data point out of 20 initial data points per each 5m interval.
+Downsampling is applied independently per each time series. It can reduce disk space usage and improve query performance if it is applied to time series with big number of samples per each series. The downsampling doesn't improve query performance if the database contains big number of time series with small number of samples per each series (aka [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate)), since downsampling doesn't reduce the number of time series. So the majority of time is spent on searching for the matching time series. It is possible to use recording rules in [vmalert](https://docs.victoriametrics.com/vmalert.html) in order to reduce the number of time series. See [these docs](https://docs.victoriametrics.com/vmalert.html#downsampling-and-aggregation-via-vmalert).
+
+The downsampling can be evaluated for free by downloading and using enterprise binaries from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).


 ## Multi-tenancy
@@ -1212,7 +1262,8 @@ Consider setting the following command-line flags:
 * `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
 * `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
 * `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
-* `-configAuthKey` for pretecting `/config` endpoint, since it may contain sensitive information such as passwords.
+* `-configAuthKey` for protecting `/config` endpoint, since it may contain sensitive information such as passwords.
+- `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).

 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
 For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`.
@@ -1248,15 +1299,16 @@ or Prometheus by adding the corresponding scrape config to it.
 Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0.
 For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval.

-There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
-There is also an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831).
+There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176). There is also an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831).
+
+Graphs on these dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.

 It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).

 The most interesting metrics are:

 * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
-  aka [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+  aka [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).
 * `increase(vm_new_timeseries_created_total[1h])` - time series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) during the previous hour.
 * `sum(vm_rows{type=~"storage/.*"})` - total number of `(timestamp, value)` data points in the database.
 * `sum(rate(vm_rows_inserted_total[5m]))` - ingestion rate, i.e. how many samples are inserted int the database per second.
@@ -1264,10 +1316,10 @@ The most interesting metrics are:
 * `sum(vm_data_size_bytes)` - the total size of data on disk.
 * `increase(vm_slow_row_inserts_total[5m])` - the number of slow inserts during the last 5 minutes.
  If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling
-  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).
 * `increase(vm_slow_metric_name_loads_total[5m])` - the number of slow loads of metric names during the last 5 minutes.
  If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling
-  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).

 VictoriaMetrics also exposes currently running queries with their execution times at `/api/v1/status/active_queries` page.

@@ -1287,7 +1339,7 @@ VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way simi

 By default VictoriaMetrics doesn't limit the number of stored time series. The limit can be enforced by setting the following command-line flags:

-* `-storage.maxHourlySeries` - limits the number of time series that can be added during the last hour. Useful for limiting the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+* `-storage.maxHourlySeries` - limits the number of time series that can be added during the last hour. Useful for limiting the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).
 * `-storage.maxDailySeries` - limits the number of time series that can be added during the last day. Useful for limiting daily [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).

 Both limits can be set simultaneously. If any of these limits is reached, then incoming samples for new time series are dropped. A sample of dropped series is put in the log with `WARNING` level.
@@ -1331,7 +1383,7 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
    See [this article for technical details](https://valyala.medium.com/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).

 * If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
-  then it is likely you have too many [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series) for the current amount of RAM.
+  then it is likely you have too many [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series) for the current amount of RAM.
  VictoriaMetrics [exposes](#monitoring) `vm_slow_*` metrics such as `vm_slow_row_inserts_total` and `vm_slow_metric_name_loads_total`, which could be used
  as an indicator of low amounts of RAM. It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
  ingestion and query performance in this case.
@@ -1366,9 +1418,7 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.

-* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then use `{__graphite__="foo.*.baz"}` syntax for selecting such metrics.
-  This expression is equivalent to `{__name__=~"foo[.][^.]*[.]baz"}`, but it works faster and it is easier to use when migrating from Graphite.
-  See also [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function, which allows extracting the given groups from Graphite metric names.
+* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `{__graphite__="foo.*.baz"}` filter can be used for selecting such metrics. See [these docs](#selecting-graphite-metrics) for details.

 * VictoriaMetrics ignores `NaN` values during data ingestion.

@@ -1378,6 +1428,31 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
 VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown (e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup. Sometimes it is needed to remove such caches on the next startup. This can be performed by placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.


+## Cache tuning
+
+VictoriaMetrics uses various in-memory caches for faster data ingestion and query performance.
+The following metrics for each type of cache are exported at [`/metrics` page](#monitoring):
+- `vm_cache_size_bytes` - the actual cache size
+- `vm_cache_size_max_bytes` - cache size limit
+- `vm_cache_requests_total` - the number of requests to the cache
+- `vm_cache_misses_total` - the number of cache misses
+- `vm_cache_entries` - the number of entries in the cache
+
+Both Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176)
+contain `Caches` section with cache metrics visualized. The panels show the current
+memory usage by each type of cache, and also a cache hit rate. If hit rate is close to 100%
+then cache efficiency is already very high and does not need any tuning.
+The panel `Cache usage %` in `Troubleshooting` section shows the percentage of used cache size
+from the allowed size by type. If the percentage is below 100%, then no further tuning needed.
+
+Please note, default cache sizes were carefully adjusted accordingly to the most
+practical scenarios and workloads. Change the defaults only if you understand the implications.
+
+To override the default values see command-line flags with `-storage.cacheSize` prefix.
+See the full description of flags [here](#list-of-command-line-flags).
+
+
 ## Data migration

 Use [vmctl](https://docs.victoriametrics.com/vmctl.html) for data migration. It supports the following data migration types:
@@ -1393,6 +1468,7 @@ See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details.
 ## Backfilling

 VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).
+See [how to backfill data with recording rules in vmalert](https://docs.victoriametrics.com/vmalert.html#rules-backfilling).
 Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.

 It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
@@ -1411,7 +1487,7 @@ cache when samples with timestamps older than `now - search.cacheTimestampOffset

 VictoriaMetrics doesn't support updating already existing sample values to new ones. It stores all the ingested data points
 for the same time series with identical timestamps. While it is possible substituting old time series with new time series via
-[removal of old time series](#how-to-delete-timeseries) and then [writing new time series](#backfilling), this approach
+[removal of old time series](#how-to-delete-time-series) and then [writing new time series](#backfilling), this approach
 should be used only for one-off updates. It shouldn't be used for frequent updates because of non-zero overhead related to data removal.


@@ -1433,22 +1509,42 @@ We also provide [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanag
 Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).


+## Benchmarks
+
+Note, that vendors (including VictoriaMetrics) are often biased when doing such tests. E.g. they try highlighting 
+the best parts of their product, while highlighting the worst parts of competing products. 
+So we encourage users and all independent third parties to conduct their becnhmarks for various products 
+they are evaluating in production and publish the results.
+
+As a reference, please see [benchmarks](https://docs.victoriametrics.com/Articles.html#benchmarks) conducted by
+VictoriaMetrics team. Please also see the [helm chart](https://github.com/VictoriaMetrics/benchmark) 
+for running ingestion benchmarks based on node_exporter metrics.
+
+
 ## Profiling

 VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):

-* Memory profile. It can be collected with the following command:
+* Memory profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<victoria-metrics-host>:8428/debug/pprof/heap > mem.pprof
+curl http://0.0.0.0:8428/debug/pprof/heap > mem.pprof
 ```

-* CPU profile. It can be collected with the following command:
+</div>
+
+* CPU profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<victoria-metrics-host>:8428/debug/pprof/profile > cpu.pprof
+curl http://0.0.0.0:8428/debug/pprof/profile > cpu.pprof
 ```

+</div>
+
 The command for collecting CPU profile waits for 30 seconds before returning.

 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
@@ -1486,9 +1582,11 @@ Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics
 Feel free asking any questions regarding VictoriaMetrics:

 * [slack](https://slack.victoriametrics.com/)
+* [linkedin](https://www.linkedin.com/company/victoriametrics/)
 * [reddit](https://www.reddit.com/r/VictoriaMetrics/)
 * [telegram-en](https://t.me/VictoriaMetrics_en)
 * [telegram-ru](https://t.me/VictoriaMetrics_ru1)
+* [articles and talks about VictoriaMetrics in Russian](https://github.com/denisgolius/victoriametrics-ru-links)
 * [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)

 If you like VictoriaMetrics and want to contribute, then we need the following:
@@ -1558,19 +1656,24 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
  -dedup.minScrapeInterval duration
-    	Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
+    	Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling
  -deleteAuthKey string
    	authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries
  -denyQueriesOutsideRetention
    	Whether to deny queries outside of the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
+  -downsampling.period array
+    	Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details
+    	Supports an array of values separated by comma or specified via multiple flags.
  -dryRun
-    	Whether to check only -promscrape.config and then exit. Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
+    	Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
  -enableTCP6
    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
  -envflag.enable
    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
  -envflag.prefix string
    	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
  -finalMergeDelay duration
    	The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge
  -forceFlushAuthKey string
@@ -1610,6 +1713,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -influx.maxLineSize size
    	The maximum size in bytes for a single InfluxDB line during parsing
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
+  -influxDBLabel string
+    	Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
  -influxListenAddr string
    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write
  -influxMeasurementFieldSeparator string
@@ -1643,8 +1748,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -maxInsertRequestSize size
    	The maximum size in bytes of a single Prometheus remote_write API request
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
+  -maxLabelValueLen int
+    	The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384)
  -maxLabelsPerTimeseries int
-    	The maximum number of labels accepted per time series. Superfluous labels are dropped (default 30)
+    	The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
  -memory.allowedBytes size
    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
@@ -1674,11 +1781,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -promscrape.cluster.replicationFactor int
    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
  -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
  -promscrape.config.dryRun
    	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
  -promscrape.config.strictParse
-    	Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped
+    	Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
  -promscrape.configCheckInterval duration
    	Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
  -promscrape.consul.waitTime duration
@@ -1708,7 +1815,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -promscrape.eurekaSDCheckInterval duration
    	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
  -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
  -promscrape.gceSDCheckInterval duration
    	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
  -promscrape.httpSDCheckInterval duration
@@ -1741,7 +1848,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -promscrape.suppressScrapeErrors
    	Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
  -relabelConfig string
-    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
+    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
  -relabelDebug
    	Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
  -retentionPeriod value
@@ -1753,6 +1860,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
    	Whether to disable automatic response cache reset if a sample with timestamp outside -search.cacheTimestampOffset is inserted into VictoriaMetrics
  -search.disableCache
    	Whether to disable response caching. This may be useful during data backfilling
+  -search.graphiteMaxPointsPerSeries int
+    	The maximum number of points per series Graphite render API can return (default 1000000)
+  -search.graphiteStorageStep duration
+    	The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overriden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API (default 10s)
  -search.latencyOffset duration
    	The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
  -search.logSlowQueryDuration duration
@@ -1814,6 +1925,15 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
    	authKey, which must be passed in query string to /snapshot* pages
  -sortLabels
    	Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
+  -storage.cacheSizeIndexDBDataBlocks size
+    	Overrides max size for indexdb/dataBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -storage.cacheSizeIndexDBIndexBlocks size
+    	Overrides max size for indexdb/indexBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -storage.cacheSizeStorageTSID size
+    	Overrides max size for storage/tsid cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
  -storage.maxDailySeries int
    	The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -storage.maxHourlySeries
  -storage.maxHourlySeries int
@@ -1826,9 +1946,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
  -tls
    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
  -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
  -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
  -version
    	Show VictoriaMetrics version
 ```
--- a/app/victoria-metrics/Makefile
+++ b/app/victoria-metrics/Makefile
@@ -27,6 +27,12 @@ victoria-metrics-ppc64le-prod:
 victoria-metrics-386-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386

+victoria-metrics-darwin-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
+
+victoria-metrics-darwin-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker

--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -25,9 +25,9 @@ import (
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
 )

 func main() {
@@ -51,7 +51,7 @@ func main() {

 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	storage.SetDedupInterval(*minScrapeInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -90,15 +90,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/vmui", "Web UI"},
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/config", "-promscrape.config contents"},
-			{"/metrics", "available service metrics"},
-			{"/flags", "command-line flags"},
-			{"/api/v1/status/tsdb", "tsdb status page"},
-			{"/api/v1/status/top_queries", "top queries"},
-			{"/api/v1/status/active_queries", "active queries"},
+			{"vmui", "Web UI"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"api/v1/status/tsdb", "tsdb status page"},
+			{"api/v1/status/top_queries", "top queries"},
+			{"api/v1/status/active_queries", "active queries"},
 		})
 		return true
 	}
--- a/app/vmagent/Makefile
+++ b/app/vmagent/Makefile
@@ -27,6 +27,15 @@ vmagent-ppc64le-prod:
 vmagent-386-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-386

+vmagent-darwin-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
+
+vmagent-darwin-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
+
+vmagent-windows-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
+
 package-vmagent:
 	APP_NAME=vmagent $(MAKE) package-via-docker

@@ -81,6 +90,3 @@ vmagent-pure:

 vmagent-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch
-
-vmagent-windows-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -12,7 +12,7 @@ or any other Prometheus-compatible storage systems that support the `remote_writ
 While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
 and RAM friendly to scrape metrics from Prometheus-compatible exporters into VictoriaMetrics.
 Also, we found that our user's infrastructure are like snowflakes in that no two are alike. Therefore we decided to add more flexibility
-to `vmagent` such as the ability to push metrics instead of pulling them. We did our best and will continue to improve vmagent.
+to `vmagent` such as the ability to push metrics additionally to pulling them. We did our best and will continue to improve `vmagent`.


 ## Features
@@ -46,7 +46,7 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and configure the following flags to the `vmagent` binary in order to start scraping Prometheus targets:

-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`)
+* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`). The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file, so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` additional command-line flag, so `vmagent` will ignore unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
 * `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified multiple times to replicate data concurrently to an arbitrary number of remote storage systems.

 Example command line:
@@ -214,15 +214,16 @@ The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders w

 ## Loading scrape configs from multiple files

-`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory plus a `single_scrape_config.yml` file:
+`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url:

 ```yml
 scrape_config_files:
 - configs/*.yml
 - single_scrape_config.yml
+- https://config-server/scrape_config.yml
 ```

-Every referred file can contain arbitrary number of any [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
+Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:

 ```yml
 - job_name: foo
@@ -236,6 +237,19 @@ Every referred file can contain arbitrary number of any [supported scrape config
 `vmagent` dynamically reloads these files on `SIGHUP` signal or on the request to `http://vmagent:8429/-/reload`.


+## Unsupported Prometheus config sections
+
+`vmagent` doesn't support the following sections in Prometheus config file passed to `-promscrape.config` command-line flag:
+
+* [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This section is substituted with various `-remoteWrite*` command-line flags. See [the full list of flags](#advanced-usage). The `remote_write` section isn't supported in order to reduce possible confusion when `vmagent` is used for accepting incoming metrics via push protocols such as InfluxDB, Graphite, OpenTSDB, DataDog, etc. In this case the `-promscrape.config` file isn't needed. See [these docs](#features) for details.
+* `remote_read`. This section isn't supported at all.
+* `rule_files` and `alerting`. These sections are supported by [vmalert](https://docs.victoriametrics.com/vmalert.html).
+
+The list of supported service discovery types is available [here](#how-to-collect-metrics-in-prometheus-format).
+
+Additionally `vmagent` doesn't support `refresh_interval` option at service discovery sections. This option is substituted with `-promscrape.*CheckInterval` command-line options, which are specific per each service discovery type. See [the full list of command-line flags for vmagent](#advanced-usage).
+
+
 ## Adding labels to metrics

 Labels can be added to metrics by the following mechanisms:
@@ -250,7 +264,7 @@ Labels can be added to metrics by the following mechanisms:

 ## Relabeling

-`vmagent` and VictoriaMetrics support Prometheus-compatible relabeling.
+VictoriaMetrics components (including `vmagent`) support Prometheus-compatible relabeling.
 They provide the following additional actions on top of actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config):

 * `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`.
@@ -275,11 +289,26 @@ The `regex` value can be split into multiple lines for improved readability and
  - "foo_.+"
 ```

+VictoriaMetrics components support an optional `if` filter, which can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops targets, which don't match `foo{bar="baz"}` series selector:
+
+```yaml
+- action: keep
+  if: 'foo{bar="baz"}'
+```
+
+This is equivalent to less clear traditional relabeling rule:
+
+```yaml
+- action: keep
+  source_labels: [__name__, bar]
+  regex: 'foo;baz'
+```
+
 The relabeling can be defined in the following places:

 * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
-* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.

 You can read more about relabeling in the following articles:
@@ -300,7 +329,6 @@ You can read more about relabeling in the following articles:
 * If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metric.
 * If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target.
 * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
-* Stale markers are sent for all the scraped metrics on graceful shutdown of `vmagent`.

 Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.

@@ -428,7 +456,7 @@ These limits are approximate, so `vmagent` can underflow/overflow the limit by a

 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. We recommend setting up regular scraping of this page
 either through `vmagent` itself or by Prometheus so that the exported metrics may be analyzed later.
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add a review to the dashboard.

 `vmagent` also exports the status for various targets at the following handlers:
@@ -520,7 +548,7 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.

 ## Kafka integration

-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read and write metrics from / to Kafka:
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read and write metrics from / to Kafka:

 * [Reading metrics from Kafka](#reading-metrics-from-kafka)
 * [Writing metrics to Kafka](#writing-metrics-to-kafka)
@@ -530,7 +558,7 @@ The enterprise version of vmagent is available for evaluation at [releases](http

 ### Reading metrics from Kafka

-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:

 * `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). Messages in this format can be sent by vmagent - see [these docs](#writing-metrics-to-kafka).
 * `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
@@ -566,7 +594,7 @@ data_format = "influx"

 #### Command-line flags for Kafka consumer

-These command-line flags are available only in [enterprise](https://victoriametrics.com/enterprise.html) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+These command-line flags are available only in [enterprise](https://victoriametrics.com/products/enterprise/) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.

 ```
  -kafka.consumer.topic array
@@ -599,7 +627,7 @@ These command-line flags are available only in [enterprise](https://victoriametr

 ### Writing metrics to Kafka

-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.

 Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).

@@ -628,7 +656,7 @@ We recommend using [binary releases](https://github.com/VictoriaMetrics/Victoria

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds the `vmagent` binary and puts it into the `bin` folder.

@@ -657,7 +685,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 ### Development ARM build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent-arm` or `make vmagent-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
   It builds `vmagent-arm` or `vmagent-arm64` binary respectively and puts it into the `bin` folder.

@@ -672,18 +700,26 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 `vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):

-* Memory profile can be collected with the following command:
+* Memory profile can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<vmagent-host>:8429/debug/pprof/heap > mem.pprof
+curl http://0.0.0.0:8429/debug/pprof/heap > mem.pprof
 ```

-* CPU profile can be collected with the following command:
+</div>
+
+* CPU profile can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<vmagent-host>:8429/debug/pprof/profile > cpu.pprof
+curl http://0.0.0.0:8429/debug/pprof/profile > cpu.pprof
 ```

+</div>
+
 The command for collecting CPU profile waits for 30 seconds before returning.

 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
@@ -708,13 +744,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
  -dryRun
-    	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
+    	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
  -enableTCP6
    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
  -envflag.enable
    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
  -envflag.prefix string
    	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
  -fs.disableMmap
    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
  -graphiteListenAddr string
@@ -748,6 +786,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
  -influx.maxLineSize size
    	The maximum size in bytes for a single InfluxDB line during parsing
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
+  -influxDBLabel string
+    	Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
  -influxListenAddr string
    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
  -influxMeasurementFieldSeparator string
@@ -760,6 +800,32 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
  -insert.maxQueueDuration duration
    	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
+  -kafka.consumer.topic array
+    	Kafka topic names for data consumption.
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.password array
+    	Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.username array
+    	Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.brokers array
+    	List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.defaultFormat string
+    	Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+  -kafka.consumer.topic.format array
+    	data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.groupID array
+    	Defines group.id for topic
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.isGzipped array
+    	Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+    	Supports array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.options array
+    	Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+    	Supports an array of values separated by comma or specified via multiple flags.
  -loggerDisableTimestamps
    	Whether to disable writing timestamps in logs
  -loggerErrorsPerSecondLimit int
@@ -806,11 +872,11 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
  -promscrape.cluster.replicationFactor int
    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
  -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
  -promscrape.config.dryRun
    	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
  -promscrape.config.strictParse
-    	Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped
+    	Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
  -promscrape.configCheckInterval duration
    	Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
  -promscrape.consul.waitTime duration
@@ -840,7 +906,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
  -promscrape.eurekaSDCheckInterval duration
    	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
  -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
+    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
  -promscrape.gceSDCheckInterval duration
    	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
  -promscrape.httpSDCheckInterval duration
@@ -931,7 +997,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
    	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
    	Supports array of values separated by comma or specified via multiple flags.
  -remoteWrite.relabelConfig string
-    	Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+    	Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
  -remoteWrite.relabelDebug
    	Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
  -remoteWrite.roundDigits array
@@ -966,7 +1032,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
    	Supports an array of values separated by comma or specified via multiple flags.
  -remoteWrite.urlRelabelConfig array
-    	Optional path to relabel config for the corresponding -remoteWrite.url
+    	Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
    	Supports an array of values separated by comma or specified via multiple flags.
  -remoteWrite.urlRelabelDebug array
    	Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
@@ -976,9 +1042,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
  -tls
    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
  -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
  -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
  -version
    	Show VictoriaMetrics version
 ```
--- a/app/vmagent/influx/request_handler.go
+++ b/app/vmagent/influx/request_handler.go
@@ -24,6 +24,7 @@ var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
 	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
+	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )

 var (
@@ -80,7 +81,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			if tag.Key == "db" {
+			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
 			commonLabels = append(commonLabels, prompbmarshal.Label{
@@ -90,7 +91,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		}
 		if len(db) > 0 && !hasDBKey {
 			commonLabels = append(commonLabels, prompbmarshal.Label{
-				Name:  "db",
+				Name:  *dbLabel,
 				Value: db,
 			})
 		}
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -53,7 +53,7 @@ var (
 	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )

 var (
@@ -158,12 +158,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/config", "-promscrape.config contents"},
-			{"/metrics", "available service metrics"},
-			{"/flags", "command-line flags"},
-			{"/-/reload", "reload configuration"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}
@@ -236,32 +236,40 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "/datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/target_response":
+		promscrapeTargetResponseRequests.Inc()
+		if err := promscrape.WriteTargetResponse(w, r); err != nil {
+			promscrapeTargetResponseErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
 	case "/config":
 		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
 			err := &httpserver.ErrorWithStatusCode{
@@ -277,7 +285,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
@@ -391,19 +399,19 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	default:
@@ -443,6 +451,9 @@ var (
 	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)

+	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
+	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
+
 	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)

 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -254,7 +254,7 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -295,6 +295,17 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
+		body, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+		if err != nil {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
+		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -15,12 +15,14 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
-	relabelDebug       = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
 )
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -390,6 +390,8 @@ var labelsHashBufPool bytesutil.ByteBufferPool
 func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
+		// Do not use logger.WithThrottler() here, since this will increase CPU usage
+		// because every call to logSkippedSeries will result to a call to labelsToString.
 		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
 	default:
 	}
--- a/app/vmalert/Makefile
+++ b/app/vmalert/Makefile
@@ -27,6 +27,15 @@ vmalert-ppc64le-prod:
 vmalert-386-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-386

+vmalert-darwin-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
+
+vmalert-darwin-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
+
+vmalert-windows-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
+
 package-vmalert:
 	APP_NAME=vmalert $(MAKE) package-via-docker

@@ -70,6 +79,13 @@ run-vmalert: vmalert
 		-evaluationInterval=3s \
 		-rule.configCheckInterval=10s

+run-vmalert-sd: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-configCheckInterval=10s
+
 replay-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
@@ -102,6 +118,3 @@ vmalert-pure:

 vmalert-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch
-
-vmalert-windows-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -2,7 +2,11 @@

 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured address. It is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
+rules against configured `-datasource.url`. For sending alerting notifications
+vmalert relies on [Alertmanager]((https://github.com/prometheus/alertmanager)) configured via `-notifier.url` flag.
+Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+protocol and require `-remoteWrite.url` to be configured.
+Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.

 ## Features
@@ -11,19 +15,19 @@ implementation and aims to be compatible with its syntax.
 support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
 support;
-* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-aplha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
 * Keeps the alerts [state on restarts](#alerts-state-on-restarts);
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight without extra dependencies.

 ## Limitations
-* `vmalert` execute queries against remote datasource which has reliability risks because of network.
-It is recommended to configure alerts thresholds and rules expressions with understanding that network request
-may fail;
+* `vmalert` execute queries against remote datasource which has reliability risks because of the network.
+It is recommended to configure alerts thresholds and rules expressions with the understanding that network 
+requests may fail;
 * by default, rules execution is sequential within one group, but persistence of execution results to remote
 storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
-recording rule is reused in next one;
+recording rule is reused in the next one;

 ## QuickStart

@@ -33,28 +37,36 @@ git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
 ```
-The build binary will be placed to `VictoriaMetrics/bin` folder.
+The build binary will be placed in `VictoriaMetrics/bin` folder.

 To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable VictoriaMetrics instance for rules execution;
-* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts and sending notifications.
+* datasource address - reachable MetricsQL endpoint to run queries against;
+* notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
+aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via 
+[config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-compatible storage address for storing recording rules results and alerts state in for of timeseries.
+  compatible storage to persist rules and alerts state info;
+* remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.

 Then configure `vmalert` accordingly:
 ```
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
    -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
-    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
    -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
-    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules
+    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
    -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
    -external.label=cluster=east-1 \         # External label to be applied for each rule
    -external.label=replica=a                # Multiple external labels may be set
 ```

+Note there's a separate `remoteRead.url` to allow writing results of
+alerting/recording rules into a different storage than the initial data that's
+queried. This allows using `vmalert` to aggregate data from a short-term,
+high-frequency, high-cardinality storage into a long-term storage with
+decreased cardinality and a bigger interval between samples.
+
 See the full list of configuration flags in [configuration](#configuration) section.

 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
@@ -88,12 +100,24 @@ name: <string>
 # By default "prometheus" type is used.
 [ type: <string> ]

-# Optional list of label filters applied to every rule's
-# request withing a group. Is compatible only with VM datasource.
-# See more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+# Warning: DEPRECATED
+# Please use `params` instead:
+#   params:
+#     extra_label: ["job=nodeexporter", "env=prod"]
 extra_filter_labels:
  [ <labelname>: <labelvalue> ... ]

+# Optional list of HTTP URL parameters
+# applied for all rules requests within a group
+# For example:
+#  params:
+#    nocache: ["1"]                # disable caching for vmselect
+#    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+# see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+params:
+  [ <string>: [<string>, ...]]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
@@ -113,14 +137,14 @@ expression and then act according to the Rule type.

 There are two types of Rules:
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allow to define alert conditions via `expr` field and to send notifications to
+Alerting rules allow defining alert conditions via `expr` field and to send notifications to
 [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allow to define `expr` which result will be then backfilled to configured
+Recording rules allow defining `expr` which result will be then backfilled to configured
 `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
 expensive expressions and save their result as a new set of time series.

-`vmalert` forbids defining duplicates - rules with the same combination of name, expression and labels
+`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.

 #### Alerting rules
@@ -136,7 +160,7 @@ alert: <string>
 expr: <string>

 # Alerts are considered firing once they have been returned for this long.
-# Alerts which have not yet fired for long enough are considered pending.
+# Alerts which have not yet been fired for long enough are considered pending.
 # If param is omitted or set to 0 then alerts will be immediately considered
 # as firing once they return.
 [ for: <duration> | default = 0s ]
@@ -181,19 +205,19 @@ For recording rules to work `-remoteWrite.url` must be specified.
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
 into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and may be queried from VM just as any other time series.
+These are regular time series and maybe queried from VM just as any other time series.
 The state is stored to the configured address on every rule evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
 from configured address by querying time series with name `ALERTS_FOR_STATE`.

-Both flags are required for proper state restoring. Restore process may fail if time series are missing
+Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
 or received state doesn't match current `vmalert` rules configuration.


 ### Multitenancy

-The following are the approaches for alerting and recording rules across
+There are the following approaches exist for alerting and recording rules across
 [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy):

 * To run a separate `vmalert` instance per each tenant.
@@ -206,7 +230,7 @@ The following are the approaches for alerting and recording rules across
  rules to `AccountID=123`.

 * To specify `tenant` parameter per each alerting and recording group if
-  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used
+  [enterprise version of vmalert](https://victoriametrics.com/products/enterprise/) is used
  with `-clusterMode` command-line flag. For example:

 ```yaml
@@ -233,6 +257,112 @@ The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz`
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).

+### Topology examples
+
+The following sections are showing how `vmalert` may be used and configured 
+for different scenarios. 
+
+Please note, not all flags in examples are required: 
+* `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
+you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+* `-notifier.url` is optional and is needed only if you have alerting rules.
+
+#### Single-node VictoriaMetrics
+
+The simplest configuration where one single-node VM server is used for
+rules execution, storing recording rules results and alerts state.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                    # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093          # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert single" width="500" src="vmalert_single.png">
+
+
+#### Cluster VictoriaMetrics
+
+In [cluster mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html)
+VictoriaMetrics has separate components for writing and reading path:
+`vminsert` and `vmselect` components respectively. `vmselect` is used for executing rules expressions
+and `vminsert` is used to persist recording rules results and alerts state.
+Cluster mode could have multiple `vminsert` and `vmselect` components. 
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                                # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for executing rules expressions
+    -remoteWrite.url=http://vminsert:8480/insert/0/prometheuss  # vminsert addr to persist alerts state and recording rules results
+    -remoteRead.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093                      # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert cluster" src="vmalert_cluster.png">
+
+In case when you want to spread the load on these components - add balancers before them and configure
+`vmalert` with balancer's addresses. Please, see more about VM's cluster architecture
+[here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview).
+
+#### HA vmalert
+
+For HA user can run multiple identically configured `vmalert` instances.
+It means all of them will execute the same rules, write state and results to
+the same destinations, and send alert notifications to multiple configured
+Alertmanagers.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml \                   # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager1:9093 \       # Multiple AlertManager addresses to send alerts when they trigger
+    -notifier.url=http://alertmanagerN:9093         # The same alert will be sent to all configured notifiers
+```
+
+<img alt="vmalert ha" width="800px" src="vmalert_ha.png">
+
+To avoid recording rules results and alerts state duplication in VictoriaMetrics server
+don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+
+Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
+all `vmalert`s are having the same config.
+
+Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
+for Alertmanagers for better reliability.
+
+This example uses single-node VM server for the sake of simplicity. 
+Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. 
+
+
+#### Downsampling and aggregation via vmalert
+
+The following example shows how to build a topology where `vmalert` will process data from one cluster
+and write results into another. Such clusters may be called as "hot" (low retention,
+high-speed disks, used for operative monitoring) and "cold" (long term retention,
+slower/cheaper disks, low resolution data). With help of `vmalert`, user can setup
+recording rules to process raw data from "hot" cluster (by applying additional transformations
+or reducing resolution) and push results to "cold" cluster.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheuss   # vminsert addr to persist recording rules results
+```
+
+<img alt="vmalert multi cluster" src="vmalert_multicluster.png">
+
+Please note, [replay](#rules-backfilling) feature may be used for transforming historical data.
+
+Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only recording rules are used.
+
+See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
+

 ### Web

@@ -252,7 +382,7 @@ vmalert sends requests to `<-datasource.url>/render?format=json` during evaluati
 if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
 implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
 When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
-to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on query type.
+to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.

 ## Rules backfilling

@@ -311,11 +441,11 @@ to prevent cache pollution and unwanted time range boundaries adjustment during

 #### Recording rules

-Result of recording rules `replay` should match with results of normal rules evaluation.
+The result of recording rules `replay` should match with results of normal rules evaluation.

 #### Alerting rules

-Result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
+The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
 To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
 ```
 ALERTS{alertname="your_alertname", alertstate="firing"}
@@ -328,7 +458,7 @@ There are following non-required `replay` flags:

 * `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
 In two words, it affects the max time range for every `/query_range` request. The higher the value,
-the less requests will be issued during `replay`.
+the fewer requests will be issued during `replay`.
 * `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
 per rule before giving up.
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
@@ -350,18 +480,24 @@ See full description for these flags in `./vmalert --help`.
 We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
 metrics may be analyzed later.

-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview.
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.


 ## Configuration

+### Flags
+
 Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.

 The shortlist of configuration flags is the following:
 ```
+  -clusterMode
+    	If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+  -configCheckInterval duration
+    	Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
  -datasource.appendTypePrefix
    	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
  -datasource.basicAuth.password string
@@ -394,8 +530,12 @@ The shortlist of configuration flags is the following:
    	Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
  -datasource.url string
    	VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+  -defaultTenant.graphite string
+    	Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+  -defaultTenant.prometheus string
+    	Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
  -disableAlertgroupLabel
-    	Whether to disable adding group's name as label to generated alerts and time series.
+    	Whether to disable adding group's Name as label to generated alerts and time series.
  -dryRun -rule
    	Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
  -enableTCP6
@@ -404,13 +544,15 @@ The shortlist of configuration flags is the following:
    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
  -envflag.prefix string
    	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
  -evaluationInterval duration
    	How often to evaluate the rules (default 1m0s)
  -external.alert.source string
    	External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
    	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
  -external.label array
-    	Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
+    	Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
    	Supports an array of values separated by comma or specified via multiple flags.
  -external.url string
    	External URL is used as alert's source for sent alerts to the notifier
@@ -454,13 +596,20 @@ The shortlist of configuration flags is the following:
  -memory.allowedPercent float
    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
  -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -notifier.basicAuth.password array
    	Optional basic auth password for -notifier.url
    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.basicAuth.passwordFile array
+    	Optional path to basic auth password file for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
  -notifier.basicAuth.username array
    	Optional basic auth username for -notifier.url
    	Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.config string
+    	Path to configuration file for notifiers
+  -notifier.suppressDuplicateTargetErrors
+    	Whether to suppress 'duplicate target' errors during discovery
  -notifier.tlsCAFile array
    	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
    	Supports an array of values separated by comma or specified via multiple flags.
@@ -477,10 +626,18 @@ The shortlist of configuration flags is the following:
    	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
    	Supports an array of values separated by comma or specified via multiple flags.
  -notifier.url array
-    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+    	Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
    	Supports an array of values separated by comma or specified via multiple flags.
  -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -promscrape.consul.waitTime duration
+    	Wait time used by Consul service discovery. Default value is used if not set
+  -promscrape.consulSDCheckInterval duration
+    	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+  -promscrape.discovery.concurrency int
+    	The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
+  -promscrape.discovery.concurrentWaitTime duration
+    	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
  -remoteRead.basicAuth.password string
    	Optional basic auth password for -remoteRead.url
  -remoteRead.basicAuth.passwordFile string
@@ -561,7 +718,7 @@ The shortlist of configuration flags is the following:
    	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
    	Supports an array of values separated by comma or specified via multiple flags.
  -rule.configCheckInterval duration
-    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
  -rule.maxResolveDuration duration
    	Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
  -rule.validateExpressions
@@ -571,19 +728,121 @@ The shortlist of configuration flags is the following:
  -tls
    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
  -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
  -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
  -version
    	Show VictoriaMetrics version
 ```

+### Hot config reload
 `vmalert` supports "hot" config reload via the following methods:
 * send SIGHUP signal to `vmalert` process;
 * send GET request to `/-/reload` endpoint;
-* configure `-rule.configCheckInterval` flag for periodic reload
+* configure `-configCheckInterval` flag for periodic reload
 on config change.

+### URL params
+
+To set additional URL params for `datasource.url`, `remoteWrite.url` or `remoteRead.url`
+just add them in address: `-datasource.url=http://localhost:8428?nocache=1`.
+
+To set additional URL params for specific [group of rules](#Groups) modify
+the `params` group:
+```yaml
+groups:
+  - name: TestGroup
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
+```
+Please note, `params` are used only for executing rules expressions (requests to `datasource.url`).
+If there would be a conflict between URL params set in `datasource.url` flag and params in group definition
+the latter will have higher priority.
+
+### Notifier configuration file
+
+Notifier also supports configuration via file specified with flag `notifier.config`:
+```
+./bin/vmalert -rule=app/vmalert/config/testdata/rules.good.rules \
+		-datasource.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
+```
+
+The configuration file allows to configure static notifiers or discover notifiers via 
+[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config).
+For example:
+```
+static_configs: 
+  - targets:
+      - localhost:9093
+      - localhost:9095
+
+consul_sd_configs:
+  - server: localhost:8500
+    services:
+      - alertmanager
+```
+
+The list of configured or discovered Notifiers can be explored via [UI](#Web).
+
+The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go)
+is the following:
+```
+# Per-target Notifier timeout when pushing alerts.
+[ timeout: <duration> | default = 10s ]
+
+# Prefix for the HTTP path alerts are pushed to.
+[ path_prefix: <path> | default = / ]
+
+# Configures the protocol scheme used for requests.
+[ scheme: <scheme> | default = http ]
+
+# Sets the `Authorization` header on every request with the
+# configured username and password.
+# password and password_file are mutually exclusive.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# Optional `Authorization` header configuration.
+authorization:
+  # Sets the authentication type.
+  [ type: <string> | default: Bearer ]
+  # Sets the credentials. It is mutually exclusive with
+  # `credentials_file`.
+  [ credentials: <secret> ]
+  # Sets the credentials to the credentials read from the configured file.
+  # It is mutually exclusive with `credentials`.
+  [ credentials_file: <filename> ]
+
+# Configures the scrape request's TLS settings.
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
+tls_config:
+  [ <tls_config> ]
+
+# List of labeled statically configured Notifiers.
+static_configs:
+  targets:
+    [ - '<host>' ]
+
+# List of Consul service discovery configurations.
+# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+consul_sd_configs:
+  [ - <consul_sd_config> ... ]
+
+# List of relabel configurations.
+# Supports the same relabeling features as the rest of VictoriaMetrics components.
+# See https://docs.victoriametrics.com/vmagent.html#relabeling
+relabel_configs:
+  [ - <relabel_config> ... ]
+
+```
+
+The configuration file can be [hot-reloaded](#hot-config-reload).
+
+
 ## Contributing

 `vmalert` is mostly designed and built by VictoriaMetrics community.
@@ -599,7 +858,7 @@ It is recommended using

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmalert` binary and puts it into the `bin` folder.

@@ -616,7 +875,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 ### Development ARM build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.

--- a/app/vmalert/alerting.go
+++ b/app/vmalert/alerting.go
@@ -12,9 +12,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )

 // AlertingRule is basic alert entity
@@ -50,10 +50,10 @@ type AlertingRule struct {
 }

 type alertingRuleMetrics struct {
-	errors  *gauge
-	pending *gauge
-	active  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	pending *utils.Gauge
+	active  *utils.Gauge
+	samples *utils.Gauge
 }

 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
@@ -71,14 +71,14 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
 		metrics: &alertingRuleMetrics{},
 	}

 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
-	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -90,7 +90,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -102,7 +102,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -111,7 +111,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return 1
 		})
-	ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -122,10 +122,10 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule

 // Close unregisters rule metrics
 func (ar *AlertingRule) Close() {
-	metrics.UnregisterMetric(ar.metrics.active.name)
-	metrics.UnregisterMetric(ar.metrics.pending.name)
-	metrics.UnregisterMetric(ar.metrics.errors.name)
-	metrics.UnregisterMetric(ar.metrics.samples.name)
+	ar.metrics.active.Unregister()
+	ar.metrics.pending.Unregister()
+	ar.metrics.errors.Unregister()
+	ar.metrics.samples.Unregister()
 }

 // String implements Stringer interface
@@ -153,6 +153,13 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
+		// set additional labels to identify group and rule Name
+		if ar.Name != "" {
+			s.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			s.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(s, qFn, ar)
 		if err != nil {
@@ -163,13 +170,6 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			// so the hash key will be consistent on restore
 			s.SetLabel(k, v)
 		}
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			s.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			s.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
 		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
@@ -225,6 +225,13 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			m.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			m.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(m, qFn, ar)
 		if err != nil {
@@ -235,14 +242,6 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 			// so the hash key will be consistent on restore
 			m.SetLabel(k, v)
 		}
-		// set additional labels to identify group and rule name
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			m.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			m.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
 		h := hash(m)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
--- a/app/vmalert/alerting_test.go
+++ b/app/vmalert/alerting_test.go
@@ -715,6 +715,44 @@ func TestAlertingRule_Template(t *testing.T) {
 				},
 			},
 		},
+		{
+			&AlertingRule{
+				Name:      "ExtraTemplating",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"name":     "alert_{{ $labels.alertname }}",
+					"group":    "group_{{ $labels.alertgroup }}",
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
+					"name", "alert_ExtraTemplating",
+					alertGroupNameLabel, "Testing",
+					"group", "group_Testing",
+					"instance", "foo")): {
+					Labels: map[string]string{
+						alertNameLabel:      "ExtraTemplating",
+						"name":              "alert_ExtraTemplating",
+						alertGroupNameLabel: "Testing",
+						"group":             "group_Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
+						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+					},
+				},
+			},
+		},
 	}
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {
--- a/app/vmalert/config/config.go
+++ b/app/vmalert/config/config.go
@@ -5,17 +5,19 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
+	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"

-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"gopkg.in/yaml.v2"

+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )

 // Group contains list of Rules grouped into
@@ -24,12 +26,13 @@ type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
 	Name        string             `yaml:"name"`
-	Interval    utils.PromDuration `yaml:"interval"`
+	Interval    promutils.Duration `yaml:"interval"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+	// DEPRECATED: use Params field instead
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
@@ -37,6 +40,8 @@ type Group struct {
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`

 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -57,6 +62,22 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		g.Type.Set(datasource.NewPrometheusType())
 	}

+	// backward compatibility with deprecated `ExtraFilterLabels` param
+	if len(g.ExtraFilterLabels) > 0 {
+		if g.Params == nil {
+			g.Params = url.Values{}
+		}
+		// Sort extraFilters for consistent order for query args across runs.
+		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
+		for k, v := range g.ExtraFilterLabels {
+			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(extraFilters)
+		for _, extraFilter := range extraFilters {
+			g.Params.Add("extra_label", extraFilter)
+		}
+	}
+
 	h := md5.New()
 	h.Write(b)
 	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
@@ -109,7 +130,7 @@ type Rule struct {
 	Record      string             `yaml:"record,omitempty"`
 	Alert       string             `yaml:"alert,omitempty"`
 	Expr        string             `yaml:"expr"`
-	For         utils.PromDuration `yaml:"for"`
+	For         promutils.Duration `yaml:"for"`
 	Labels      map[string]string  `yaml:"labels,omitempty"`
 	Annotations map[string]string  `yaml:"annotations,omitempty"`

@@ -178,6 +199,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
+	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -197,6 +219,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
+			if len(g.ExtraFilterLabels) > 0 {
+				isExtraFilterLabelsUsed = true
+			}
 			groups = append(groups, g)
 		}
 	}
@@ -206,6 +231,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
+	if isExtraFilterLabelsUsed {
+		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
+	}
 	return groups, nil
 }

--- a/app/vmalert/config/config_test.go
+++ b/app/vmalert/config/config_test.go
@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"

+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )

 func TestMain(m *testing.M) {
@@ -259,7 +260,7 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sum(up == 0 ) by (host)",
@@ -274,7 +275,7 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
@@ -341,7 +342,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: utils.NewPromDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -472,6 +473,85 @@ concurrency: 16
 rules:
  - alert: ExampleAlertWithFor
    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
 `)
 	})
 }
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+
+	t.Run("extra labels", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+  env: prod
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+
+	t.Run("extra labels and params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+params:
+  nocache: ["1"]
+  extra_label: ["env=prod"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+}
--- a/app/vmalert/config/testdata/rules0-good.rules
+++ b/app/vmalert/config/testdata/rules0-good.rules
@@ -1,5 +1,8 @@
 groups:
  - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
    rules:
      - alert: VMRows
        for: 10s
--- a/app/vmalert/config/testdata/rules2-good.rules
+++ b/app/vmalert/config/testdata/rules2-good.rules
@@ -2,8 +2,11 @@ groups:
  - name: TestGroup
    interval: 2s
    concurrency: 2
-    extra_filter_labels:
+    extra_filter_labels: # deprecated param, use `params` instead
        job: victoriametrics
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
    rules:
      - alert: Conns
        expr: sum(vm_tcplistener_conns) by(instance) > 1
@@ -22,6 +25,7 @@ groups:
          dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}'
        annotations:
          description: Job {{ $labels.job }} is up!
+          external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }}
          summary: All instances up {{ range query "up" }}
            {{ . | label "instance" }}
            {{ end }}
--- a/app/vmalert/datasource/datasource.go
+++ b/app/vmalert/datasource/datasource.go
@@ -2,6 +2,7 @@ package datasource

 import (
 	"context"
+	"net/url"
 	"time"
 )

@@ -20,8 +21,7 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     *Type
 	EvaluationInterval time.Duration
-	// see https://docs.victoriametrics.com/#prometheus-querying-api-enhancements
-	ExtraLabels map[string]string
+	QueryParams        url.Values
 }

 // Metric is the basic entity which should be return by datasource
--- a/app/vmalert/datasource/init.go
+++ b/app/vmalert/datasource/init.go
@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -40,9 +41,9 @@ type Param struct {
 }

 // Init creates a Querier from provided flag values.
-// Provided extraParams will be added as GET params to
+// Provided extraParams will be added as GET params for
 // each request.
-func Init(extraParams []Param) (QuerierBuilder, error) {
+func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
@@ -56,11 +57,11 @@ func Init(extraParams []Param) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}

+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
 	if *roundDigits > 0 {
-		extraParams = append(extraParams, Param{
-			Key:   "round_digits",
-			Value: fmt.Sprintf("%d", *roundDigits),
-		})
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}

 	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
--- a/app/vmalert/datasource/vm.go
+++ b/app/vmalert/datasource/vm.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"

@@ -22,8 +23,7 @@ type VMStorage struct {

 	dataSourceType     Type
 	evaluationInterval time.Duration
-	extraLabels        []string
-	extraParams        []Param
+	extraParams        url.Values
 	disablePathAppend  bool
 }

@@ -47,9 +47,7 @@ func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 		s.dataSourceType = *params.DataSourceType
 	}
 	s.evaluationInterval = params.EvaluationInterval
-	for k, v := range params.ExtraLabels {
-		s.extraLabels = append(s.extraLabels, fmt.Sprintf("%s=%s", k, v))
-	}
+	s.extraParams = params.QueryParams
 	return s
 }

@@ -150,7 +148,7 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
 		if auth := s.authCfg.GetAuthHeader(); auth != "" {
 			req.Header.Set("Authorization", auth)
--- a/app/vmalert/datasource/vm_graphite_api.go
+++ b/app/vmalert/datasource/vm_graphite_api.go
@@ -54,6 +54,14 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("format", "json")
 	q.Set("target", query)
 	from := "-5min"
--- a/app/vmalert/datasource/vm_prom_api.go
+++ b/app/vmalert/datasource/vm_prom_api.go
@@ -150,20 +150,24 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s

 func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("query", query)
-	if s.evaluationInterval > 0 {
-		// set step as evaluationInterval by default
-		q.Set("step", s.evaluationInterval.String())
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if s.queryStep > 0 {
-		// override step with user-specified value
-		q.Set("step", s.queryStep.String())
-	}
-	for _, l := range s.extraLabels {
-		q.Add("extra_label", l)
-	}
-	for _, p := range s.extraParams {
-		q.Add(p.Key, p.Value)
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
 }
--- a/app/vmalert/datasource/vm_test.go
+++ b/app/vmalert/datasource/vm_test.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
@@ -435,15 +436,28 @@ func TestRequestParams(t *testing.T) {
 				queryStep: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, time.Minute, timestamp.Unix())
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"round digits",
+			"step to seconds",
 			false,
 			&VMStorage{
-				extraParams: []Param{{"round_digits", "10"}},
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
 			},
 			func(t *testing.T, r *http.Request) {
 				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
@@ -451,45 +465,32 @@ func TestRequestParams(t *testing.T) {
 			},
 		},
 		{
-			"extra labels",
-			false,
-			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-		{
-			"extra labels range",
+			"prometheus extra params range",
 			true,
 			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&start=%d",
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
 					timestamp.Unix(), query, timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"extra params",
+			"graphite extra params",
 			false,
 			&VMStorage{
-				extraParams: []Param{
-					{Key: "nocache", Value: "1"},
-					{Key: "max_lookback", Value: "1h"},
+				dataSourceType: NewGraphiteType(),
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("max_lookback=1h&nocache=1&query=%s&time=%d", query, timestamp.Unix())
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
@@ -519,7 +520,7 @@ func TestRequestParams(t *testing.T) {
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {
-		t.Errorf("expected to get %q; got %q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }

--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"net/url"
 	"sync"
 	"time"

@@ -27,8 +28,8 @@ type Group struct {
 	Concurrency int
 	Checksum    string

-	ExtraFilterLabels map[string]string
-	Labels            map[string]string
+	Labels map[string]string
+	Params url.Values

 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -40,15 +41,15 @@ type Group struct {
 }

 type groupMetrics struct {
-	iterationTotal    *counter
-	iterationDuration *summary
+	iterationTotal    *utils.Counter
+	iterationDuration *utils.Summary
 }

 func newGroupMetrics(name, file string) *groupMetrics {
 	m := &groupMetrics{}
 	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
-	m.iterationTotal = getOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = getOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	return m
 }

@@ -71,14 +72,14 @@ func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[s

 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
-		Type:              cfg.Type,
-		Name:              cfg.Name,
-		File:              cfg.File,
-		Interval:          cfg.Interval.Duration(),
-		Concurrency:       cfg.Concurrency,
-		Checksum:          cfg.Checksum,
-		ExtraFilterLabels: cfg.ExtraFilterLabels,
-		Labels:            cfg.Labels,
+		Type:        cfg.Type,
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval.Duration(),
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		Params:      cfg.Params,
+		Labels:      cfg.Labels,

 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -121,7 +122,7 @@ func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
 }

 // ID return unique group ID that consists of
-// rules file and group name
+// rules file and group Name
 func (g *Group) ID() uint64 {
 	g.mu.RLock()
 	defer g.mu.RUnlock()
@@ -198,7 +199,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
-	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Params = newGroup.Params
 	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
@@ -212,8 +213,8 @@ func (g *Group) close() {
 	close(g.doneCh)
 	<-g.finishedCh

-	metrics.UnregisterMetric(g.metrics.iterationDuration.name)
-	metrics.UnregisterMetric(g.metrics.iterationTotal.name)
+	g.metrics.iterationDuration.Unregister()
+	g.metrics.iterationTotal.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -221,7 +222,7 @@ func (g *Group) close() {

 var skipRandSleepOnGroupStart bool

-func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()

 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
@@ -245,16 +246,7 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 	}

 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{rw: rw}
-	for _, nt := range nts {
-		ent := eNotifier{
-			Notifier:         nt,
-			alertsSent:       getOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", nt.Addr())),
-			alertsSendErrors: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", nt.Addr())),
-		}
-		e.notifiers = append(e.notifiers, ent)
-	}
-
+	e := &executor{rw: rw, notifiers: nts}
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -309,16 +301,10 @@ func getResolveDuration(groupInterval time.Duration) time.Duration {
 }

 type executor struct {
-	notifiers []eNotifier
+	notifiers func() []notifier.Notifier
 	rw        *remotewrite.Client
 }

-type eNotifier struct {
-	notifier.Notifier
-	alertsSent       *counter
-	alertsSendErrors *counter
-}
-
 func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
@@ -355,6 +341,7 @@ var (
 	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)

 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )

 func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
@@ -368,6 +355,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur

 	if len(tss) > 0 && e.rw != nil {
 		for _, ts := range tss {
+			remoteWriteTotal.Inc()
 			if err := e.rw.Push(ts); err != nil {
 				remoteWriteErrors.Inc()
 				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
@@ -397,11 +385,9 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur
 	}

 	errGr := new(utils.ErrGroup)
-	for _, nt := range e.notifiers {
-		nt.alertsSent.Add(len(alerts))
+	for _, nt := range e.notifiers() {
 		if err := nt.Send(ctx, alerts); err != nil {
-			nt.alertsSendErrors.Inc()
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
+			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
 		}
 	}
 	return errGr.Err()
--- a/app/vmalert/group_test.go
+++ b/app/vmalert/group_test.go
@@ -9,7 +9,7 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )

 func init() {
@@ -34,7 +34,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -46,7 +46,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -212,7 +212,7 @@ func TestGroupStart(t *testing.T) {
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), []notifier.Notifier{fn}, nil)
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil)
 		close(finished)
 	}()

--- a/app/vmalert/helpers_test.go
+++ b/app/vmalert/helpers_test.go
@@ -63,6 +63,7 @@ type fakeNotifier struct {
 	alerts []notifier.Alert
 }

+func (*fakeNotifier) Close()       {}
 func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -35,7 +35,10 @@ absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)

 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
-		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes")
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.")

 	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
@@ -47,14 +50,14 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
-	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+
+	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")

 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")

-	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's name as label to generated alerts and time series.")
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")

 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
@@ -97,6 +100,9 @@ func main() {
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
 		}
+		if rw == nil {
+			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
+		}
 		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
@@ -104,8 +110,7 @@ func main() {
 		}
 		// prevent queries from caching and boundaries aligning
 		// when querying VictoriaMetrics datasource.
-		noCache := datasource.Param{Key: "nocache", Value: "1"}
-		q, err := datasource.Init([]datasource.Param{noCache})
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
 		if err != nil {
 			logger.Fatalf("failed to init datasource: %s", err)
 		}
@@ -162,7 +167,20 @@ func newManager(ctx context.Context) (*manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	nts, err := notifier.Init(alertURLGeneratorFn)
+
+	labels := make(map[string]string, 0)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -170,7 +188,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		groups:         make(map[uint64]*Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         map[string]string{},
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -184,16 +202,6 @@ func newManager(ctx context.Context) (*manager, error) {
 	}
 	manager.rr = rr

-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		manager.labels[s[:n]] = s[n+1:]
-	}
 	return manager, nil
 }

@@ -252,8 +260,13 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .

 func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
 	var configCheckCh <-chan time.Time
-	if *rulesCheckInterval > 0 {
-		ticker := time.NewTicker(*rulesCheckInterval)
+	checkInterval := *configCheckInterval
+	if checkInterval == 0 && *rulesCheckInterval > 0 {
+		logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
+		checkInterval = *rulesCheckInterval
+	}
+	if checkInterval > 0 {
+		ticker := time.NewTicker(checkInterval)
 		configCheckCh = ticker.C
 		defer ticker.Stop()
 	}
@@ -270,6 +283,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
+		if err := notifier.Reload(); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to reload notifier config: %s", err)
+			continue
+		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
@@ -284,13 +303,13 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			// config didn't change - skip it
 			continue
 		}
-		groupsCfg = newGroupsCfg
-		if err := m.update(ctx, groupsCfg, false); err != nil {
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)
--- a/app/vmalert/main_test.go
+++ b/app/vmalert/main_test.go
@@ -10,6 +10,7 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )

@@ -99,6 +100,8 @@ groups:
 		querierBuilder: &fakeQuerier{},
 		groups:         make(map[uint64]*Group),
 		labels:         map[string]string{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		rw:             &remotewrite.Client{},
 	}

 	syncCh := make(chan struct{})
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"sort"
 	"sync"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -15,7 +17,7 @@ import (
 // manager controls group states
 type manager struct {
 	querierBuilder datasource.QuerierBuilder
-	notifiers      []notifier.Notifier
+	notifiers      func() []notifier.Notifier

 	rw *remotewrite.Client
 	// remote read builder.
@@ -85,12 +87,31 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 }

 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
+	var rrPresent, arPresent bool
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
+		for _, r := range cfg.Rules {
+			if rrPresent && arPresent {
+				continue
+			}
+			if r.Record != "" {
+				rrPresent = true
+			}
+			if r.Alert != "" {
+				arPresent = true
+			}
+		}
 		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}

+	if rrPresent && m.rw == nil {
+		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
+	}
+	if arPresent && m.notifiers == nil {
+		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
+	}
+
 	type updateItem struct {
 		old *Group
 		new *Group
@@ -142,13 +163,13 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),

-		Name:              g.Name,
-		Type:              g.Type.String(),
-		File:              g.File,
-		Interval:          g.Interval.String(),
-		Concurrency:       g.Concurrency,
-		ExtraFilterLabels: g.ExtraFilterLabels,
-		Labels:            g.Labels,
+		Name:        g.Name,
+		Type:        g.Type.String(),
+		File:        g.File,
+		Interval:    g.Interval.String(),
+		Concurrency: g.Concurrency,
+		Params:      urlValuesToStrings(g.Params),
+		Labels:      g.Labels,
 	}
 	for _, r := range g.Rules {
 		switch v := r.(type) {
@@ -160,3 +181,24 @@ func (g *Group) toAPI() APIGroup {
 	}
 	return ag
 }
+
+func urlValuesToStrings(values url.Values) []string {
+	if len(values) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		params := values[k]
+		for _, v := range params {
+			res = append(res, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return res
+}
--- a/app/vmalert/manager_test.go
+++ b/app/vmalert/manager_test.go
@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 )

 func TestMain(m *testing.M) {
@@ -38,7 +40,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: &fakeQuerier{},
-		notifiers:      []notifier.Notifier{&fakeNotifier{}},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -218,7 +220,11 @@ func TestManagerUpdate(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}}
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+			}

 			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
 			if err := m.update(ctx, cfgInit, false); err != nil {
@@ -247,6 +253,80 @@ func TestManagerUpdate(t *testing.T) {
 	}
 }

+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        *remotewrite.Client
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&fakeNotifier{}},
+			nil,
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				rw:             tc.rw,
+			}
+			if tc.notifiers != nil {
+				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
+}
+
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
 	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)
--- a/app/vmalert/metrics.go
+++ b/app/vmalert/metrics.go
@@ -1,39 +0,0 @@
-package main
-
-import "github.com/VictoriaMetrics/metrics"
-
-type gauge struct {
-	name string
-	*metrics.Gauge
-}
-
-func getOrCreateGauge(name string, f func() float64) *gauge {
-	return &gauge{
-		name:  name,
-		Gauge: metrics.GetOrCreateGauge(name, f),
-	}
-}
-
-type counter struct {
-	name string
-	*metrics.Counter
-}
-
-func getOrCreateCounter(name string) *counter {
-	return &counter{
-		name:    name,
-		Counter: metrics.GetOrCreateCounter(name),
-	}
-}
-
-type summary struct {
-	name string
-	*metrics.Summary
-}
-
-func getOrCreateSummary(name string) *summary {
-	return &summary{
-		name:    name,
-		Summary: metrics.GetOrCreateSummary(name),
-	}
-}
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -70,7 +70,13 @@ type AlertTplData struct {
 	Expr   string
 }

-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+}

 // ExecTemplate executes the Alert template for given
 // map of annotations.
@@ -100,13 +106,15 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
 	for key, text := range annotations {
 		buf.Reset()
 		builder.Reset()
-		builder.Grow(len(tplHeader) + len(text))
-		builder.WriteString(tplHeader)
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -116,7 +124,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	return r, eg.Err()
 }

-func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error {
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
 	t := template.New("").Funcs(funcs).Option("missingkey=zero")
 	tpl, err := t.Parse(text)
 	if err != nil {
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@@ -1,12 +1,24 @@
 package notifier

 import (
+	"fmt"
 	"testing"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )

 func TestAlert_ExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string, 0)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
 	testCases := []struct {
 		name        string
 		alert       *Alert
@@ -74,6 +86,26 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"desc":    "bar 1;garply 2;",
 			},
 		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
 	}

 	qFn := func(q string) ([]datasource.Metric, error) {
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@@ -6,18 +6,41 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )

 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	addr          string
-	alertURL      string
-	basicAuthUser string
-	basicAuthPass string
-	argFunc       AlertURLGenerator
-	client        *http.Client
+	addr    string
+	argFunc AlertURLGenerator
+	client  *http.Client
+	timeout time.Duration
+
+	authCfg *promauth.Config
+
+	metrics *metrics
+}
+
+type metrics struct {
+	alertsSent       *utils.Counter
+	alertsSendErrors *utils.Counter
+}
+
+func newMetrics(addr string) *metrics {
+	return &metrics{
+		alertsSent:       utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+	}
+}
+
+// Close is a destructor method for AlertManager
+func (am *AlertManager) Close() {
+	am.metrics.alertsSent.Unregister()
+	am.metrics.alertsSendErrors.Unregister()
 }

 // Addr returns address where alerts are sent.
@@ -25,17 +48,36 @@ func (am AlertManager) Addr() string { return am.addr }

 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
+	am.metrics.alertsSent.Add(len(alerts))
+	err := am.send(ctx, alerts)
+	if err != nil {
+		am.metrics.alertsSendErrors.Add(len(alerts))
+	}
+	return err
+}
+
+func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc)

-	req, err := http.NewRequest("POST", am.alertURL, b)
+	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
+
+	if am.timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, am.timeout)
+		defer cancel()
+	}
+
 	req = req.WithContext(ctx)
-	if am.basicAuthPass != "" {
-		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
+
+	if am.authCfg != nil {
+		if auth := am.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -47,9 +89,9 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return fmt.Errorf("failed to read response from %q: %w", am.alertURL, err)
+			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
 		}
-		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body))
 	}
 	return nil
 }
@@ -60,14 +102,31 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"

 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
-	url := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
-	return &AlertManager{
-		addr:          alertManagerURL,
-		alertURL:      url,
-		argFunc:       fn,
-		client:        c,
-		basicAuthUser: user,
-		basicAuthPass: pass,
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+	tls := &promauth.TLSConfig{}
+	if authCfg.TLSConfig != nil {
+		tls = authCfg.TLSConfig
 	}
+	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+
+	ba := &promauth.BasicAuthConfig{}
+	if authCfg.BasicAuth != nil {
+		ba = authCfg.BasicAuth
+	}
+	aCfg, err := utils.AuthConfig(ba.Username, ba.Password.String(), ba.PasswordFile, authCfg.BearerToken.String(), authCfg.BearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
+	return &AlertManager{
+		addr:    alertManagerURL,
+		argFunc: fn,
+		authCfg: aCfg,
+		client:  &http.Client{Transport: tr},
+		timeout: timeout,
+		metrics: newMetrics(alertManagerURL),
+	}, nil
 }
--- a/app/vmalert/notifier/alertmanager_test.go
+++ b/app/vmalert/notifier/alertmanager_test.go
@@ -8,11 +8,16 @@ import (
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )

 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am := NewAlertManager(addr, "", "", nil, nil)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
 	if am.Addr() != addr {
 		t.Errorf("expected to have %q; got %q", addr, am.Addr())
 	}
@@ -75,9 +80,19 @@ func TestAlertManager_Send(t *testing.T) {
 	})
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewAlertManager(srv.URL, baUser, baPass, func(alert Alert) string {
+
+	aCfg := promauth.HTTPClientConfig{
+		BasicAuth: &promauth.BasicAuthConfig{
+			Username: baUser,
+			Password: promauth.NewSecret(baPass),
+		},
+	}
+	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, srv.Client())
+	}, aCfg, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil {
 		t.Error("expected connection error got nil")
 	}
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@@ -0,0 +1,182 @@
+package notifier
+
+import (
+	"crypto/md5"
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// Config contains list of supported configuration settings
+// for Notifier
+type Config struct {
+	// Scheme defines the HTTP scheme for Notifier address
+	Scheme string `yaml:"scheme,omitempty"`
+	// PathPrefix is added to URL path before adding alertManagerPath value
+	PathPrefix string `yaml:"path_prefix,omitempty"`
+
+	// ConsulSDConfigs contains list of settings for service discovery via Consul
+	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// StaticConfigs contains list of static targets
+	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
+
+	// HTTPClientConfig contains HTTP configuration for Notifier clients
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
+	// RelabelConfigs contains list of relabeling rules
+	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
+
+	// The timeout used when sending alerts.
+	Timeout promutils.Duration `yaml:"timeout,omitempty"`
+
+	// Checksum stores the hash of yaml definition for the config.
+	// May be used to detect any changes to the config file.
+	Checksum string
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+
+	// This is set to the directory from where the config has been loaded.
+	baseDir string
+
+	// stores already parsed RelabelConfigs object
+	parsedRelabelConfigs *promrelabel.ParsedConfigs
+}
+
+// StaticConfig contains list of static targets in the following form:
+// 	targets:
+//  	[ - '<host>' ]
+type StaticConfig struct {
+	Targets []string `yaml:"targets"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type config Config
+	if err := unmarshal((*config)(cfg)); err != nil {
+		return err
+	}
+	if cfg.Scheme == "" {
+		cfg.Scheme = "http"
+	}
+	if cfg.Timeout.Duration() == 0 {
+		cfg.Timeout = promutils.NewDuration(time.Second * 10)
+	}
+	rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse relabeling config: %w", err)
+	}
+	cfg.parsedRelabelConfigs = rCfg
+
+	b, err := yaml.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)
+	}
+	h := md5.New()
+	h.Write(b)
+	cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
+func parseConfig(path string) (*Config, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading config file: %w", err)
+	}
+	var cfg *Config
+	err = yaml.Unmarshal(data, &cfg)
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.XXX) > 0 {
+		var keys []string
+		for k := range cfg.XXX {
+			keys = append(keys, k)
+		}
+		return nil, fmt.Errorf("unknown fields in %s", strings.Join(keys, ", "))
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
+	}
+	cfg.baseDir = filepath.Dir(absPath)
+	return cfg, nil
+}
+
+func parseLabels(target string, metaLabels map[string]string, cfg *Config) (string, []prompbmarshal.Label, error) {
+	labels := mergeLabels(target, metaLabels, cfg)
+	labels = cfg.parsedRelabelConfigs.Apply(labels, 0, false)
+	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
+	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
+	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details.
+	labels = append([]prompbmarshal.Label{}, labels...)
+
+	if len(labels) == 0 {
+		return "", nil, nil
+	}
+	schemeRelabeled := promrelabel.GetLabelValueByName(labels, "__scheme__")
+	if len(schemeRelabeled) == 0 {
+		schemeRelabeled = "http"
+	}
+	addressRelabeled := promrelabel.GetLabelValueByName(labels, "__address__")
+	if len(addressRelabeled) == 0 {
+		return "", nil, nil
+	}
+	if strings.Contains(addressRelabeled, "/") {
+		return "", nil, nil
+	}
+	addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled)
+	alertsPathRelabeled := promrelabel.GetLabelValueByName(labels, "__alerts_path__")
+	if !strings.HasPrefix(alertsPathRelabeled, "/") {
+		alertsPathRelabeled = "/" + alertsPathRelabeled
+	}
+	u := fmt.Sprintf("%s://%s%s", schemeRelabeled, addressRelabeled, alertsPathRelabeled)
+	if _, err := url.Parse(u); err != nil {
+		return "", nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q, metrics_path=%q (%q): %w",
+			u, cfg.Scheme, schemeRelabeled, target, addressRelabeled, alertsPathRelabeled, err)
+	}
+	return u, labels, nil
+}
+
+func addMissingPort(scheme, target string) string {
+	if strings.Contains(target, ":") {
+		return target
+	}
+	if scheme == "https" {
+		target += ":443"
+	} else {
+		target += ":80"
+	}
+	return target
+}
+
+func mergeLabels(target string, metaLabels map[string]string, cfg *Config) []prompbmarshal.Label {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+	m := make(map[string]string)
+	m["__address__"] = target
+	m["__scheme__"] = cfg.Scheme
+	m["__alerts_path__"] = path.Join("/", cfg.PathPrefix, alertManagerPath)
+	for k, v := range metaLabels {
+		m[k] = v
+	}
+	result := make([]prompbmarshal.Label, 0, len(m))
+	for k, v := range m {
+		result = append(result, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	return result
+}
--- a/app/vmalert/notifier/config_test.go
+++ b/app/vmalert/notifier/config_test.go
@@ -0,0 +1,31 @@
+package notifier
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestConfigParseGood(t *testing.T) {
+	f := func(path string) {
+		_, err := parseConfig(path)
+		checkErr(t, err)
+	}
+	f("testdata/mixed.good.yaml")
+	f("testdata/consul.good.yaml")
+	f("testdata/static.good.yaml")
+}
+
+func TestConfigParseBad(t *testing.T) {
+	f := func(path, expErr string) {
+		_, err := parseConfig(path)
+		if err == nil {
+			t.Fatalf("expected to get non-nil err for config %q", path)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("testdata/unknownFields.bad.yaml", "unknown field")
+	f("non-existing-file", "error reading")
+}
--- a/app/vmalert/notifier/config_watcher.go
+++ b/app/vmalert/notifier/config_watcher.go
@@ -0,0 +1,235 @@
+package notifier
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+)
+
+// configWatcher supports dynamic reload of Notifier objects
+// from static configuration and service discovery.
+// Use newWatcher to create a new object.
+type configWatcher struct {
+	cfg   *Config
+	genFn AlertURLGenerator
+	wg    sync.WaitGroup
+
+	reloadCh chan struct{}
+	syncCh   chan struct{}
+
+	targetsMu sync.RWMutex
+	targets   map[TargetType][]Target
+}
+
+func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) {
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return nil, err
+	}
+	cw := &configWatcher{
+		cfg:       cfg,
+		wg:        sync.WaitGroup{},
+		reloadCh:  make(chan struct{}, 1),
+		syncCh:    make(chan struct{}),
+		genFn:     gen,
+		targetsMu: sync.RWMutex{},
+		targets:   make(map[TargetType][]Target),
+	}
+	return cw, cw.start()
+}
+
+func (cw *configWatcher) notifiers() []Notifier {
+	cw.targetsMu.RLock()
+	defer cw.targetsMu.RUnlock()
+
+	var notifiers []Notifier
+	for _, ns := range cw.targets {
+		for _, n := range ns {
+			notifiers = append(notifiers, n.Notifier)
+		}
+
+	}
+	return notifiers
+}
+
+func (cw *configWatcher) reload(path string) error {
+	select {
+	case cw.reloadCh <- struct{}{}:
+	default:
+		return nil
+	}
+
+	defer func() { <-cw.reloadCh }()
+
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return err
+	}
+	if cfg.Checksum == cw.cfg.Checksum {
+		return nil
+	}
+
+	// stop existing discovery
+	cw.mustStop()
+
+	// re-start cw with new config
+	cw.syncCh = make(chan struct{})
+	cw.cfg = cfg
+	return cw.start()
+}
+
+func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
+	targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+	for _, err := range errors {
+		return fmt.Errorf("failed to init notifier for %q: %s", typeK, err)
+	}
+
+	cw.setTargets(typeK, targets)
+
+	cw.wg.Add(1)
+	go func() {
+		defer cw.wg.Done()
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-cw.syncCh:
+				return
+			case <-ticker.C:
+			}
+			updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+			for _, err := range errors {
+				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+			}
+			cw.setTargets(typeK, updateTargets)
+		}
+	}()
+	return nil
+}
+
+func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
+	metaLabels, err := labelsFn()
+	if err != nil {
+		return nil, []error{fmt.Errorf("failed to get labels: %s", err)}
+	}
+	var targets []Target
+	var errors []error
+	duplicates := make(map[string]struct{})
+	for _, labels := range metaLabels {
+		target := labels["__address__"]
+		u, processedLabels, err := parseLabels(target, labels, cfg)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		if len(u) == 0 {
+			continue
+		}
+		if _, ok := duplicates[u]; ok { // check for duplicates
+			if !*suppressDuplicateTargetErrors {
+				logger.Errorf("skipping duplicate target with identical address %q; "+
+					"make sure service discovery and relabeling is set up properly; "+
+					"original labels: %s; resulting labels: %s",
+					u, labels, processedLabels)
+			}
+			continue
+		}
+		duplicates[u] = struct{}{}
+
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		targets = append(targets, Target{
+			Notifier: am,
+			Labels:   processedLabels,
+		})
+	}
+	return targets, errors
+}
+
+type getLabels func() ([]map[string]string, error)
+
+func (cw *configWatcher) start() error {
+	if len(cw.cfg.StaticConfigs) > 0 {
+		var targets []Target
+		for _, cfg := range cw.cfg.StaticConfigs {
+			for _, target := range cfg.Targets {
+				address, labels, err := parseLabels(target, nil, cw.cfg)
+				if err != nil {
+					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
+				}
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				if err != nil {
+					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
+				}
+				targets = append(targets, Target{
+					Notifier: notifier,
+					Labels:   labels,
+				})
+			}
+		}
+		cw.setTargets(TargetStatic, targets)
+	}
+
+	if len(cw.cfg.ConsulSDConfigs) > 0 {
+		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.ConsulSDConfigs {
+				sdc := &cw.cfg.ConsulSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start consulSD discovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func (cw *configWatcher) mustStop() {
+	close(cw.syncCh)
+	cw.wg.Wait()
+
+	cw.targetsMu.Lock()
+	for _, targets := range cw.targets {
+		for _, t := range targets {
+			t.Close()
+		}
+	}
+	cw.targets = make(map[TargetType][]Target)
+	cw.targetsMu.Unlock()
+
+	for i := range cw.cfg.ConsulSDConfigs {
+		cw.cfg.ConsulSDConfigs[i].MustStop()
+	}
+	cw.cfg = nil
+}
+
+func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
+	cw.targetsMu.Lock()
+	newT := make(map[string]Target)
+	for _, t := range targets {
+		newT[t.Addr()] = t
+	}
+	oldT := cw.targets[key]
+
+	for _, ot := range oldT {
+		if _, ok := newT[ot.Addr()]; !ok {
+			ot.Notifier.Close()
+		}
+	}
+	cw.targets[key] = targets
+	cw.targetsMu.Unlock()
+}
--- a/app/vmalert/notifier/config_watcher_test.go
+++ b/app/vmalert/notifier/config_watcher_test.go
@@ -0,0 +1,301 @@
+package notifier
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"testing"
+)
+
+func TestConfigWatcherReload(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f.Name()) }()
+
+	writeToFile(t, f.Name(), `
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9094
+`)
+	cw, err := newWatcher(f.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+	ns := cw.notifiers()
+	if len(ns) != 2 {
+		t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns)
+	}
+
+	f2, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f2.Name()) }()
+
+	writeToFile(t, f2.Name(), `
+static_configs:
+  - targets:
+      - 127.0.0.1:9093
+`)
+	checkErr(t, cw.reload(f2.Name()))
+
+	ns = cw.notifiers()
+	if len(ns) != 1 {
+		t.Fatalf("expected to have 1 notifier; got %d", len(ns))
+	}
+	expAddr := "http://127.0.0.1:9093/api/v2/alerts"
+	if ns[0].Addr() != expAddr {
+		t.Fatalf("expected to get %q; got %q instead", expAddr, ns[0].Addr())
+	}
+}
+
+func TestConfigWatcherStart(t *testing.T) {
+	consulSDServer := newFakeConsulServer()
+	defer consulSDServer.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+scheme: https
+path_prefix: proxy
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+`, consulSDServer.URL))
+
+	cw, err := newWatcher(consulSDFile.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	if len(cw.notifiers()) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers()))
+	}
+
+	expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1)
+	expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2)
+
+	n1, n2 := cw.notifiers()[0], cw.notifiers()[1]
+	if n1.Addr() != expAddr1 {
+		t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr())
+	}
+	if n2.Addr() != expAddr2 {
+		t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
+	}
+}
+
+// TestConfigWatcherReloadConcurrent supposed to test concurrent
+// execution of configuration update.
+// Should be executed with -race flag
+func TestConfigWatcherReloadConcurrent(t *testing.T) {
+	consulSDServer1 := newFakeConsulServer()
+	defer consulSDServer1.Close()
+	consulSDServer2 := newFakeConsulServer()
+	defer consulSDServer2.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	staticAndConsulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(staticAndConsulSDFile.Name()) }()
+
+	writeToFile(t, staticAndConsulSDFile.Name(), fmt.Sprintf(`
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	paths := []string{
+		staticAndConsulSDFile.Name(),
+		consulSDFile.Name(),
+		"testdata/static.good.yaml",
+		"unknownFields.bad.yaml",
+	}
+
+	cw, err := newWatcher(paths[0], nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	const workers = 500
+	const iterations = 10
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				_ = cw.reload(paths[rnd]) // update can fail and this is expected
+				_ = cw.notifiers()
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	checkErr(t, ioutil.WriteFile(file, []byte(b), 0644))
+}
+
+func checkErr(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+}
+
+const (
+	fakeConsulService1 = "127.0.0.1:9093"
+	fakeConsulService2 = "127.0.0.1:9095"
+)
+
+func newFakeConsulServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
+	})
+	mux.HandleFunc("/v1/catalog/services", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`{
+    "alertmanager": [
+        "alertmanager",
+        "__scheme__=http"
+    ]
+}`))
+	})
+	mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`
+[
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am1",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "__scheme__=http"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9093,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 16,
+            "ModifyIndex": 16
+        }
+    },
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am2",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "bad-node"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9095,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 15,
+            "ModifyIndex": 15
+        }
+    }
+]`))
+	})
+
+	return httptest.NewServer(mux)
+}
--- a/app/vmalert/notifier/init.go
+++ b/app/vmalert/notifier/init.go
@@ -1,17 +1,24 @@
 package notifier

 import (
+	"flag"
 	"fmt"
-	"net/http"
+	"strings"
+	"time"

-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )

 var (
-	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
-	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
+	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
+
+	addrs                 = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")

 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
 	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
@@ -22,24 +29,135 @@ var (
 		"By default the server name from -notifier.url is used")
 )

-// Init creates a Notifier object based on provided flags.
-func Init(gen AlertURLGenerator) ([]Notifier, error) {
-	if len(*addrs) == 0 {
-		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
+// cw holds a configWatcher for configPath configuration file
+// configWatcher provides a list of Notifier objects discovered
+// from static config or via service discovery.
+// cw is not nil only if configPath is provided.
+var cw *configWatcher
+
+// Reload checks the changes in configPath configuration file
+// and applies changes if any.
+func Reload() error {
+	if cw == nil {
+		return nil
+	}
+	return cw.reload(*configPath)
+}
+
+var staticNotifiersFn func() []Notifier
+
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
+// Init returns a function for retrieving actual list of Notifier objects.
+// Init works in two mods:
+//   * configuration via flags (for backward compatibility). Is always static
+//     and don't support live reloads.
+//   * configuration via file. Supports live reloads and service discovery.
+// Init returns an error if both mods are used.
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	if externalLabels != nil || externalURL != "" {
+		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
 	}

+	externalURL = extURL
+	externalLabels = extLabels
+
+	if *configPath == "" && len(*addrs) == 0 {
+		return nil, nil
+	}
+	if *configPath != "" && len(*addrs) > 0 {
+		return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
+	}
+
+	if len(*addrs) > 0 {
+		notifiers, err := notifiersFromFlags(gen)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create notifier from flag values: %s", err)
+		}
+		staticNotifiersFn = func() []Notifier {
+			return notifiers
+		}
+		return staticNotifiersFn, nil
+	}
+
+	var err error
+	cw, err = newWatcher(*configPath, gen)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init config watcher: %s", err)
+	}
+	return cw.notifiers, nil
+}
+
+func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 	var notifiers []Notifier
 	for i, addr := range *addrs {
-		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)
-		ca, serverName := tlsCAFile.GetOptionalArg(i), tlsServerName.GetOptionalArg(i)
-		tr, err := utils.Transport(addr, cert, key, ca, serverName, tlsInsecureSkipVerify.GetOptionalArg(i))
-		if err != nil {
-			return nil, fmt.Errorf("failed to create transport: %w", err)
+		authCfg := promauth.HTTPClientConfig{
+			TLSConfig: &promauth.TLSConfig{
+				CAFile:             tlsCAFile.GetOptionalArg(i),
+				CertFile:           tlsCertFile.GetOptionalArg(i),
+				KeyFile:            tlsKeyFile.GetOptionalArg(i),
+				ServerName:         tlsServerName.GetOptionalArg(i),
+				InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i),
+			},
+			BasicAuth: &promauth.BasicAuthConfig{
+				Username:     basicAuthUsername.GetOptionalArg(i),
+				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
+				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
+			},
+		}
+		addr = strings.TrimSuffix(addr, "/")
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		if err != nil {
+			return nil, err
 		}
-		user, pass := basicAuthUsername.GetOptionalArg(i), basicAuthPassword.GetOptionalArg(i)
-		am := NewAlertManager(addr, user, pass, gen, &http.Client{Transport: tr})
 		notifiers = append(notifiers, am)
 	}
-
 	return notifiers, nil
 }
+
+// Target represents a Notifier and optional
+// list of labels added during discovery.
+type Target struct {
+	Notifier
+	Labels []prompbmarshal.Label
+}
+
+// TargetType defines how the Target was discovered
+type TargetType string
+
+const (
+	// TargetStatic is for targets configured statically
+	TargetStatic TargetType = "static"
+	// TargetConsul is for targets discovered via Consul
+	TargetConsul TargetType = "consulSD"
+)
+
+// GetTargets returns list of static or discovered targets
+// via notifier configuration.
+func GetTargets() map[TargetType][]Target {
+	var targets = make(map[TargetType][]Target)
+
+	if staticNotifiersFn != nil {
+		for _, ns := range staticNotifiersFn() {
+			targets[TargetStatic] = append(targets[TargetStatic], Target{
+				Notifier: ns,
+			})
+		}
+	}
+
+	if cw != nil {
+		cw.targetsMu.RLock()
+		for key, ns := range cw.targets {
+			targets[key] = append(targets[key], ns...)
+		}
+		cw.targetsMu.RUnlock()
+	}
+	return targets
+}
--- a/app/vmalert/notifier/notifier.go
+++ b/app/vmalert/notifier/notifier.go
@@ -10,4 +10,6 @@ type Notifier interface {
 	Send(ctx context.Context, alerts []Alert) error
 	// Addr returns address where alerts are sent.
 	Addr() string
+	// Close is a destructor for the Notifier
+	Close()
 }
--- a/app/vmalert/notifier/template_func.go
+++ b/app/vmalert/notifier/template_func.go
@@ -17,6 +17,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net"
 	"net/url"
 	"regexp"
 	"strings"
@@ -26,6 +27,7 @@ import (
 	textTpl "text/template"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )

 // metric is private copy of datasource.Metric,
@@ -61,6 +63,7 @@ var tmplFunc textTpl.FuncMap

 // InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
 	tmplFunc = textTpl.FuncMap{
 		/* Strings */

@@ -91,6 +94,24 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// alias for https://golang.org/pkg/strings/#ToLower
 		"toLower": strings.ToLower,

+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
 		/* Numbers */

 		// humanize converts given number to a human readable format
--- a/app/vmalert/notifier/testdata/consul.good.yaml
+++ b/app/vmalert/notifier/testdata/consul.good.yaml
@@ -0,0 +1,13 @@
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__
--- a/app/vmalert/notifier/testdata/mixed.good.yaml
+++ b/app/vmalert/notifier/testdata/mixed.good.yaml
@@ -0,0 +1,18 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__
--- a/app/vmalert/notifier/testdata/static.good.yaml
+++ b/app/vmalert/notifier/testdata/static.good.yaml
@@ -0,0 +1,4 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
--- a/app/vmalert/notifier/testdata/unknownFields.bad.yaml
+++ b/app/vmalert/notifier/testdata/unknownFields.bad.yaml
@@ -0,0 +1,5 @@
+scheme: https
+unknown: field
+static_configs:
+  - targets:
+      - localhost:9093
--- a/app/vmalert/recording.go
+++ b/app/vmalert/recording.go
@@ -10,8 +10,8 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )

 // RecordingRule is a Rule that supposed
@@ -43,8 +43,8 @@ type RecordingRule struct {
 }

 type recordingRuleMetrics struct {
-	errors  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	samples *utils.Gauge
 }

 // String implements Stringer interface
@@ -70,12 +70,12 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 	}

 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
-	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
+	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -84,7 +84,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 			}
 			return 1
 		})
-	rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -95,8 +95,8 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul

 // Close unregisters rule metrics
 func (rr *RecordingRule) Close() {
-	metrics.UnregisterMetric(rr.metrics.errors.name)
-	metrics.UnregisterMetric(rr.metrics.samples.name)
+	rr.metrics.errors.Unregister()
+	rr.metrics.samples.Unregister()
 }

 // ExecRange executes recording rule on the given time range similarly to Exec.
--- a/app/vmalert/replay_test.go
+++ b/app/vmalert/replay_test.go
@@ -8,7 +8,7 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )

 type fakeReplayQuerier struct {
@@ -83,7 +83,7 @@ func TestReplay(t *testing.T) {
 			to:    "2021-01-01T15:02:30.000Z",
 			maxDP: 60,
 			cfg: []config.Group{
-				{Interval: utils.NewPromDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 			},
 			qb: &fakeReplayQuerier{
 				registry: map[string]map[string]struct{}{
--- a/app/vmalert/utils/metrics.go
+++ b/app/vmalert/utils/metrics.go
@@ -0,0 +1,54 @@
+package utils
+
+import "github.com/VictoriaMetrics/metrics"
+
+type namedMetric struct {
+	Name string
+}
+
+// Unregister removes the metric by name from default registry
+func (nm namedMetric) Unregister() {
+	metrics.UnregisterMetric(nm.Name)
+}
+
+// Gauge is a metrics.Gauge with Name
+type Gauge struct {
+	namedMetric
+	*metrics.Gauge
+}
+
+// GetOrCreateGauge creates a new Gauge with the given name
+func GetOrCreateGauge(name string, f func() float64) *Gauge {
+	return &Gauge{
+		namedMetric: namedMetric{Name: name},
+		Gauge:       metrics.GetOrCreateGauge(name, f),
+	}
+}
+
+// Counter is a metrics.Counter with Name
+type Counter struct {
+	namedMetric
+	*metrics.Counter
+}
+
+// GetOrCreateCounter creates a new Counter with the given name
+func GetOrCreateCounter(name string) *Counter {
+	return &Counter{
+		namedMetric: namedMetric{Name: name},
+		Counter:     metrics.GetOrCreateCounter(name),
+	}
+}
+
+// Summary is a metrics.Summary with Name
+type Summary struct {
+	namedMetric
+	*metrics.Summary
+}
+
+// GetOrCreateSummary creates a new Summary with the given name
+func GetOrCreateSummary(name string) *Summary {
+	return &Summary{
+		namedMetric: namedMetric{Name: name},
+		Summary:     metrics.GetOrCreateSummary(name),
+	}
+}
--- a/app/vmalert/utils/prom_duration.go
+++ b/app/vmalert/utils/prom_duration.go
@@ -1,43 +0,0 @@
-package utils
-
-import (
-	"time"
-
-	"github.com/VictoriaMetrics/metricsql"
-)
-
-// PromDuration is Prometheus duration.
-type PromDuration struct {
-	milliseconds int64
-}
-
-// NewPromDuration returns PromDuration for given d.
-func NewPromDuration(d time.Duration) PromDuration {
-	return PromDuration{
-		milliseconds: d.Milliseconds(),
-	}
-}
-
-// MarshalYAML implements yaml.Marshaler interface.
-func (pd PromDuration) MarshalYAML() (interface{}, error) {
-	return pd.Duration().String(), nil
-}
-
-// UnmarshalYAML implements yaml.Unmarshaler interface.
-func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var s string
-	if err := unmarshal(&s); err != nil {
-		return err
-	}
-	ms, err := metricsql.DurationValue(s, 0)
-	if err != nil {
-		return err
-	}
-	pd.milliseconds = ms
-	return nil
-}
-
-// Duration returns duration for pd.
-func (pd *PromDuration) Duration() time.Duration {
-	return time.Duration(pd.milliseconds) * time.Millisecond
-}
--- a/app/vmalert/vmalert_cluster.excalidraw
+++ b/app/vmalert/vmalert_cluster.excalidraw
@@ -0,0 +1,525 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 794,
+      "versionNonce": 1855937036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 659,
+      "versionNonce": 247957684,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347948032,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1670,
+      "versionNonce": 2021681972,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 750.3317260742188,
+      "y": 226.5509033203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348054411
+    },
+    {
+      "type": "text",
+      "version": 1311,
+      "versionNonce": 1283453068,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 759.2862243652344,
+      "y": 238.68240356445312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert:8480",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 897,
+      "versionNonce": 1983434892,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 61.744873046875,
+      "y": 224.9600830078125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638348077724
+    },
+    {
+      "type": "text",
+      "version": 719,
+      "versionNonce": 457402292,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 70.69937133789062,
+      "y": 237.33523559570312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect:8481",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1098,
+      "versionNonce": 1480603788,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 371.7434387207031,
+      "y": 398.50787353515625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 864,
+      "versionNonce": 1115813900,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 393.73809814453125,
+      "y": 410.5976257324219,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348009180,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2405,
+      "versionNonce": 959767732,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 556.6860961914062,
+      "y": 252.2582773408825,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 184.9195556640625,
+      "height": 1.6022679018915937,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.0344528515859526,
+        "gap": 10.57574462890625
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.039393828258510157,
+        "gap": 8.72607421875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.9195556640625,
+          -1.6022679018915937
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1173,
+      "versionNonce": 1248255756,
+      "isDeleted": false,
+      "id": "wRO0q9xKPHc8e8XPPsQWh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.0439244722469,
+      "y": 246.80533728741074,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 157.86774649373126,
+      "height": 0.1417938392881979,
+      "seed": 656189364,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.13736472619498497,
+        "gap": 16.306295254315614
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.013200835330936087,
+        "gap": 14.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -157.86774649373126,
+          0.1417938392881979
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 557,
+      "versionNonce": 995289780,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 552.4987182617188,
+      "y": 212.27996826171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 76,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348059525,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist alerts state\n\n\nand recording rules",
+      "baseline": 72,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 803,
+      "versionNonce": 1576507444,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 290.34130859375,
+      "y": 210.56927490234375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1471,
+      "versionNonce": 1361321140,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 484.18669893674246,
+      "y": 302.3424013553929,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 1.0484739253853945,
+      "height": 84.72775855671654,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348083348,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.010768924644894236,
+        "gap": 14.650750964767894
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.051051952959743775,
+        "gap": 11.437713623046818
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0484739253853945,
+          84.72775855671654
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 576,
+      "versionNonce": 2112088460,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 530.5612182617188,
+      "y": 318.60687255859375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348023735,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 1835119500,
+      "isDeleted": false,
+      "id": "ff5OkfgmkKLifS13_TFj3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 291.37474060058594,
+      "y": 261.4861297607422,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 264004620,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "restore state",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}
--- a/app/vmalert/vmalert_cluster.png
+++ b/app/vmalert/vmalert_cluster.png
--- a/app/vmalert/vmalert_ha.excalidraw
+++ b/app/vmalert/vmalert_ha.excalidraw
@@ -0,0 +1,911 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 906,
+      "versionNonce": 448716468,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 302.2676696777344,
+      "y": 275.59356689453125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "OTKoeHmKtqCxFArDbY-sP"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 762,
+      "versionNonce": 223660724,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 318.0538024902344,
+      "y": 296.6778106689453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2"
+      ],
+      "updated": 1638355102070,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert1",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 2067,
+      "versionNonce": 817347852,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 329.6426086425781,
+      "y": 90.3275146484375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355058507
+    },
+    {
+      "type": "text",
+      "version": 1717,
+      "versionNonce": 2040797452,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 349.01177978515625,
+      "y": 102.45901489257812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1553,
+      "versionNonce": 39381044,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 173.67257690429688,
+      "y": 470.5100402832031,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "OTKoeHmKtqCxFArDbY-sP",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 1309,
+      "versionNonce": 1112872844,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 189.667236328125,
+      "y": 482.59979248046875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager1:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 835,
+      "versionNonce": 2036483596,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 525.83837890625,
+      "y": 147.33470153808594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 777,
+      "versionNonce": 990468492,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 617.0690307617188,
+      "y": 376.9822692871094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638355227582,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1112,
+      "versionNonce": 999136652,
+      "isDeleted": false,
+      "id": "mIu-d0lmShCxzMLD5iA_p",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 492.29505920410156,
+      "y": 272.3052215576172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 756416780,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "S_dOHQrhGmu8SFJzobJK7",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "Ar-hcDLlzVSoTPs2MaywO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 970,
+      "versionNonce": 430760500,
+      "isDeleted": false,
+      "id": "ZqIR6SaLNDQl8s0zZbdnE",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 508.08119201660156,
+      "y": 293.38946533203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 1477404084,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355105020,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalertN",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "qZFezRGU4Chwxgvb2451t",
+      "type": "line",
+      "x": 449.93653869628906,
+      "y": 306.30010986328125,
+      "width": 19.48321533203125,
+      "height": 0.3865966796875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1833426828,
+      "version": 69,
+      "versionNonce": 871948300,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          19.48321533203125,
+          0.3865966796875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": null,
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": null
+    },
+    {
+      "type": "rectangle",
+      "version": 1649,
+      "versionNonce": 93981708,
+      "isDeleted": false,
+      "id": "gNHiZJKo0ap69ALDobtZ-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518.5596466064453,
+      "y": 471.4539489746094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 454422412,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "Ar-hcDLlzVSoTPs2MaywO",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 1412,
+      "versionNonce": 75038348,
+      "isDeleted": false,
+      "id": "O-zgjZBvt4RI1PrkBNQnb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 534.3148040771484,
+      "y": 483.543701171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1026268980,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanagerN:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "m9_BptFOFxbV2sS_xJDu2",
+      "type": "arrow",
+      "x": 378.57185562792466,
+      "y": 262.1596984863281,
+      "width": 79.96146539019026,
+      "height": 111.53411865234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 141224884,
+      "version": 521,
+      "versionNonce": 283254540,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          79.96146539019026,
+          -111.53411865234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.23996018586441184,
+        "gap": 13.433868408203125
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.14207100087207922,
+        "gap": 15.550811767578125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "S_dOHQrhGmu8SFJzobJK7",
+      "type": "arrow",
+      "x": 558.1990515577129,
+      "y": 263.4271240234375,
+      "width": 88.18940317574186,
+      "height": 114.15780639648438,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1004789812,
+      "version": 314,
+      "versionNonce": 1259171724,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355105019,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -88.18940317574186,
+          -114.15780639648438
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4315094049079832,
+        "gap": 8.878097534179688
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": 0.14840834302306677,
+        "gap": 14.194549560546875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "fsGFp4NW4JlrCdF0HR3uA",
+      "type": "arrow",
+      "x": 356.6613630516658,
+      "y": 354.95416259765625,
+      "width": 278.74351860741064,
+      "height": 106.90020751953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 712384692,
+      "version": 334,
+      "versionNonce": 266493324,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          278.74351860741064,
+          106.90020751953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.7721210277890177,
+        "gap": 7.228485107421875
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4493598001028791,
+        "gap": 9.599578857421875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "OTKoeHmKtqCxFArDbY-sP",
+      "type": "arrow",
+      "x": 361.02563221226757,
+      "y": 356.2252197265625,
+      "width": 95.3594006000182,
+      "height": 105.52130126953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1992252596,
+      "version": 466,
+      "versionNonce": 472553100,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355221749,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -95.3594006000182,
+          105.52130126953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.39325294425055324,
+        "gap": 8.499542236328125
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.400636314282374,
+        "gap": 8.763519287109375
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "XhPgLRBk-YhWAFcSQi9TJ",
+      "type": "arrow",
+      "x": 561.7189961327852,
+      "y": 349.238037109375,
+      "width": 266.271510370216,
+      "height": 112.383544921875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 588869260,
+      "version": 453,
+      "versionNonce": 584723212,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355150131,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -266.271510370216,
+          112.383544921875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": -0.7084007026732048,
+        "gap": 4.8007049560546875
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.4180430111580123,
+        "gap": 8.888458251953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "Ar-hcDLlzVSoTPs2MaywO",
+      "type": "arrow",
+      "x": 557.5999880535809,
+      "y": 352.71795654296875,
+      "width": 112.87813113656136,
+      "height": 106.3255615234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1508782476,
+      "version": 331,
+      "versionNonce": 1137561908,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355153683,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          112.87813113656136,
+          106.3255615234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4358123206211808,
+        "gap": 8.280624389648438
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4783744286829653,
+        "gap": 12.410430908203125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "WqHnv9_g3SdkLZuy4WSHa",
+      "type": "text",
+      "x": 186.5891876220703,
+      "y": 523.145263671875,
+      "width": 272,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1310147468,
+      "version": 281,
+      "versionNonce": 949157044,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "text": "Alertmanagers in cluster mode",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "id": "yA0kgvdF71wZbHJ7Cg2p8",
+      "type": "rectangle",
+      "x": 159.41685485839844,
+      "y": 440.7666015625,
+      "width": 625.9590148925781,
+      "height": 110.13494873046878,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1465986996,
+      "version": 366,
+      "versionNonce": 196040844,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939
+    },
+    {
+      "id": "D6fkQH1E_MFbCuL697ArO",
+      "type": "arrow",
+      "x": 422.4853057861328,
+      "y": 494.8779132311229,
+      "width": 90.5517578125,
+      "height": 0.12427004064892344,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 2025208204,
+      "version": 316,
+      "versionNonce": 2144102156,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          90.5517578125,
+          0.12427004064892344
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.08064204025505255,
+        "gap": 8.706283569335938
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": -0.05976220061952895,
+        "gap": 5.5225830078125
+      },
+      "startArrowhead": "arrow",
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 463,
+      "versionNonce": 2118914484,
+      "isDeleted": false,
+      "id": "p4EDvSKPqZzfM_SReybeq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 274.6086883544922,
+      "y": 56.77886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 394,
+      "height": 19,
+      "seed": 1214462348,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355074872,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VictoriaMetrics with deduplication enabled",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 712,
+      "versionNonce": 737833612,
+      "isDeleted": false,
+      "id": "YfNFeOucMo8BJk2P2JRex",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 229.77923583984375,
+      "y": 227.84378051757812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 448.5542297363281,
+      "height": 134.06329345703122,
+      "seed": 932207756,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355133244
+    },
+    {
+      "type": "text",
+      "version": 427,
+      "versionNonce": 1458527796,
+      "isDeleted": false,
+      "id": "oldM7Q_aJtpWd2jXQ0iKf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 238.9442901611328,
+      "y": 230.862548828125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 225,
+      "height": 38,
+      "seed": 2131899572,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355259358,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "vmalerts with identical \nconfigurations",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}
--- a/app/vmalert/vmalert_ha.png
+++ b/app/vmalert/vmalert_ha.png
--- a/app/vmalert/vmalert_multicluster.excalidraw
+++ b/app/vmalert/vmalert_multicluster.excalidraw
@@ -0,0 +1,915 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 791,
+      "versionNonce": 48874036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "pD9DcILMxa6GaR1U5YyMO",
+        "HPEwr85wL4IedW0AgdArp",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 658,
+      "versionNonce": 1653816076,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347796775,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 802,
+      "versionNonce": 995326644,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.05322265625,
+      "y": 228.65371704101562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 624,
+      "versionNonce": 707755700,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 640.7635803222656,
+      "y": 241.02886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701075,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 802,
+      "versionNonce": 1974403340,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 401.241943359375,
+      "y": 342.4627990722656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 178,
+      "height": 38,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347863144,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute aggregating\nrecording rules",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "jrFZeFNsxlss7IsEZpA-J",
+      "type": "rectangle",
+      "x": 594.1509857177734,
+      "y": 192.09194946289062,
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 48379060,
+      "version": 665,
+      "versionNonce": 617456652,
+      "isDeleted": false,
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347763716
+    },
+    {
+      "id": "6ibhLp94HJFIdfP5HCEv8",
+      "type": "text",
+      "x": 609.7205657958984,
+      "y": 199.81723022460938,
+      "width": 225,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1053164852,
+      "version": 256,
+      "versionNonce": 538595124,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347761092,
+      "text": "VM cluster with raw data",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "type": "rectangle",
+      "version": 914,
+      "versionNonce": 1576666164,
+      "isDeleted": false,
+      "id": "R5v-yZCVJ97BkJqr0Qb7H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 807.5664215087891,
+      "y": 287.8628387451172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1794212620,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "bZXA8PH9gYu-clotqJ4f7",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 725,
+      "versionNonce": 267455500,
+      "isDeleted": false,
+      "id": "pWRC_smX7TuOI8_8UrA4H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 840.0209197998047,
+      "y": 300.2379913330078,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 421856180,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 843,
+      "versionNonce": 1244193972,
+      "isDeleted": false,
+      "id": "EqROOfYulSPsZm7ovxfQN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 604.9091339111328,
+      "y": 345.2847442626953,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 2043521972,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347718537
+    },
+    {
+      "type": "text",
+      "version": 676,
+      "versionNonce": 143370892,
+      "isDeleted": false,
+      "id": "ddQH1nnmT7HbKW7Xmv4zx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 642.6199798583984,
+      "y": 357.90354919433594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 335223180,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "bZXA8PH9gYu-clotqJ4f7",
+      "type": "arrow",
+      "x": 785.2667708463345,
+      "y": 367.2342882272049,
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1153029388,
+      "version": 420,
+      "versionNonce": 1726025228,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347704644,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "EqROOfYulSPsZm7ovxfQN",
+        "focus": 0.5247890891198189,
+        "gap": 8.36404562660789
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": -0.6931056940383433,
+        "gap": 9.943033572650961
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "arrow",
+      "version": 528,
+      "versionNonce": 1908259468,
+      "isDeleted": false,
+      "id": "MGdu6GDIPNBAaEUr0Gt-a",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 781.4456641707259,
+      "y": 248.777857603323,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1695061516,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347737174,
+      "startBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 883,
+      "versionNonce": 845352076,
+      "isDeleted": false,
+      "id": "4pW8hvBu3bo1eMtFvg_gS",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 601.6520690917969,
+      "y": 473.7677536010742,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1678097076,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 703,
+      "versionNonce": 519371956,
+      "isDeleted": false,
+      "id": "w_i2hO06oLa0bWbyAfFzU",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 639.3624267578125,
+      "y": 486.14290618896484,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 340753036,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 745,
+      "versionNonce": 879581324,
+      "isDeleted": false,
+      "id": "U5U-67wL5fPwvzlxOAF5A",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 592.7498321533203,
+      "y": 437.2059860229492,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "seed": 912540724,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347776748
+    },
+    {
+      "type": "text",
+      "version": 345,
+      "versionNonce": 1628116148,
+      "isDeleted": false,
+      "id": "0IGVrvICMVeZp2RCaqTeP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 608.3194122314453,
+      "y": 444.93126678466797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 291,
+      "height": 19,
+      "seed": 68700428,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347784373,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VM cluster with aggregated data",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 996,
+      "versionNonce": 2112461580,
+      "isDeleted": false,
+      "id": "9QMf0HXPE1W4M9S-zMJVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 806.1652679443359,
+      "y": 532.9768753051758,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 523607476,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "9WBH34em4CdU2OwzqYIl5",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 804,
+      "versionNonce": 1660832052,
+      "isDeleted": false,
+      "id": "WOrOD5vn6EtMUQRJomt3-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 838.6197662353516,
+      "y": 545.3520278930664,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 1321420684,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 925,
+      "versionNonce": 2052807604,
+      "isDeleted": false,
+      "id": "4dtJZpXEUxSK3biwFG7vd",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.5079803466797,
+      "y": 590.3987808227539,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1738524468,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "9WBH34em4CdU2OwzqYIl5",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 756,
+      "versionNonce": 2040967820,
+      "isDeleted": false,
+      "id": "rAbyooo-X08-86qjoK0WR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 641.2188262939453,
+      "y": 603.0175857543945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 948598284,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "9WBH34em4CdU2OwzqYIl5"
+      ],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 660,
+      "versionNonce": 1560678196,
+      "isDeleted": false,
+      "id": "9WBH34em4CdU2OwzqYIl5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.8656172818814,
+      "y": 612.3483247872634,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "seed": 1141424308,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": 0.5247890891198189,
+        "gap": 8.364045626608004
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": -0.6931056940383404,
+        "gap": 9.943033572650847
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 768,
+      "versionNonce": 1653264948,
+      "isDeleted": false,
+      "id": "5W90eDBjtfZvkSuQTG0Iw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.0445106062728,
+      "y": 493.8918941633816,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1852298380,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4pW8hvBu3bo1eMtFvg_gS",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "type": "arrow",
+      "x": 423.70701599121094,
+      "y": 428.34434509277344,
+      "width": 179.54901123046875,
+      "height": 182.9937744140625,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 389863732,
+      "version": 47,
+      "versionNonce": 1364399028,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347805500,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          179.54901123046875,
+          -182.9937744140625
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "type": "arrow",
+      "x": 424.7585906982422,
+      "y": 441.12806701660156,
+      "width": 174.29449462890625,
+      "height": 170.56607055664062,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 981082124,
+      "version": 92,
+      "versionNonce": 1261546252,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347812431,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          174.29449462890625,
+          170.56607055664062
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": -0.8207814548026305,
+        "gap": 4.45489501953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 775,
+      "versionNonce": 95530764,
+      "isDeleted": false,
+      "id": "o-yIJ_WZzVubhbVODvhcC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 407.27012634277344,
+      "y": 489.33091735839844,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 775116812,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347824876,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}
--- a/app/vmalert/vmalert_multicluster.png
+++ b/app/vmalert/vmalert_multicluster.png
--- a/app/vmalert/vmalert_single.excalidraw
+++ b/app/vmalert/vmalert_single.excalidraw
@@ -0,0 +1,354 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 795,
+      "versionNonce": 1195460364,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 660,
+      "versionNonce": 1034125236,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1690,
+      "versionNonce": 236788620,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 344.9837951660156,
+      "y": 64.64248657226562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348176044
+    },
+    {
+      "type": "text",
+      "version": 1331,
+      "versionNonce": 945335476,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 364.35296630859375,
+      "y": 76.77398681640625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1099,
+      "versionNonce": 671872012,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 360.6495666503906,
+      "y": 382.2536315917969,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 865,
+      "versionNonce": 635839156,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 382.64422607421875,
+      "y": 394.3433837890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2444,
+      "versionNonce": 361351692,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.0465703465111,
+      "y": 204.98379516601562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.7962088410123442,
+      "height": 86.86798095703125,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "gap": 10.57574462890625,
+        "focus": 0.0344528515859526
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "gap": 8.72607421875,
+        "focus": -0.039393828258510157
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.7962088410123442,
+          -86.86798095703125
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 640,
+      "versionNonce": 1892951180,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 500.609619140625,
+      "y": 134.11544799804688,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348163424,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1472,
+      "versionNonce": 768565516,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.5245361328125,
+      "y": 300.5478957313172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.3521007656264601,
+      "height": 70.26802223743277,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "startBinding": {
+        "elementId": "E9Run6wCm2chQ6JHrmc_y",
+        "focus": 1.1925203824459496,
+        "gap": 11.77154541015625
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.0441077573536454,
+        "gap": 11.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.3521007656264601,
+          70.26802223743277
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 577,
+      "versionNonce": 1646003636,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 498.29608154296875,
+      "y": 298.6573791503906,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}
--- a/app/vmalert/vmalert_single.png
+++ b/app/vmalert/vmalert_single.png
--- a/app/vmalert/web.go
+++ b/app/vmalert/web.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"

+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -24,6 +25,9 @@ var (

 func initLinks() {
 	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
 	apiLinks = [][2]string{
 		{path.Join(pathPrefix, "api/v1/groups"), "list all loaded groups and rules"},
 		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
@@ -33,9 +37,10 @@ func initLinks() {
 		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
 	}
 	navItems = []tpl.NavItem{
-		{Name: "vmalert", Url: pathPrefix},
+		{Name: "vmalert", Url: path.Join(pathPrefix, "/")},
 		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
 		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
+		{Name: "Notifiers", Url: path.Join(pathPrefix, "notifiers")},
 		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
 	}
 }
@@ -49,6 +54,11 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		initLinks()
 	})

+	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
+
 	switch r.URL.Path {
 	case "/":
 		if r.Method != "GET" {
@@ -57,18 +67,21 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteWelcome(w)
 		return true
 	case "/alerts":
-		WriteListAlerts(w, rh.groupAlerts())
+		WriteListAlerts(w, pathPrefix, rh.groupAlerts())
 		return true
 	case "/groups":
 		WriteListGroups(w, rh.groups())
 		return true
+	case "/notifiers":
+		WriteListTargets(w, notifier.GetTargets())
+		return true
 	case "/api/v1/groups":
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
@@ -77,7 +90,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -102,13 +115,13 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
 				return true
 			}
-			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.Header().Set("Content-Type", "application/json")
 			w.Write(data)
 			return true
 		}

 		// <groupID>/<alertID>/status
-		WriteAlert(w, alert)
+		WriteAlert(w, pathPrefix, alert)
 		return true
 	}
 }
--- a/app/vmalert/web.qtpl
+++ b/app/vmalert/web.qtpl
@@ -3,8 +3,10 @@
 {% import (
    "time"
    "sort"
+    "path"

    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 ) %}


@@ -54,10 +56,10 @@
                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
                <p class="fs-6 fw-lighter">{%s g.File %}</p>
-                {% if len(g.ExtraFilterLabels) > 0 %}
-                    <div class="fs-6 fw-lighter">Extra filter labels
-                    {% for k, v := range g.ExtraFilterLabels %}
-                            <span class="float-left badge bg-primary">{%s k %}={%s v %}</span>
+                {% if len(g.Params) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra params
+                    {% for _, param := range g.Params %}
+                            <span class="float-left badge bg-primary">{%s param %}</span>
                    {% endfor %}
                    </div>
                {% endif %}
@@ -119,7 +121,7 @@
 {% endfunc %}


-{% func ListAlerts(groupAlerts []GroupAlerts) %}
+{% func ListAlerts(pathPrefix string, groupAlerts []GroupAlerts) %}
    {%= tpl.Header("Alerts", navItems) %}
    {% if len(groupAlerts) > 0 %}
         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
@@ -184,7 +186,7 @@
                                </td>
                                <td>{%s ar.Value %}</td>
                                <td>
-                                    <a href="/{%s g.ID %}/{%s ar.ID %}/status">Details</a>
+                                    <a href="{%s path.Join(pathPrefix, g.ID, ar.ID, "status") %}">Details</a>
                                </td>
                            </tr>
                        {% endfor %}
@@ -205,7 +207,63 @@

 {% endfunc %}

-{% func Alert(alert *APIAlert) %}
+{% func ListTargets(targets map[notifier.TargetType][]notifier.Target) %}
+    {%= tpl.Header("Notifiers", navItems) %}
+    {% if len(targets) > 0 %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+
+         {%code
+             var keys []string
+             for key := range targets {
+                 keys = append(keys, string(key))
+             }
+             sort.Strings(keys)
+         %}
+
+         {% for i := range keys  %}
+           {%code typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
+                count := len(ns)
+            %}
+           <div class="group-heading data-bs-target="rules-{%s typeK %}">
+             <span class="anchor" id="notifiers-{%s typeK %}"></span>
+             <a href="#notifiers-{%s typeK %}">{%s typeK %} ({%d count %})</a>
+         </div>
+         <div class="collapse show" id="notifiers-{%s typeK %}">
+             <table class="table table-striped table-hover table-sm">
+                 <thead>
+                     <tr>
+                         <th scope="col">Labels</th>
+                         <th scope="col">Address</th>
+                     </tr>
+                 </thead>
+                 <tbody>
+                 {% for _, n := range ns %}
+                     <tr>
+                         <td>
+                              {% for _, l := range n.Labels %}
+                                      <span class="ms-1 badge bg-primary">{%s l.Name %}={%s l.Value %}</span>
+                              {% endfor %}
+                          </td>
+                         <td>{%s n.Notifier.Addr() %}</td>
+                     </tr>
+                 {% endfor %}
+              </tbody>
+             </table>
+         </div>
+     {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func Alert(pathPrefix string, alert *APIAlert) %}
    {%= tpl.Header("", navItems) %}
    {%code
        var labelKeys []string
@@ -272,7 +330,7 @@
          Group
        </div>
        <div class="col">
-           <a target="_blank" href="/groups#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
+           <a target="_blank" href="{%s path.Join(pathPrefix,"groups") %}#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
        </div>
      </div>
    </div>
--- a/app/vmalert/web.qtpl.go
+++ b/app/vmalert/web.qtpl.go
--- a/app/vmalert/web_types.go
+++ b/app/vmalert/web_types.go
@@ -23,16 +23,16 @@ type APIAlert struct {

 // APIGroup represents Group for WEB view
 type APIGroup struct {
-	Name              string             `json:"name"`
-	Type              string             `json:"type"`
-	ID                string             `json:"id"`
-	File              string             `json:"file"`
-	Interval          string             `json:"interval"`
-	Concurrency       int                `json:"concurrency"`
-	ExtraFilterLabels map[string]string  `json:"extra_filter_labels"`
-	Labels            map[string]string  `json:"labels,omitempty"`
-	AlertingRules     []APIAlertingRule  `json:"alerting_rules"`
-	RecordingRules    []APIRecordingRule `json:"recording_rules"`
+	Name           string             `json:"name"`
+	Type           string             `json:"type"`
+	ID             string             `json:"id"`
+	File           string             `json:"file"`
+	Interval       string             `json:"interval"`
+	Concurrency    int                `json:"concurrency"`
+	Params         []string           `json:"params"`
+	Labels         map[string]string  `json:"labels,omitempty"`
+	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
+	RecordingRules []APIRecordingRule `json:"recording_rules"`
 }

 // APIAlertingRule represents AlertingRule for WEB view
--- a/app/vmauth/Makefile
+++ b/app/vmauth/Makefile
@@ -27,6 +27,15 @@ vmauth-ppc64le-prod:
 vmauth-386-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-386

+vmauth-darwin-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-amd64
+
+vmauth-darwin-arm64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-arm64
+
+vmauth-windows-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-windows-amd64
+
 package-vmauth:
 	APP_NAME=vmauth $(MAKE) package-via-docker

@@ -80,6 +89,3 @@ vmauth-pure:

 vmauth-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmauth $(MAKE) app-local-windows-with-goarch
-
-vmauth-windows-amd64-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-windows-amd64
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -3,7 +3,7 @@
 `vmauth` is a simple auth proxy, router and [load balancer](#load-balancing) for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
 It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) and `Bearer token` is supported),
 matches them against configs pointed by [-auth.config](#auth-config) command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
-
+The `-auth.config` can point to either local file or to http url.

 ## Quick start

@@ -26,12 +26,10 @@ Pass `-help` to `vmauth` in order to see all the supported command-line flags wi
 Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML,
 accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.com/vmgateway.html).

-
 ## Load balancing

 Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).

-
 ## Auth config

 `-auth.config` is represented in the following simple `yml` format:
@@ -43,6 +41,7 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl
 users:
  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
  url_prefix: "http://localhost:8428"

@@ -123,9 +122,10 @@ users:
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.

-
 ## Security

+It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.
+
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:

 ```
@@ -141,21 +141,27 @@ Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termi

 It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.

-
 ## Monitoring

 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.

+`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+
+```yml
+users:
+- username: "secret_user"
+  name: "foobar"
+  # other config options here
+```

 ## How to build from sources

 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.

-
 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmauth` binary and puts it into the `bin` folder.

@@ -178,28 +184,34 @@ by setting it via `<ROOT_IMAGE>` environment variable. For example, the followin
 ROOT_IMAGE=scratch make package-vmauth
 ```

-
 ## Profiling

 `vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):

-* Memory profile. It can be collected with the following command:
+* Memory profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<vmauth-host>:8427/debug/pprof/heap > mem.pprof
+curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof
 ```

-* CPU profile. It can be collected with the following command:
+</div>
+
+* CPU profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">

 ```bash
-curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
+curl http://0.0.0.0:8427/debug/pprof/profile > cpu.pprof
 ```

+</div>
+
 The command for collecting CPU profile waits for 30 seconds before returning.

 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

-
 ## Advanced usage

 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
@@ -212,13 +224,15 @@ vmauth authenticates and authorizes incoming requests and proxies them to Victor
 See the docs at https://docs.victoriametrics.com/vmauth.html .

  -auth.config string
-    	Path to auth config. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+    	Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
  -enableTCP6
    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
  -envflag.enable
    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
  -envflag.prefix string
    	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
  -fs.disableMmap
    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
  -http.connTimeout duration
@@ -240,7 +254,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
  -httpListenAddr string
    	TCP address to listen for http connections (default ":8427")
  -logInvalidAuthTokens
-    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmagent_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
+    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
  -loggerDisableTimestamps
    	Whether to disable writing timestamps in logs
  -loggerErrorsPerSecondLimit int
@@ -263,17 +277,17 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
  -memory.allowedPercent float
    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
  -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -reloadAuthKey string
    	Auth key for /-/reload http endpoint. It must be passed as authKey=...
  -tls
    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
  -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
  -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
  -version
    	Show VictoriaMetrics version
 ```
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"regexp"
@@ -14,6 +13,7 @@ import (
 	"sync/atomic"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,8 +21,8 @@ import (
 )

 var (
-	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://docs.victoriametrics.com/vmauth.html "+
-		"for details on the format of this auth config")
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
 )

 // AuthConfig represents auth config.
@@ -32,6 +32,7 @@ type AuthConfig struct {

 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
+	Name        string     `yaml:"name,omitempty"`
 	BearerToken string     `yaml:"bearer_token,omitempty"`
 	Username    string     `yaml:"username,omitempty"`
 	Password    string     `yaml:"password,omitempty"`
@@ -236,9 +237,9 @@ var authConfigWG sync.WaitGroup
 var stopCh chan struct{}

 func readAuthConfig(path string) (map[string]*UserInfo, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := fs.ReadFileOrHTTP(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read %q: %w", path, err)
+		return nil, err
 	}
 	m, err := parseAuthConfig(data)
 	if err != nil {
@@ -275,9 +276,12 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if byUsername[ui.Username] {
 			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
 		}
-		authToken := getAuthToken(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[authToken] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", authToken, ui.BearerToken, ui.Username)
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+		}
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
 		}
 		if ui.URLPrefix != nil {
 			if err := ui.URLPrefix.sanitize(); err != nil {
@@ -299,21 +303,41 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
 		if ui.BearerToken != "" {
+			name := "bearer_token"
+			if ui.Name != "" {
+				name = ui.Name
+			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
-			ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total{username="bearer_token"}`)
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byBearerToken[ui.BearerToken] = true
 		}
 		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+			name := ui.Username
+			if ui.Name != "" {
+				name = ui.Name
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byUsername[ui.Username] = true
 		}
-		byAuthToken[authToken] = ui
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }

+func getAuthTokens(bearerToken, username, password string) (string, string) {
+	if bearerToken != "" {
+		// Accept the bearerToken as Basic Auth username with empty password
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
+	}
+	at := getAuthToken("", username, password)
+	return at, at
+}
+
 func getAuthToken(bearerToken, username, password string) string {
 	if bearerToken != "" {
 		return "Bearer " + bearerToken
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@@ -290,6 +290,32 @@ users:
 				},
 			},
 		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMap: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+		},
 	})
 }

--- a/app/vmauth/example_config.yml
+++ b/app/vmauth/example_config.yml
@@ -4,6 +4,7 @@
 users:
  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
  url_prefix: "http://localhost:8428"

--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"sync"
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -108,7 +109,7 @@ func proxyRequest(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	reverseProxy.ServeHTTP(w, r)
+	getReverseProxy().ServeHTTP(w, r)
 }

 var (
@@ -117,29 +118,42 @@ var (
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
 )

-var reverseProxy = &httputil.ReverseProxy{
-	Director: func(r *http.Request) {
-		targetURL := r.Header.Get("vm-target-url")
-		target, err := url.Parse(targetURL)
-		if err != nil {
-			logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
-		}
-		r.URL = target
-	},
-	Transport: func() *http.Transport {
-		tr := http.DefaultTransport.(*http.Transport).Clone()
-		// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
-		tr.DisableCompression = true
-		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
-		tr.ForceAttemptHTTP2 = false
-		tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
-		if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
-			tr.MaxIdleConns = tr.MaxIdleConnsPerHost
-		}
-		return tr
-	}(),
-	FlushInterval: time.Second,
-	ErrorLog:      logger.StdErrorLogger(),
+var (
+	reverseProxy     *httputil.ReverseProxy
+	reverseProxyOnce sync.Once
+)
+
+func getReverseProxy() *httputil.ReverseProxy {
+	reverseProxyOnce.Do(initReverseProxy)
+	return reverseProxy
+}
+
+// initReverseProxy must be called after flag.Parse(), since it uses command-line flags.
+func initReverseProxy() {
+	reverseProxy = &httputil.ReverseProxy{
+		Director: func(r *http.Request) {
+			targetURL := r.Header.Get("vm-target-url")
+			target, err := url.Parse(targetURL)
+			if err != nil {
+				logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
+			}
+			r.URL = target
+		},
+		Transport: func() *http.Transport {
+			tr := http.DefaultTransport.(*http.Transport).Clone()
+			// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+			tr.DisableCompression = true
+			// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+			tr.ForceAttemptHTTP2 = false
+			tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
+			if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+				tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+			}
+			return tr
+		}(),
+		FlushInterval: time.Second,
+		ErrorLog:      logger.StdErrorLogger(),
+	}
 }

 func usage() {
--- a/app/vmbackup/Makefile
+++ b/app/vmbackup/Makefile
@@ -27,6 +27,12 @@ vmbackup-ppc64le-prod:
 vmbackup-386-prod:
 	APP_NAME=vmbackup $(MAKE) app-via-docker-386

+vmbackup-darwin-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-darwin-amd64
+
+vmbackup-darwin-arm64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-darwin-arm64
+
 package-vmbackup:
 	APP_NAME=vmbackup $(MAKE) package-via-docker

--- a/app/vmbackup/README.md
+++ b/app/vmbackup/README.md
@@ -6,7 +6,7 @@ Supported storage systems for backups:

 * [GCS](https://cloud.google.com/storage/). Example: `gs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/en/pacific/radosgw/s3/) or [Swift](https://platform.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`

 `vmbackup` supports incremental and full backups. Incremental backups are created automatically if the destination path already contains data from the previous backup.
@@ -185,12 +185,32 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
  -dst string
    	Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
  -envflag.enable
    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
  -envflag.prefix string
    	Prefix for environment variables if -envflag.enable is set
  -fs.disableMmap
    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address for exporting metrics at /metrics page (default ":8420")
  -loggerDisableTimestamps
    	Whether to disable writing timestamps in logs
  -loggerErrorsPerSecondLimit int
@@ -213,8 +233,14 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
  -memory.allowedPercent float
    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -origin string
    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -s3ForcePathStyle
+    	Prefixing endpoint with bucket name when set false, true by default. (default true)
  -snapshot.createURL string
    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
  -snapshot.deleteURL string
@@ -223,6 +249,12 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
    	Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set
  -storageDataPath string
    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
  -version
    	Show VictoriaMetrics version
 ```
@@ -235,7 +267,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmbackup` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmbackup` binary and puts it into the `bin` folder.

--- a/app/vmbackup/main.go
+++ b/app/vmbackup/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
@@ -14,10 +15,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )

 var (
+	httpListenAddr    = flag.String("httpListenAddr", ":8420", "TCP address for exporting metrics at /metrics page")
 	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
 	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set")
 	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+
@@ -70,6 +73,8 @@ func main() {
 		}()
 	}

+	go httpserver.Serve(*httpListenAddr, nil)
+
 	srcFS, err := newSrcFS()
 	if err != nil {
 		logger.Fatalf("%s", err)
@@ -94,6 +99,13 @@ func main() {
 	srcFS.MustStop()
 	dstFS.MustStop()
 	originFS.MustStop()
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }

 func usage() {
--- a/app/vmbackupmanager/README.md
+++ b/app/vmbackupmanager/README.md
@@ -1,6 +1,6 @@
 ## vmbackupmanager

-***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
+***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/products/enterprise/). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***

 The VictoriaMetrics backup manager automates regular backup procedures. It supports the following backup intervals: **hourly**, **daily**, **weekly** and **monthly**. Multiple backup intervals may be configured simultaneously. I.e. the backup manager creates hourly backups every hour, while it creates daily backups every day, etc. Backup manager must have read access to the storage data, so best practice is to install it on the same machine (or as a sidecar) where the storage node is installed.
 The backup service makes a backup every hour and puts it to the latest folder and then copies data to the folders which represent the backup intervals (hourly, daily, weekly and monthly)
--- a/app/vmctl/Makefile
+++ b/app/vmctl/Makefile
@@ -27,6 +27,15 @@ vmctl-ppc64le-prod:
 vmctl-386-prod:
 	APP_NAME=vmctl $(MAKE) app-via-docker-386

+vmctl-darwin-amd64-prod:
+	APP_NAME=vmctl $(MAKE) app-via-docker-darwin-amd64
+
+vmctl-darwin-arm64-prod:
+	APP_NAME=vmctl $(MAKE) app-via-docker-darwin-arm64
+
+vmctl-windows-amd64-prod:
+	APP_NAME=vmctl $(MAKE) app-via-docker-windows-amd64
+
 package-vmctl:
 	APP_NAME=vmctl $(MAKE) package-via-docker

@@ -75,5 +84,3 @@ vmctl-pure:
 vmctl-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmctl $(MAKE) app-local-windows-with-goarch

-vmctl-windows-amd64-prod:
-	APP_NAME=vmctl $(MAKE) app-via-docker-windows-amd64
--- a/app/vmctl/README.md
+++ b/app/vmctl/README.md
@@ -560,6 +560,15 @@ results such as `average`, `rate`, etc.
 If multiple labels needs to be added, set flag for each label, for example, `--vm-extra-label label1=value1 --vm-extra-label label2=value2`.
 If timeseries already have label, that must be added with `--vm-extra-label` flag, flag has priority and will override label value from timeseries.

+### Rate limiting
+
+Limiting the rate of data transfer could help to reduce pressure on disk or on destination database.
+The rate limit may be set in bytes-per-second via `--vm-rate-limit` flag.
+
+Please note, you can also use [vmagent](https://docs.victoriametrics.com/vmagent.html) 
+as a proxy between `vmctl` and destination with `-remoteWrite.rateLimit` flag enabled.
+
+
 ## How to build

 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmctl` is located in `vmutils-*` archives there.
@@ -567,7 +576,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmctl` binary and puts it into the `bin` folder.

@@ -596,7 +605,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 #### Development ARM build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmctl-arm` or `make vmctl-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmctl-arm` or `vmctl-arm64` binary respectively and puts it into the `bin` folder.

--- a/app/vmctl/flags.go
+++ b/app/vmctl/flags.go
@@ -7,7 +7,8 @@ import (
 )

 const (
-	globalSilent = "s"
+	globalSilent  = "s"
+	globalVerbose = "verbose"
 )

 var (
@@ -17,6 +18,11 @@ var (
 			Value: false,
 			Usage: "Whether to run in silent mode. If set to true no confirmation prompts will appear.",
 		},
+		&cli.BoolFlag{
+			Name:  globalVerbose,
+			Value: false,
+			Usage: "Whether to enable verbosity in logs output.",
+		},
 	}
 )

@@ -30,7 +36,10 @@ const (
 	vmBatchSize          = "vm-batch-size"
 	vmSignificantFigures = "vm-significant-figures"
 	vmRoundDigits        = "vm-round-digits"
-	vmExtraLabel         = "vm-extra-label"
+
+	// also used in vm-native
+	vmExtraLabel = "vm-extra-label"
+	vmRateLimit  = "vm-rate-limit"
 )

 var (
@@ -95,6 +104,11 @@ var (
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
 				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
+		&cli.Int64Flag{
+			Name: vmRateLimit,
+			Usage: "Optional data transfer rate limit in bytes per second.\n" +
+				"By default the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+		},
 	}
 )

@@ -155,7 +169,7 @@ var (
 		&cli.IntFlag{
 			Name:  otsdbQueryLimit,
 			Usage: "Result limit on meta queries to OpenTSDB (affects both metric name and tag value queries, recommended to use a value exceeding your largest series)",
-			Value: 100e3,
+			Value: 100e6,
 		},
 		&cli.BoolFlag{
 			Name:  otsdbMsecsTime,
@@ -354,6 +368,11 @@ var (
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
 				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
+		&cli.Int64Flag{
+			Name: vmRateLimit,
+			Usage: "Optional data transfer rate limit in bytes per second.\n" +
+				"By default the rate limit is disabled. It can be useful for limiting load on source or destination databases.",
+		},
 	}
 )

--- a/app/vmctl/influx.go
+++ b/app/vmctl/influx.go
@@ -30,7 +30,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }

-func (ip *influxProcessor) run(silent bool) error {
+func (ip *influxProcessor) run(silent, verbose bool) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -70,7 +70,7 @@ func (ip *influxProcessor) run(silent bool) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("influx error: %s", infErr)
 		case vmErr := <-ip.im.Errors():
-			return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 		case seriesCh <- s:
 		}
 	}
@@ -80,7 +80,9 @@ func (ip *influxProcessor) run(silent bool) error {
 	ip.im.Close()
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	bar.Finish()
 	log.Println("Import finished!")
--- a/app/vmctl/limiter/limiter.go
+++ b/app/vmctl/limiter/limiter.go
@@ -0,0 +1,53 @@
+package limiter
+
+import (
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+)
+
+// NewLimiter creates a Limiter object
+// for the given perSecondLimit
+func NewLimiter(perSecondLimit int64) *Limiter {
+	return &Limiter{perSecondLimit: perSecondLimit}
+}
+
+// Limiter controls the amount of budget
+// that can be spent according to configured perSecondLimit
+type Limiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+}
+
+// Register blocks for amount of time
+// needed to process the given dataLen according
+// to the configured perSecondLimit.
+func (l *Limiter) Register(dataLen int) {
+	limit := l.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	for l.budget <= 0 {
+		if d := time.Until(l.deadline); d > 0 {
+			t := timerpool.Get(d)
+			<-t.C
+			timerpool.Put(t)
+		}
+		l.budget += limit
+		l.deadline = time.Now().Add(time.Second)
+	}
+	l.budget -= int64(dataLen)
+}
--- a/app/vmctl/limiter/writer.go
+++ b/app/vmctl/limiter/writer.go
@@ -0,0 +1,37 @@
+package limiter
+
+import (
+	"io"
+)
+
+// NewWriteLimiter creates a new WriteLimiter object
+// for the give writer and Limiter.
+func NewWriteLimiter(w io.Writer, limiter *Limiter) *WriteLimiter {
+	return &WriteLimiter{
+		writer:  w,
+		limiter: limiter,
+	}
+}
+
+// WriteLimiter limits the amount of bytes written
+// per second via Write() method.
+// Must be created via NewWriteLimiter.
+type WriteLimiter struct {
+	writer  io.Writer
+	limiter *Limiter
+}
+
+// Close implements io.Closer
+// also calls Close for wrapped io.WriteCloser
+func (wl *WriteLimiter) Close() error {
+	if c, ok := wl.writer.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
+}
+
+// Write implements io.Writer
+func (wl *WriteLimiter) Write(p []byte) (n int, err error) {
+	wl.limiter.Register(len(p))
+	return wl.writer.Write(p)
+}
--- a/app/vmctl/main.go
+++ b/app/vmctl/main.go
@@ -18,6 +18,11 @@ import (
 )

 func main() {
+	var (
+		err      error
+		importer *vm.Importer
+	)
+
 	start := time.Now()
 	app := &cli.App{
 		Name:    "vmctl",
@@ -53,7 +58,7 @@ func main() {
 					}

 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency))
-					return otsdbProcessor.run(c.Bool(globalSilent))
+					return otsdbProcessor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -82,14 +87,14 @@ func main() {
 					}

 					vmCfg := initConfigVM(c)
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}

 					processor := newInfluxProcessor(influxClient, importer,
 						c.Int(influxConcurrency), c.String(influxMeasurementFieldSeparator))
-					return processor.run(c.Bool(globalSilent))
+					return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -100,7 +105,7 @@ func main() {
 					fmt.Println("Prometheus import mode")

 					vmCfg := initConfigVM(c)
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -123,7 +128,7 @@ func main() {
 						im: importer,
 						cc: c.Int(promConcurrency),
 					}
-					return pp.run(c.Bool(globalSilent))
+					return pp.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -138,6 +143,7 @@ func main() {
 					}

 					p := vmNativeProcessor{
+						rateLimit: c.Int64(vmRateLimit),
 						filter: filter{
 							match:     c.String(vmNativeFilterMatch),
 							timeStart: c.String(vmNativeFilterTimeStart),
@@ -166,12 +172,14 @@ func main() {
 	go func() {
 		<-c
 		fmt.Println("\r- Execution cancelled")
-		os.Exit(0)
+		if importer != nil {
+			importer.Close()
+		}
 	}()

-	err := app.Run(os.Args)
+	err = app.Run(os.Args)
 	if err != nil {
-		log.Fatal(err)
+		log.Println(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
 }
@@ -188,5 +196,6 @@ func initConfigVM(c *cli.Context) vm.Config {
 		SignificantFigures: c.Int(vmSignificantFigures),
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
+		RateLimit:          c.Int64(vmRateLimit),
 	}
 }
--- a/app/vmctl/opentsdb.go
+++ b/app/vmctl/opentsdb.go
@@ -35,7 +35,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int) *otsdb
 	}
 }

-func (op *otsdbProcessor) run(silent bool) error {
+func (op *otsdbProcessor) run(silent, verbose bool) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -111,7 +111,7 @@ func (op *otsdbProcessor) run(silent bool) error {
 					case otsdbErr := <-errCh:
 						return fmt.Errorf("opentsdb error: %s", otsdbErr)
 					case vmErr := <-op.im.Errors():
-						return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+						return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 					case seriesCh <- queryObj{
 						Tr: tr, StartTime: startTime,
 						Series: series, Rt: opentsdb.RetentionMeta{
@@ -133,7 +133,9 @@ func (op *otsdbProcessor) run(silent bool) error {
 	}
 	op.im.Close()
 	for vmErr := range op.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	log.Println("Import finished!")
 	log.Print(op.im.Stats())
@@ -143,7 +145,7 @@ func (op *otsdbProcessor) run(silent bool) error {
 func (op *otsdbProcessor) do(s queryObj) error {
 	start := s.StartTime - s.Tr.Start
 	end := s.StartTime - s.Tr.End
-	data, err := op.oc.GetData(s.Series, s.Rt, start, end)
+	data, err := op.oc.GetData(s.Series, s.Rt, start, end, op.oc.MsecsTime)
 	if err != nil {
 		return fmt.Errorf("failed to collect data for %v in %v:%v :: %v", s.Series, s.Rt, s.Tr, err)
 	}
--- a/app/vmctl/opentsdb/opentsdb.go
+++ b/app/vmctl/opentsdb/opentsdb.go
@@ -1,7 +1,6 @@
 package opentsdb

 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -47,6 +46,7 @@ type Client struct {
 	Filters    []string
 	Normalize  bool
 	HardTS     int64
+	MsecsTime  bool
 }

 // Config contains fields required
@@ -83,12 +83,20 @@ type MetaResults struct {
 // Meta A meta object about a metric
 // only contain the tags/etc. and no data
 type Meta struct {
-	//tsuid  string
 	Metric string            `json:"metric"`
 	Tags   map[string]string `json:"tags"`
+	//tsuid  string
 }

-// Metric holds the time series data
+// OtsdbMetric is a single series in OpenTSDB's returned format
+type OtsdbMetric struct {
+	Metric        string
+	Tags          map[string]string
+	AggregateTags []string
+	Dps           map[int64]float64
+}
+
+// Metric holds the time series data in VictoriaMetrics format
 type Metric struct {
 	Metric     string
 	Tags       map[string]string
@@ -96,83 +104,6 @@ type Metric struct {
 	Values     []float64
 }

-// ExpressionOutput contains results from actual data queries
-type ExpressionOutput struct {
-	Outputs []qoObj     `json:"outputs"`
-	Query   interface{} `json:"query"`
-}
-
-// QoObj contains actual timeseries data from the returned data query
-type qoObj struct {
-	ID    string      `json:"id"`
-	Alias string      `json:"alias"`
-	Dps   [][]float64 `json:"dps"`
-	//dpsMeta interface{}
-	//meta    interface{}
-}
-
-// Expression objects format our data queries
-/*
-All of the following structs are to build a OpenTSDB expression object
-*/
-type Expression struct {
-	Time    timeObj     `json:"time"`
-	Filters []filterObj `json:"filters"`
-	Metrics []metricObj `json:"metrics"`
-	// this just needs to be an empty object, so the value doesn't matter
-	Expressions []int       `json:"expressions"`
-	Outputs     []outputObj `json:"outputs"`
-}
-
-type timeObj struct {
-	Start       int64  `json:"start"`
-	End         int64  `json:"end"`
-	Aggregator  string `json:"aggregator"`
-	Downsampler dSObj  `json:"downsampler"`
-}
-
-type dSObj struct {
-	Interval   string  `json:"interval"`
-	Aggregator string  `json:"aggregator"`
-	FillPolicy fillObj `json:"fillPolicy"`
-}
-
-type fillObj struct {
-	// we'll always hard-code to NaN here, so we don't need value
-	Policy string `json:"policy"`
-}
-
-type filterObj struct {
-	Tags []tagObj `json:"tags"`
-	ID   string   `json:"id"`
-}
-
-type tagObj struct {
-	Type    string `json:"type"`
-	Tagk    string `json:"tagk"`
-	Filter  string `json:"filter"`
-	GroupBy bool   `json:"groupBy"`
-}
-
-type metricObj struct {
-	ID         string  `json:"id"`
-	Metric     string  `json:"metric"`
-	Filter     string  `json:"filter"`
-	FillPolicy fillObj `json:"fillPolicy"`
-}
-
-type outputObj struct {
-	ID    string `json:"id"`
-	Alias string `json:"alias"`
-}
-
-/* End expression object structs */
-
-var (
-	exprOutput     = outputObj{ID: "a", Alias: "query"}
-	exprFillPolicy = fillObj{Policy: "nan"}
-)
-
 // FindMetrics discovers all metrics that OpenTSDB knows about (given a filter)
 // e.g. /api/suggest?type=metrics&q=system&max=100000
 func (c Client) FindMetrics(q string) ([]string, error) {
@@ -221,74 +152,119 @@ func (c Client) FindSeries(metric string) ([]Meta, error) {
 }

 // GetData actually retrieves data for a series at a specified time range
-func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (Metric, error) {
+// e.g. /api/query?start=1&end=200&m=sum:1m-avg-none:system.load5{host=host1}
+func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64, mSecs bool) (Metric, error) {
 	/*
-		Here we build the actual exp query we'll send to OpenTSDB
-
-		This is comprised of a number of different settings. We hard-code
-		a few to simplify the JSON object creation.
-		There are examples queries available, so not too much detail here...
+		First, build our tag string.
+		It's literally just key=value,key=value,...
 	*/
-	expr := Expression{}
-	expr.Outputs = []outputObj{exprOutput}
-	expr.Metrics = append(expr.Metrics, metricObj{ID: "a", Metric: series.Metric,
-		Filter: "f1", FillPolicy: exprFillPolicy})
-	expr.Time = timeObj{Start: start, End: end, Aggregator: rt.FirstOrder,
-		Downsampler: dSObj{Interval: rt.AggTime,
-			Aggregator: rt.SecondOrder,
-			FillPolicy: exprFillPolicy}}
-	var TagList []tagObj
+	tagStr := ""
 	for k, v := range series.Tags {
-		/*
-			every tag should be a literal_or because that's the closest to a full "==" that
-			this endpoint allows for
-		*/
-		TagList = append(TagList, tagObj{Type: "literal_or", Tagk: k,
-			Filter: v, GroupBy: true})
-	}
-	expr.Filters = append(expr.Filters, filterObj{ID: "f1", Tags: TagList})
-	// "expressions" is required in the query object or we get a 5xx, so force it to exist
-	expr.Expressions = make([]int, 0)
-	inputData, err := json.Marshal(expr)
-	if err != nil {
-		return Metric{}, fmt.Errorf("failed to marshal query JSON %s", err)
+		tagStr += fmt.Sprintf("%s=%s,", k, v)
 	}
+	// obviously we don't want trailing commas...
+	tagStr = strings.Trim(tagStr, ",")

-	q := fmt.Sprintf("%s/api/query/exp", c.Addr)
-	resp, err := http.Post(q, "application/json", bytes.NewBuffer(inputData))
+	/*
+		The aggregation policy should already be somewhat formatted:
+		FirstOrder (e.g. sum/avg/max/etc.)
+		SecondOrder (e.g. sum/avg/max/etc.)
+		AggTime	(e.g. 1m/10m/1d/etc.)
+		This will build into m=<FirstOrder>:<AggTime>-<SecondOrder>-none:
+		Or an example: m=sum:1m-avg-none
+	*/
+	aggPol := fmt.Sprintf("%s:%s-%s-none", rt.FirstOrder, rt.AggTime, rt.SecondOrder)
+
+	/*
+		Our actual query string:
+		Start and End are just timestamps
+		We then add the aggregation policy, the metric, and the tag set
+	*/
+	queryStr := fmt.Sprintf("start=%v&end=%v&m=%s:%s{%s}", start, end, aggPol,
+		series.Metric, tagStr)
+
+	q := fmt.Sprintf("%s/api/query?%s", c.Addr, queryStr)
+	resp, err := http.Get(q)
 	if err != nil {
 		return Metric{}, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
+	/*
+		There are three potential failures here, none of which should kill the entire
+		migration run:
+		1. bad response code
+		2. failure to read response body
+		3. bad format of response body
+	*/
 	if resp.StatusCode != 200 {
-		return Metric{}, fmt.Errorf("Bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		log.Println(fmt.Sprintf("bad response code from OpenTSDB query %v for %q...skipping", resp.StatusCode, q))
+		return Metric{}, nil
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return Metric{}, fmt.Errorf("could not retrieve series data from %q: %s", q, err)
+		log.Println("couldn't read response body from OpenTSDB query...skipping")
+		return Metric{}, nil
 	}
-	var output ExpressionOutput
+	var output []OtsdbMetric
 	err = json.Unmarshal(body, &output)
 	if err != nil {
-		return Metric{}, fmt.Errorf("failed to unmarshal response from %q: %s", q, err)
+		log.Println(fmt.Sprintf("couldn't marshall response body from OpenTSDB query (%s)...skipping", body))
+		return Metric{}, nil
 	}
-	if len(output.Outputs) < 1 {
+	/*
+		We expect results to look like:
+		[
+		  {
+			"metric": "zfs_filesystem.available",
+			"tags": {
+			  "rack": "6",
+			  "replica": "1",
+			  "host": "c7-bfyii-115",
+			  "pool": "dattoarray",
+			  "row": "c",
+			  "dc": "us-west-3",
+			  "group": "legonode"
+			},
+			"aggregateTags": [],
+			"dps": {
+			  "1626019200": 32490602877610.668,
+			  "1626033600": 32486439014058.668
+			}
+		  }
+		]
+		There are two things that could be bad here:
+			1. There are no actual stats returned (an empty array -> [])
+			2. There are aggregate tags in the results
+		An empty array doesn't cast to a OtsdbMetric struct well, and there's no reason to try, so we should just skip it
+		Because we're trying to migrate data without transformations, seeing aggregate tags could mean
+		we're dropping series on the floor.
+
+		In all "bad" cases, we don't end the migration, we just don't process that particular message
+	*/
+	if len(output) < 1 {
 		// no results returned...return an empty object without error
 		return Metric{}, nil
 	}
+	if len(output) > 1 {
+		// multiple series returned for a single query. We can't process this right, so...
+		return Metric{}, nil
+	}
+	if len(output[0].AggregateTags) > 0 {
+		// This failure means we've suppressed potential series somehow...
+		return Metric{}, nil
+	}
 	data := Metric{}
-	data.Metric = series.Metric
-	data.Tags = series.Tags
+	data.Metric = output[0].Metric
+	data.Tags = output[0].Tags
 	/*
 		We evaluate data for correctness before formatting the actual values
 		to skip a little bit of time if the series has invalid formatting
-
-		First step is to enforce Prometheus' data model
 	*/
 	data, err = modifyData(data, c.Normalize)
 	if err != nil {
-		return Metric{}, fmt.Errorf("invalid series data from %q: %s", q, err)
+		return Metric{}, nil
 	}
+
 	/*
 		Convert data from OpenTSDB's output format ([[ts,val],[ts,val]...])
 		to VictoriaMetrics format: {"timestamps": [ts,ts,ts...], "values": [val,val,val...]}
@@ -296,9 +272,13 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (
 		can be a float64, we have to initially cast _all_ objects that way
 		then convert the timestamp back to something reasonable.
 	*/
-	for _, tsobj := range output.Outputs[0].Dps {
-		data.Timestamps = append(data.Timestamps, int64(tsobj[0]))
-		data.Values = append(data.Values, tsobj[1])
+	for ts, val := range output[0].Dps {
+		if !mSecs {
+			data.Timestamps = append(data.Timestamps, ts*1000)
+		} else {
+			data.Timestamps = append(data.Timestamps, ts)
+		}
+		data.Values = append(data.Values, val)
 	}
 	return data, nil
 }
@@ -308,9 +288,13 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (
 func NewClient(cfg Config) (*Client, error) {
 	var retentions []Retention
 	offsetPrint := int64(time.Now().Unix())
+	// convert a number of days to seconds
+	offsetSecs := cfg.Offset * 24 * 60 * 60
 	if cfg.MsecsTime {
 		// 1000000 == Nanoseconds -> Milliseconds difference
 		offsetPrint = int64(time.Now().UnixNano() / 1000000)
+		// also bump offsetSecs to milliseconds
+		offsetSecs = offsetSecs * 1000
 	}
 	if cfg.HardTS > 0 {
 		/*
@@ -318,20 +302,16 @@ func NewClient(cfg Config) (*Client, error) {
 			Just present that if it is defined
 		*/
 		offsetPrint = cfg.HardTS
-	} else if cfg.Offset > 0 {
+	} else if offsetSecs > 0 {
 		/*
-			Our "offset" is the number of days we should step
+			Our "offset" is the number of days (in seconds) we should step
 			back before starting to scan for data
 		*/
-		if cfg.MsecsTime {
-			offsetPrint = offsetPrint - (cfg.Offset * 24 * 60 * 60 * 1000)
-		} else {
-			offsetPrint = offsetPrint - (cfg.Offset * 24 * 60 * 60)
-		}
+		offsetPrint = offsetPrint - offsetSecs
 	}
 	log.Println(fmt.Sprintf("Will collect data starting at TS %v", offsetPrint))
 	for _, r := range cfg.Retentions {
-		ret, err := convertRetention(r, cfg.Offset, cfg.MsecsTime)
+		ret, err := convertRetention(r, offsetSecs, cfg.MsecsTime)
 		if err != nil {
 			return &Client{}, fmt.Errorf("Couldn't parse retention %q :: %v", r, err)
 		}
@@ -344,6 +324,7 @@ func NewClient(cfg Config) (*Client, error) {
 		Filters:    cfg.Filters,
 		Normalize:  cfg.Normalize,
 		HardTS:     cfg.HardTS,
+		MsecsTime:  cfg.MsecsTime,
 	}
 	return client, nil
 }
--- a/app/vmctl/opentsdb/parser.go
+++ b/app/vmctl/opentsdb/parser.go
@@ -87,6 +87,34 @@ func convertRetention(retention string, offset int64, msecTime bool) (Retention,
 	if len(chunks) != 3 {
 		return Retention{}, fmt.Errorf("invalid retention string: %q", retention)
 	}
+	queryLengthDuration, err := convertDuration(chunks[2])
+	if err != nil {
+		return Retention{}, fmt.Errorf("invalid ttl (second order) duration string: %q: %s", chunks[2], err)
+	}
+	// set ttl in milliseconds, unless we aren't using millisecond time in OpenTSDB...then use seconds
+	queryLength := queryLengthDuration.Milliseconds()
+	if !msecTime {
+		queryLength = queryLength / 1000
+	}
+	queryRange := queryLength
+	// bump by the offset so we don't look at empty ranges any time offset > ttl
+	queryLength += offset
+
+	// first/second order aggregations for queries defined in chunk 0...
+	aggregates := strings.Split(chunks[0], "-")
+	if len(aggregates) != 3 {
+		return Retention{}, fmt.Errorf("invalid aggregation string: %q", chunks[0])
+	}
+
+	aggTimeDuration, err := convertDuration(aggregates[1])
+	if err != nil {
+		return Retention{}, fmt.Errorf("invalid aggregation time duration string: %q: %s", aggregates[1], err)
+	}
+	aggTime := aggTimeDuration.Milliseconds()
+	if !msecTime {
+		aggTime = aggTime / 1000
+	}
+
 	rowLengthDuration, err := convertDuration(chunks[1])
 	if err != nil {
 		return Retention{}, fmt.Errorf("invalid row length (first order) duration string: %q: %s", chunks[1], err)
@@ -96,26 +124,35 @@ func convertRetention(retention string, offset int64, msecTime bool) (Retention,
 	if !msecTime {
 		rowLength = rowLength / 1000
 	}
-	ttlDuration, err := convertDuration(chunks[2])
-	if err != nil {
-		return Retention{}, fmt.Errorf("invalid ttl (second order) duration string: %q: %s", chunks[2], err)
+
+	var querySize int64
+	/*
+		The idea here is to ensure each individual query sent to OpenTSDB is *at least*
+		large enough to ensure no single query requests essentially 0 data.
+	*/
+	if rowLength > aggTime {
+		/*
+			We'll look at 2x the row size for each query we perform
+			This is a strange function, but the logic works like this:
+			1. we discover the "number" of ranges we should split the time range into
+			   This is found with queryRange / (rowLength * 4)...kind of a percentage query
+			2. we discover the actual size of each "chunk"
+			   This is second division step
+		*/
+		querySize = int64(queryRange / (queryRange / (rowLength * 4)))
+	} else {
+		/*
+			Unless the aggTime (how long a range of data we're requesting per individual point)
+			is greater than the row size. Then we'll need to use that to determine
+			how big each individual query should be
+		*/
+		querySize = int64(queryRange / (queryRange / (aggTime * 4)))
 	}
-	// set ttl in milliseconds, unless we aren't using millisecond time in OpenTSDB...then use seconds
-	ttl := ttlDuration.Milliseconds()
-	if !msecTime {
-		ttl = ttl / 1000
-	}
-	// bump by the offset so we don't look at empty ranges any time offset > ttl
-	ttl += offset
+
 	var timeChunks []TimeRange
 	var i int64
-	for i = offset; i <= ttl; i = i + rowLength {
-		timeChunks = append(timeChunks, TimeRange{Start: i + rowLength, End: i})
-	}
-	// first/second order aggregations for queries defined in chunk 0...
-	aggregates := strings.Split(chunks[0], "-")
-	if len(aggregates) != 3 {
-		return Retention{}, fmt.Errorf("invalid aggregation string: %q", chunks[0])
+	for i = offset; i <= queryLength; i = i + querySize {
+		timeChunks = append(timeChunks, TimeRange{Start: i + querySize, End: i})
 	}

 	ret := Retention{FirstOrder: aggregates[0],
--- a/app/vmctl/opentsdb/parser_test.go
+++ b/app/vmctl/opentsdb/parser_test.go
@@ -8,7 +8,7 @@ func TestConvertRetention(t *testing.T) {
 	/*
 		2592000 seconds in 30 days
 		3600 in one hour
-		2592000 / 3600 = 720 individual query "ranges" should exist, plus one because time ranges can be weird
+		2592000 / 14400 = 180 individual query "ranges" should exist, plus one because time ranges can be weird
 		First order should == "sum"
 		Second order should == "avg"
 		AggTime should == "1m"
@@ -17,8 +17,8 @@ func TestConvertRetention(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Error parsing valid retention string: %v", err)
 	}
-	if len(res.QueryRanges) != 721 {
-		t.Fatalf("Found %v query ranges. Should have found 720", len(res.QueryRanges))
+	if len(res.QueryRanges) != 181 {
+		t.Fatalf("Found %v query ranges. Should have found 181", len(res.QueryRanges))
 	}
 	if res.FirstOrder != "sum" {
 		t.Fatalf("Incorrect first order aggregation %q. Should have been 'sum'", res.FirstOrder)
--- a/app/vmctl/opentsdb/testdata/exampleOutput.json
+++ b/app/vmctl/opentsdb/testdata/exampleOutput.json
@@ -1,398 +0,0 @@
-{
-  "outputs": [
-    {
-      "id": "a",
-      "alias": "query",
-      "dps": [
-        [
-          1614099600000,
-          0.28
-        ],
-        [
-          1614099660000,
-          0.22
-        ],
-        [
-          1614099720000,
-          0.18
-        ],
-        [
-          1614099780000,
-          0.14
-        ],
-        [
-          1614099840000,
-          0.24
-        ],
-        [
-          1614099900000,
-          0.19
-        ],
-        [
-          1614099960000,
-          0.22
-        ],
-        [
-          1614100020000,
-          0.2
-        ],
-        [
-          1614100080000,
-          0.18
-        ],
-        [
-          1614100140000,
-          0.22
-        ],
-        [
-          1614100200000,
-          0.17
-        ],
-        [
-          1614100260000,
-          0.16
-        ],
-        [
-          1614100320000,
-          0.22
-        ],
-        [
-          1614100380000,
-          0.3
-        ],
-        [
-          1614100440000,
-          0.28
-        ],
-        [
-          1614100500000,
-          0.27
-        ],
-        [
-          1614100560000,
-          0.26
-        ],
-        [
-          1614100620000,
-          0.23
-        ],
-        [
-          1614100680000,
-          0.18
-        ],
-        [
-          1614100740000,
-          0.3
-        ],
-        [
-          1614100800000,
-          0.24
-        ],
-        [
-          1614100860000,
-          0.19
-        ],
-        [
-          1614100920000,
-          0.16
-        ],
-        [
-          1614100980000,
-          0.19
-        ],
-        [
-          1614101040000,
-          0.23
-        ],
-        [
-          1614101100000,
-          0.18
-        ],
-        [
-          1614101160000,
-          0.15
-        ],
-        [
-          1614101220000,
-          0.12
-        ],
-        [
-          1614101280000,
-          0.1
-        ],
-        [
-          1614101340000,
-          0.24
-        ],
-        [
-          1614101400000,
-          0.19
-        ],
-        [
-          1614101460000,
-          0.16
-        ],
-        [
-          1614101520000,
-          0.14
-        ],
-        [
-          1614101580000,
-          0.12
-        ],
-        [
-          1614101640000,
-          0.14
-        ],
-        [
-          1614101700000,
-          0.12
-        ],
-        [
-          1614101760000,
-          0.13
-        ],
-        [
-          1614101820000,
-          0.12
-        ],
-        [
-          1614101880000,
-          0.11
-        ],
-        [
-          1614101940000,
-          0.36
-        ],
-        [
-          1614102000000,
-          0.35
-        ],
-        [
-          1614102060000,
-          0.3
-        ],
-        [
-          1614102120000,
-          0.32
-        ],
-        [
-          1614102180000,
-          0.27
-        ],
-        [
-          1614102240000,
-          0.26
-        ],
-        [
-          1614102300000,
-          0.21
-        ],
-        [
-          1614102360000,
-          0.18
-        ],
-        [
-          1614102420000,
-          0.15
-        ],
-        [
-          1614102480000,
-          0.12
-        ],
-        [
-          1614102540000,
-          0.24
-        ],
-        [
-          1614102600000,
-          0.2
-        ],
-        [
-          1614102660000,
-          0.17
-        ],
-        [
-          1614102720000,
-          0.18
-        ],
-        [
-          1614102780000,
-          0.14
-        ],
-        [
-          1614102840000,
-          0.39
-        ],
-        [
-          1614102900000,
-          0.31
-        ],
-        [
-          1614102960000,
-          0.3
-        ],
-        [
-          1614103020000,
-          0.24
-        ],
-        [
-          1614103080000,
-          0.26
-        ],
-        [
-          1614103140000,
-          0.21
-        ],
-        [
-          1614103200000,
-          0.17
-        ],
-        [
-          1614103260000,
-          0.15
-        ],
-        [
-          1614103320000,
-          0.2
-        ],
-        [
-          1614103380000,
-          0.2
-        ],
-        [
-          1614103440000,
-          0.22
-        ],
-        [
-          1614103500000,
-          0.19
-        ],
-        [
-          1614103560000,
-          0.22
-        ],
-        [
-          1614103620000,
-          0.29
-        ],
-        [
-          1614103680000,
-          0.31
-        ],
-        [
-          1614103740000,
-          0.28
-        ],
-        [
-          1614103800000,
-          0.23
-        ]
-      ],
-      "dpsMeta": {
-        "firstTimestamp": 1614099600000,
-        "lastTimestamp": 1614103800000,
-        "setCount": 71,
-        "series": 1
-      },
-      "meta": [
-        {
-          "index": 0,
-          "metrics": [
-            "timestamp"
-          ]
-        },
-        {
-          "index": 1,
-          "metrics": [
-            "system.load5"
-          ],
-          "commonTags": {
-            "rack": "undef",
-            "host": "use1-mon-metrics-1",
-            "row": "undef",
-            "dc": "us-east-1",
-            "group": "monitoring"
-          },
-          "aggregatedTags": []
-        }
-      ]
-    }
-  ],
-  "query": {
-    "name": null,
-    "time": {
-      "start": "1h-ago",
-      "end": null,
-      "timezone": null,
-      "downsampler": {
-        "interval": "1m",
-        "aggregator": "avg",
-        "fillPolicy": {
-          "policy": "nan",
-          "value": "NaN"
-        }
-      },
-      "aggregator": "sum",
-      "rate": false
-    },
-    "filters": [
-      {
-        "id": "f1",
-        "tags": [
-          {
-            "tagk": "host",
-            "filter": "use1-mon-metrics-1",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "group",
-            "filter": "monitoring",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "dc",
-            "filter": "us-east-1",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "rack",
-            "filter": "undef",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "row",
-            "filter": "undef",
-            "group_by": true,
-            "type": "literal_or"
-          }
-        ],
-        "explicitTags": false
-      }
-    ],
-    "metrics": [
-      {
-        "metric": "system.load5",
-        "id": "a",
-        "filter": "f1",
-        "aggregator": null,
-        "timeOffset": null,
-        "fillPolicy": {
-          "policy": "nan",
-          "value": "NaN"
-        }
-      }
-    ],
-    "expressions": [],
-    "outputs": [
-      {
-        "id": "a",
-        "alias": "query"
-      }
-    ]
-  }
-}
--- a/app/vmctl/opentsdb/testdata/exampleQuery.json
+++ b/app/vmctl/opentsdb/testdata/exampleQuery.json
@@ -1,62 +0,0 @@
-{
-   "time": {
-       "start": "1h-ago",
-       "aggregator":"sum",
-       "downsampler": {
-           "interval": "1m",
-           "aggregator": "avg",
-           "fillPolicy": {
-               "policy": "nan"
-           }
-       }
-   },
-   "filters": [
-       {
-           "tags": [
-               {
-                   "type": "literal_or",
-                   "tagk": "host",
-                   "filter": "use1-mon-metrics-1",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "group",
-                   "filter": "monitoring",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "dc",
-                   "filter": "us-east-1",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "rack",
-                   "filter": "undef",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "row",
-                   "filter": "undef",
-                   "groupBy": true
-               }
-           ],
-           "id": "f1"
-       }
-   ],
-   "metrics": [
-       {
-           "id": "a",
-           "metric": "system.load5",
-           "filter": "f1",
-           "fillPolicy":{"policy":"nan"}
-       }
-   ],
-   "expressions": [],
-   "outputs":[
-     {"id":"a", "alias":"query"}
-   ]
-}
--- a/app/vmctl/prometheus.go
+++ b/app/vmctl/prometheus.go
@@ -25,7 +25,7 @@ type prometheusProcessor struct {
 	cc int
 }

-func (pp *prometheusProcessor) run(silent bool) error {
+func (pp *prometheusProcessor) run(silent, verbose bool) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -66,7 +66,7 @@ func (pp *prometheusProcessor) run(silent bool) error {
 			return fmt.Errorf("prometheus error: %s", promErr)
 		case vmErr := <-pp.im.Errors():
 			close(blockReadersCh)
-			return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 		case blockReadersCh <- br:
 		}
 	}
@@ -77,7 +77,9 @@ func (pp *prometheusProcessor) run(silent bool) error {
 	pp.im.Close()
 	// drain import errors channel
 	for vmErr := range pp.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	bar.Finish()
 	log.Println("Import finished!")
--- a/app/vmctl/utils.go
+++ b/app/vmctl/utils.go
@@ -23,11 +23,29 @@ func prompt(question string) bool {
 	return false
 }

-func wrapErr(vmErr *vm.ImportError) error {
+func wrapErr(vmErr *vm.ImportError, verbose bool) error {
 	var errTS string
+	var maxTS, minTS int64
 	for _, ts := range vmErr.Batch {
-		errTS += fmt.Sprintf("%s for timestamps range %d - %d\n",
-			ts.String(), ts.Timestamps[0], ts.Timestamps[len(ts.Timestamps)-1])
+		if minTS < ts.Timestamps[0] || minTS == 0 {
+			minTS = ts.Timestamps[0]
+		}
+		if maxTS < ts.Timestamps[len(ts.Timestamps)-1] {
+			maxTS = ts.Timestamps[len(ts.Timestamps)-1]
+		}
+		if verbose {
+			errTS += fmt.Sprintf("%s for timestamps range %d - %d\n",
+				ts.String(), ts.Timestamps[0], ts.Timestamps[len(ts.Timestamps)-1])
+		}
 	}
-	return fmt.Errorf("%s with error: %s", errTS, vmErr.Err)
+	var verboseMsg string
+	if !verbose {
+		verboseMsg = "(enable `--verbose` output to get more details)"
+	}
+	if vmErr.Err == nil {
+		return fmt.Errorf("%s\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
+			vmErr.Err, minTS, maxTS, verboseMsg, errTS)
+	}
+	return fmt.Errorf("%s\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
+		vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 }
--- a/app/vmctl/vm/vm.go
+++ b/app/vmctl/vm/vm.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"

+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )

@@ -47,6 +48,9 @@ type Config struct {
 	RoundDigits int
 	// ExtraLabels that will be added to all imported series. Must be in label=value format.
 	ExtraLabels []string
+	// RateLimit defines a data transfer speed in bytes per second.
+	// Is applied to each worker (see Concurrency) independently.
+	RateLimit int64
 }

 // Importer performs insertion of timeseries
@@ -63,6 +67,8 @@ type Importer struct {
 	input  chan *TimeSeries
 	errors chan *ImportError

+	rl *limiter.Limiter
+
 	wg   sync.WaitGroup
 	once sync.Once

@@ -123,6 +129,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		compress:   cfg.Compress,
 		user:       cfg.User,
 		password:   cfg.Password,
+		rl:         limiter.NewLimiter(cfg.RateLimit),
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
@@ -149,9 +156,11 @@ func NewImporter(cfg Config) (*Importer, error) {
 // ImportError is type of error generated
 // in case of unsuccessful import request
 type ImportError struct {
-	// The batch of timeseries that failed
+	// The batch of timeseries processed by importer at the moment
 	Batch []*TimeSeries
 	// The error that appeared during insert
+	// If err is nil - no error happened and Batch
+	// Is the latest delivered Batch.
 	Err error
 }

@@ -180,12 +189,13 @@ func (im *Importer) startWorker(batchSize, significantFigures, roundDigits int)
 	for {
 		select {
 		case <-im.close:
-			if err := im.Import(batch); err != nil {
-				im.errors <- &ImportError{
-					Batch: batch,
-					Err:   err,
-				}
+			exitErr := &ImportError{
+				Batch: batch,
 			}
+			if err := im.Import(batch); err != nil {
+				exitErr.Err = err
+			}
+			im.errors <- exitErr
 			return
 		case ts := <-im.input:
 			// init waitForBatch when first
@@ -301,12 +311,13 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {

 	w := io.Writer(pw)
 	if im.compress {
-		zw, err := gzip.NewWriterLevel(pw, 1)
+		zw, err := gzip.NewWriterLevel(w, 1)
 		if err != nil {
 			return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
 		}
 		w = zw
 	}
+	w = limiter.NewWriteLimiter(w, im.rl)
 	bw := bufio.NewWriterSize(w, 16*1024)

 	var totalSamples, totalBytes int
@@ -321,8 +332,8 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	if err := bw.Flush(); err != nil {
 		return err
 	}
-	if im.compress {
-		err := w.(*gzip.Writer).Close()
+	if closer, ok := w.(io.Closer); ok {
+		err := closer.Close()
 		if err != nil {
 			return err
 		}
--- a/app/vmctl/vm_native.go
+++ b/app/vmctl/vm_native.go
@@ -7,12 +7,15 @@ import (
 	"log"
 	"net/http"

-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/cheggaaa/pb/v3"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 )

 type vmNativeProcessor struct {
-	filter filter
+	filter    filter
+	rateLimit int64

 	dst *vmNativeClient
 	src *vmNativeClient
@@ -84,7 +87,12 @@ func (p *vmNativeProcessor) run() error {
 	bar := pb.ProgressBarTemplate(barTpl).Start64(0)
 	barReader := bar.NewProxyReader(exportReader)

-	_, err = io.Copy(pw, barReader)
+	w := io.Writer(pw)
+	if p.rateLimit > 0 {
+		rl := limiter.NewLimiter(p.rateLimit)
+		w = limiter.NewWriteLimiter(pw, rl)
+	}
+	_, err = io.Copy(w, barReader)
 	if err != nil {
 		return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
 	}
--- a/app/vmgateway/README.md
+++ b/app/vmgateway/README.md
@@ -1,6 +1,6 @@
 # vmgateway

-***vmgateway is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
+***vmgateway is a part of [enterprise package](https://victoriametrics.com/products/enterprise/). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***


 <img alt="vmgateway" src="vmgateway-overview.jpeg">
@@ -14,7 +14,7 @@
  * Provides access by tenantID in the Cluster version
  * Allows for separate write/read/admin access to data

-`vmgateway` is included in our [enterprise packages](https://victoriametrics.com/enterprise.html).
+`vmgateway` is included in our [enterprise packages](https://victoriametrics.com/products/enterprise/).


 ## Access Control
@@ -36,6 +36,7 @@ jwt token must be in following format:
         "team": "dev",
         "project": "mobile"
      },
+      "extra_filters": ["{env~=\"prod|dev\",team!=\"test\"}"],
      "mode": 1
  }
 }
@@ -44,7 +45,8 @@ Where:
 - `exp` - required, expire time in unix_timestamp. If the token expires then `vmgateway` rejects the request.
 - `vm_access` - required, dict with claim info, minimum form: `{"vm_access": {"tenand_id": {}}`
 - `tenant_id` - optional, for cluster mode, routes requests to the corresponding tenant.
- `extra_labels` - optional, key-value pairs for label filters added to the ingested or selected metrics.
+- `extra_labels` - optional, key-value pairs for label filters added to the ingested or selected metrics. Multiple filters are added with `and` operation. If defined, `extra_label` from original request removed.
+- `extra_filters` - optional, [series selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) added to the select query requests. Multiple selectors are added with `or` operation. If defined, `extra_filter` from original request removed.
 - `mode` - optional, access mode for api - read, write, or full. Supported values: 0 - full (default value), 1 - read, 2 - write.

 ## QuickStart
@@ -205,7 +207,7 @@ The shortlist of configuration flags include the following:
  -envflag.prefix string
        Prefix for environment variables if -envflag.enable is set
  -eula
-        By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+        By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/legal/eula/
  -fs.disableMmap
        Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches as they cannot read data files larger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
  -http.connTimeout duration
--- a/app/vminsert/influx/request_handler.go
+++ b/app/vminsert/influx/request_handler.go
@@ -23,6 +23,7 @@ var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
 	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
+	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )

 var (
@@ -80,13 +81,13 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
 		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			if tag.Key == "db" {
+			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
 			ic.AddLabel(tag.Key, tag.Value)
 		}
 		if !hasDBKey {
-			ic.AddLabel("db", db)
+			ic.AddLabel(*dbLabel, db)
 		}
 		for j := range extraLabels {
 			label := &extraLabels[j]
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -43,7 +43,8 @@ var (
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
 	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
+	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )

 var (
@@ -57,6 +58,7 @@ var (
 func Init() {
 	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+	storage.SetMaxLabelValueLen(*maxLabelValueLen)
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
@@ -146,6 +148,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
+		addInfluxResponseHeaders(w)
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
@@ -155,6 +158,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/influx/query", "/query":
 		influxQueryRequests.Inc()
+		addInfluxResponseHeaders(w)
 		influxutils.WriteDatabaseNames(w)
 		return true
 	case "/datadog/api/v1/series":
@@ -165,26 +169,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "/datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	case "/prometheus/targets", "/targets":
@@ -193,10 +197,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
+	case "/prometheus/target_response", "/target_response":
+		promscrapeTargetResponseRequests.Inc()
+		if err := promscrape.WriteTargetResponse(w, r); err != nil {
+			promscrapeTargetResponseErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
 	case "/prometheus/config", "/config":
 		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
 			err := &httpserver.ErrorWithStatusCode{
@@ -231,6 +243,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }

+func addInfluxResponseHeaders(w http.ResponseWriter) {
+	// This is needed for some clients, which expect InfluxDB version header.
+	// See, for example, https://github.com/ntop/ntopng/issues/5449#issuecomment-1005347597
+	w.Header().Set("X-Influxdb-Version", "1.8.0")
+}
+
 var (
 	requestDuration = metrics.NewHistogram(`vminsert_request_duration_seconds`)

@@ -264,6 +282,9 @@ var (
 	promscrapeTargetsRequests      = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)

+	promscrapeTargetResponseRequests = metrics.NewCounter(`vm_http_requests_total{path="/target_response"}`)
+	promscrapeTargetResponseErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/target_response"}`)
+
 	promscrapeConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/config"}`)

 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
--- a/app/vminsert/relabel/relabel.go
+++ b/app/vminsert/relabel/relabel.go
@@ -16,6 +16,7 @@ import (

 var (
 	relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+
+		"The path can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal")
 	relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+
 		"then the metrics aren't sent to storage. This is useful for debugging the relabeling configs")
--- a/Show More
+++ b/Show More