docs/CHANGELOG.md: cut v1.77.2

* lib/promscrape/discovery/kubernetes: properly updates discovered scrape works
previously, added or updated scrapeworks may override previuosly
discovered.
it happens because swosByKey may contain small subset of kubernetes
objects with it's labels.
It happens for objectsUpdated and objectsAdded maps, which include only changed elements

* Properly calculate vm_promscrape_discovery_kubernetes_scrape_works

Co-authored-by: f41gh7 <nik@victoriametrics.com>
Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,10 +8,10 @@ assignees: ''
 
 **Describe the bug**
 A clear and concise description of what the bug is.
-It would be a great [upgrading](https://docs.victoriametrics.com/#how-to-upgrade) 
+It would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
 to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-and verifying whether the bug is reproducible there.
-It is also recommended reading [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
+and verify whether the bug is reproducible there.
+It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
 
 **To Reproduce**
 Steps to reproduce the behavior.
@@ -33,15 +33,14 @@ of Grafana dashboards if possible:
 
 See how to setup monitoring here:
 * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-* [montioring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+* [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
 
 **Version**
-The line returned when passing `--version` command line flag to binary. For example:
+The line returned when passing `--version` command line flag to the binary. For example:
 ```
 $ ./victoria-metrics-prod --version
 victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
 ```
 
 **Used command-line flags**
-Please provide applied command-line flags used for running VictoriaMetrics and its components. 
-
+Please provide the command-line flags used for running VictoriaMetrics and its components. 

.github/dependabot.yml

@@ -16,6 +16,7 @@ updates:
     directory: "/app/vmui/packages/vmui/web"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
@@ -24,3 +25,4 @@ updates:
     directory: "/app/vmui/packages/vmui"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 0

.github/workflows/check-licenses.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     paths:
       - 'vendor'
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master, cluster ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master, cluster ]
+  schedule:
+    - cron: '30 18 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -8,6 +8,9 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**.md'
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build
@@ -60,7 +63,7 @@ jobs:
           GOOS=darwin go build -mod=vendor ./app/vmctl
           CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
       - name: Publish coverage
-        uses: codecov/codecov-action@v2.1.0
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
 

.github/workflows/wiki.yml

@@ -5,8 +5,13 @@ on:
       - 'docs/*'
     branches:
       - master
+permissions:
+  contents: read
+
 jobs:
   build:
+    permissions:
+      contents: write  # for Git to git push
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master

CODE_OF_CONDUCT.md

@@ -68,9 +68,9 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+available at <https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>
 
 [homepage]: https://www.contributor-covenant.org
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+<https://www.contributor-covenant.org/faq>

CODE_OF_CONDUCT_RU.md

@@ -79,7 +79,7 @@
 
 **Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
 На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие 
+включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
 в публичных пространствах, а так же во внешних каналах,
 таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
@@ -89,10 +89,10 @@
 **Общественное влияние**: Серьёзное нарушение стандартов сообщества,
 включая продолжительное неуместное поведение.
 
-**Последствия**: Временный запрет (бан) на любое взаимодействие 
+**Последствия**: Временный запрет (бан) на любое взаимодействие
 или публичное общение с сообществом на определенный период времени.
 На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие 
+вовлеченными в инцидент, включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса.
 Нарушение этих правил влечет за собой вечный бан.
 
@@ -108,7 +108,7 @@
 
 Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
 версии 2.0, доступной по адресу
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
 
 Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
 enforcement ladder](https://github.com/mozilla/diversity).
@@ -116,5 +116,5 @@ enforcement ladder](https://github.com/mozilla/diversity).
 [homepage]: https://www.contributor-covenant.org
 
 Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-https://www.contributor-covenant.org/faq. Переводы доступны по адресу
-https://www.contributor-covenant.org/translations.
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

Makefile

@@ -257,7 +257,7 @@ benchmark-pure:
 vendor-update:
 	GO111MODULE=on go get -u -d ./lib/...
 	GO111MODULE=on go get -u -d ./app/...
-	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod tidy -compat=1.17
 	GO111MODULE=on go mod vendor
 
 app-local:
@@ -283,7 +283,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.44.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.1
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

app/victoria-metrics/main.go

@@ -24,7 +24,7 @@ import (
 
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
-	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -86,6 +86,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")

app/vmagent/README.md

@@ -6,7 +6,6 @@ or any other Prometheus-compatible storage systems that support the `remote_writ
 
 <img alt="vmagent" src="vmagent.png">
 
-
 ## Motivation
 
 While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
@@ -14,7 +13,6 @@ and RAM friendly to scrape metrics from Prometheus-compatible exporters into Vic
 Also, we found that our user's infrastructure are like snowflakes in that no two are alike. Therefore we decided to add more flexibility
 to `vmagent` such as the ability to push metrics additionally to pulling them. We did our best and will continue to improve `vmagent`.
 
-
 ## Features
 
 * Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter). See [Quick Start](#quick-start) for details.
@@ -67,7 +65,6 @@ Then send InfluxDB data to `http://vmagent-host:8429`. See [these docs](https://
 
 Pass `-help` to `vmagent` in order to see [the full list of supported command-line flags with their descriptions](#advanced-usage).
 
-
 ## Configuration update
 
 `vmagent` should be restarted in order to update config options set via command-line args.
@@ -75,6 +72,7 @@ Pass `-help` to `vmagent` in order to see [the full list of supported command-li
 `vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
 
 * Sending `SUGHUP` signal to `vmagent` process:
+
   ```bash
   kill -SIGHUP `pidof vmagent`
   ```
@@ -83,10 +81,8 @@ Pass `-help` to `vmagent` in order to see [the full list of supported command-li
 
 There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
 
-
 ## Use cases
 
-
 ### IoT and Edge monitoring
 
 `vmagent` can run and collect metrics in IoT and industrial networks with unreliable or scheduled connections to their remote storage.
@@ -97,28 +93,24 @@ The maximum buffer size can be limited with `-remoteWrite.maxDiskUsagePerURL`.
 `vmagent` works on various architectures from the IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
 See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
 
-
 ### Drop-in replacement for Prometheus
 
 If you use Prometheus only for scraping metrics from various targets and forwarding those metrics to remote storage
 then `vmagent` can replace Prometheus. Typically, `vmagent` requires lower amounts of RAM, CPU and network bandwidth compared with Prometheus.
 See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
 
-
 ### Replication and high availability
 
 `vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
 If a single remote storage instance temporarily is out of service, then the collected data remains available in another remote storage instance.
 `vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again and then it sends the buffered data to the remote storage in order to prevent data gaps.
 
-
 ### Relabeling and filtering
 
 `vmagent` can add, remove or update labels on the collected data before sending it to the remote storage. Additionally,
 it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
 Please see [these docs](#relabeling) for details.
 
-
 ### Splitting data streams among multiple systems
 
 `vmagent` supports splitting the collected data between muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
@@ -126,7 +118,6 @@ which is applied independently for each configured `-remoteWrite.url` destinatio
 data among long-term remote storage, short-term remote storage and a real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
 Note that each destination can receive it's own subset of the collected data due to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
 
-
 ### Prometheus remote_write proxy
 
 `vmagent` can be used as a proxy for Prometheus data sent via Prometheus `remote_write` protocol. It can accept data via the `remote_write` API
@@ -134,7 +125,6 @@ at the`/api/v1/write` endpoint. Then apply relabeling and filtering and proxy it
 The `vmagent` can be configured to encrypt the incoming `remote_write` requests with `-tls*` command-line flags.
 Also, Basic Auth can be enabled for the incoming `remote_write` requests with `-httpAuth.*` command-line flags.
 
-
 ### remote_write for clustered version
 
 While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx, Prometheus, Graphite) and scrape data from various targets, writes are always peformed in Promethes remote_write protocol. Therefore for the [clustered version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html), `-remoteWrite.url` the command-line flag should be configured as `<schema>://<vminsert-host>:8480/insert/<accountID>/prometheus/api/v1/write` according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). There is also support for multitenant writes. See [these docs](#multitenancy).
@@ -143,7 +133,6 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx
 
 By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. But it can accept multitenant data if `-remoteWrite.multitenantURL` is set. In this case it accepts multitenant data at `http://vmagent:8429/insert/<accountID>/...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert/<accountID>/prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants.
 
-
 ## How to collect metrics in Prometheus format
 
 Specify the path to `prometheus.yml` file via `-promscrape.config` command-line flag. `vmagent` takes into account the following
@@ -211,7 +200,6 @@ entries to 60s. Run `vmagent -help` in order to see default values for the `-pro
 
 The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
 
-
 ## Loading scrape configs from multiple files
 
 `vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url:
@@ -236,7 +224,6 @@ Every referred file can contain arbitrary number of [supported scrape configs](#
 
 `vmagent` dynamically reloads these files on `SIGHUP` signal or on the request to `http://vmagent:8429/-/reload`.
 
-
 ## Unsupported Prometheus config sections
 
 `vmagent` doesn't support the following sections in Prometheus config file passed to `-promscrape.config` command-line flag:
@@ -249,7 +236,6 @@ The list of supported service discovery types is available [here](#how-to-collec
 
 Additionally `vmagent` doesn't support `refresh_interval` option at service discovery sections. This option is substituted with `-promscrape.*CheckInterval` command-line options, which are specific per each service discovery type. See [the full list of command-line flags for vmagent](#advanced-usage).
 
-
 ## Adding labels to metrics
 
 Labels can be added to metrics by the following mechanisms:
@@ -261,7 +247,6 @@ Labels can be added to metrics by the following mechanisms:
 /path/to/vmagent -remoteWrite.label=datacenter=foobar ...
 ```
 
-
 ## Relabeling
 
 VictoriaMetrics components (including `vmagent`) support Prometheus-compatible relabeling.
@@ -320,7 +305,6 @@ You can read more about relabeling in the following articles:
 * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
-
 ## Prometheus staleness markers
 
 `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases:
@@ -332,16 +316,15 @@ You can read more about relabeling in the following articles:
 
 Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.
 
-
 ## Stream parsing mode
 
 By default `vmagent` reads the full response body from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics.
 
 Stream parsing mode is automatically enabled for scrape targets returning response bodies with sizes bigger than the `-promscrape.minResponseSizeForStreamParse` command-line flag value. Additionally, the stream parsing mode can be explicitly enabled in the following places:
 
-- Via `-promscrape.streamParse` command-line flag. In this case all the scrape targets defined in the file pointed by `-promscrape.config` are scraped in stream parsing mode.
-- Via `stream_parse: true` option at `scrape_configs` section. In this case all the scrape targets defined in this section are scraped in stream parsing mode.
-- Via `__stream_parse__=true` label, which can be set via [relabeling](#relabeling) at `relabel_configs` section. In this case stream parsing mode is enabled for the corresponding scrape targets. Typical use case: to set the label via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets exposing big number of metrics.
+* Via `-promscrape.streamParse` command-line flag. In this case all the scrape targets defined in the file pointed by `-promscrape.config` are scraped in stream parsing mode.
+* Via `stream_parse: true` option at `scrape_configs` section. In this case all the scrape targets defined in this section are scraped in stream parsing mode.
+* Via `__stream_parse__=true` label, which can be set via [relabeling](#relabeling) at `relabel_configs` section. In this case stream parsing mode is enabled for the corresponding scrape targets. Typical use case: to set the label via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets exposing big number of metrics.
 
 Examples:
 
@@ -361,7 +344,6 @@ scrape_configs:
 
 Note that `sample_limit` and `series_limit` options cannot be used in stream parsing mode because the parsed data is pushed to remote storage as soon as it is parsed.
 
-
 ## Scraping big number of targets
 
 A single `vmagent` instance can scrape tens of thousands of scrape targets. Sometimes this isn't enough due to limitations on CPU, network, RAM, etc.
@@ -376,6 +358,8 @@ spread scrape targets among a cluster of two `vmagent` instances:
 /path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=1 -promscrape.config=/path/to/config.yml ...
 ```
 
+The `-promscrape.cluster.memberNum` can be set to a StatefulSet pod name when `vmagent` runs in Kubernetes. The pod name must end with a number in the range `0 ... promscrape.cluster.memberNum-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
+
 By default each scrape target is scraped only by a single `vmagent` instance in the cluster. If there is a need for replicating scrape targets among multiple `vmagent` instances,
 then `-promscrape.cluster.replicationFactor` command-line flag must be set to the desired number of replicas. For example, the following commands
 start a cluster of three `vmagent` instances, where each target is scraped by two `vmagent` instances:
@@ -389,7 +373,6 @@ start a cluster of three `vmagent` instances, where each target is scraped by tw
 If each target is scraped by multiple `vmagent` instances, then data deduplication must be enabled at remote storage pointed by `-remoteWrite.url`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
-
 ## Scraping targets via a proxy
 
 `vmagent` supports scraping targets via http, https and socks5 proxies. Proxy address must be specified in `proxy_url` option. For example, the following scrape config instructs
@@ -429,9 +412,9 @@ scrape_configs:
 
 By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced in the following places:
 
-- Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually to all the scrape targets defined in the file pointed by `-promscrape.config`.
-- Via `series_limit` config option at `scrape_config` section. This limit is applied individually to all the scrape targets defined in the given `scrape_config`.
-- Via `__series_limit__` label, which can be set with [relabeling](#relabeling) at `relabel_configs` section. This limit is applied to the corresponding scrape targets. Typical use case: to set the limit via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets, which may expose too high number of time series.
+* Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually to all the scrape targets defined in the file pointed by `-promscrape.config`.
+* Via `series_limit` config option at `scrape_config` section. This limit is applied individually to all the scrape targets defined in the given `scrape_config`.
+* Via `__series_limit__` label, which can be set with [relabeling](#relabeling) at `relabel_configs` section. This limit is applied to the corresponding scrape targets. Typical use case: to set the limit via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets, which may expose too high number of time series.
 
 All the scraped metrics are dropped for time series exceeding the given limit. The exceeded limit can be [monitored](#monitoring) via `promscrape_series_limit_rows_dropped_total` metric.
 
@@ -451,7 +434,6 @@ The exceeded limits can be [monitored](#monitoring) with the following metrics:
 
 These limits are approximate, so `vmagent` can underflow/overflow the limit by a small percentage (usually less than 1%).
 
-
 ## Monitoring
 
 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. We recommend setting up regular scraping of this page
@@ -470,7 +452,6 @@ This information may be useful for debugging target relabeling.
 * `http://vmagent-host:8429/ready`. This handler returns http 200 status code when `vmagent` finishes it's initialization for all service_discovery configs.
 It may be useful to perform `vmagent` rolling update without any scrape loss.
 
-
 ## Troubleshooting
 
 * We recommend you [set up the official Grafana dashboard](#monitoring) in order to monitor the state of `vmagent'.
@@ -482,7 +463,7 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
   * Disabling staleness tracking with `-promscrape.noStaleMarkers` option. See [these docs](#prometheus-staleness-markers).
   * Enabling stream parsing mode if `vmagent` scrapes targets with millions of metrics per target. See [these docs](#stream-parsing-mode).
   * Reducing the number of output queues with `-remoteWrite.queues` command-line option.
-  * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kuberntes if `vmagent` runs under these systems.
+  * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kubernetes if `vmagent` runs under these systems.
   * Reducing the number of CPU cores vmagent can use by passing `GOMAXPROCS=N` environment variable to `vmagent`, where `N` is the desired limit on CPU cores. Another option is to reduce CPU limits in Docker or Kubernetes if `vmagent` runs under these systems.
   * Passing `-promscrape.dropOriginalLabels` command-line option to `vmagent`, so it drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page. This reduces memory usage when scraping big number of targets at the cost of reduced debuggability for improperly configured per-target relabeling.
 
@@ -534,12 +515,14 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
   See the available options below if you prefer fixing the root cause of the error:
 
   The following relabeling rule may be added to `relabel_configs` section in order to filter out pods with unneeded ports:
+
   ```yml
   - action: keep_if_equal
     source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]
   ```
 
   The following relabeling rule may be added to `relabel_configs` section in order to filter out init container pods:
+
   ```yml
   - action: drop
     source_labels: [__meta_kubernetes_pod_container_init]
@@ -555,7 +538,6 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 
 The enterprise version of vmagent is available for evaluation at [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page in `vmutils-*-enteprise.tar.gz` archives and in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
 
-
 ### Reading metrics from Kafka
 
 [Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
@@ -591,7 +573,6 @@ topic = "influx"
 data_format = "influx"
 ```
 
-
 #### Command-line flags for Kafka consumer
 
 These command-line flags are available only in [enterprise](https://victoriametrics.com/products/enterprise/) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
@@ -631,7 +612,6 @@ These command-line flags are available only in [enterprise](https://victoriametr
 
 Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
 
-
 #### Kafka broker authorization and authentication
 
 Two types of auth are supported:
@@ -648,12 +628,10 @@ Two types of auth are supported:
 ./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SSL -remoteWrite.tlsCAFile=/opt/ca.pem -remoteWrite.tlsCertFile=/opt/cert.pem -remoteWrite.tlsKeyFile=/opt/key.pem
 ```
 
-
 ## How to build from sources
 
 We recommend using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in the `vmutils-*` archives .
 
-
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
@@ -695,7 +673,6 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 2. Run `make vmagent-arm-prod` or `make vmagent-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmagent-arm-prod` or `vmagent-arm64-prod` binary respectively and puts it into the `bin` folder.
 
-
 ## Profiling
 
 `vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
@@ -724,7 +701,6 @@ The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
 
-
 ## Advanced usage
 
 `vmagent` can be fine-tuned with various command-line flags. Run `./vmagent -help` in order to see the full list of these flags with their desciptions and default values:
@@ -737,314 +713,335 @@ vmagent collects metrics data via popular data ingestion protocols and routes th
 See the docs at https://docs.victoriametrics.com/vmagent.html .
 
   -configAuthKey string
-    	Authorization key for accessing /config page. It must be passed via authKey query arg
+     Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
-    	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+     Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -datadog.maxInsertRequestSize size
-    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
+     The maximum size in bytes of a single DataDog POST request to /api/v1/series
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dryRun
-    	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
+     Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
-    	Prefix for environment variables if -envflag.enable is set
+     Prefix for environment variables if -envflag.enable is set
   -eula
-    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -fs.disableMmap
-    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -graphiteListenAddr string
-    	TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty
+     TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty
   -graphiteTrimTimestamp duration
-    	Trim timestamps for Graphite data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
+     Trim timestamps for Graphite data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+     Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+     Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
-    	Timeout for incoming idle http connections (default 1m0s)
+     Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
-    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+     An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+     Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-    	TCP address to listen for http connections. Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. Note that /targets and /metrics pages aren't available if -httpListenAddr='' (default ":8429")
+     TCP address to listen for http connections. Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. Note that /targets and /metrics pages aren't available if -httpListenAddr='' (default ":8429")
   -import.maxLineLen size
-    	The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 104857600)
+     The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 104857600)
   -influx.databaseNames array
-    	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
+     Supports an array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize size
-    	The maximum size in bytes for a single InfluxDB line during parsing
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
+     The maximum size in bytes for a single InfluxDB line during parsing
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
   -influxDBLabel string
-    	Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
+     Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
   -influxListenAddr string
-    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
+     TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
   -influxMeasurementFieldSeparator string
-    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
+     Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
-    	Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
+     Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
   -influxSkipSingleField
-    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
+     Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
   -influxTrimTimestamp duration
-    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+     Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
-    	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
+     The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
   -kafka.consumer.topic array
-    	Kafka topic names for data consumption.
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Kafka topic names for data consumption.
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.basicAuth.password array
-    	Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.basicAuth.username array
-    	Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.brokers array
-    	List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
-    	Supports an array of values separated by comma or specified via multiple flags.
+     List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.defaultFormat string
-    	Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+     Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
   -kafka.consumer.topic.format array
-    	data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
-    	Supports an array of values separated by comma or specified via multiple flags.
+     data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.groupID array
-    	Defines group.id for topic
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Defines group.id for topic
+     Supports an array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.isGzipped array
-    	Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
-    	Supports array of values separated by comma or specified via multiple flags.
+     Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+     Supports array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.options array
-    	Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+     Supports an array of values separated by comma or specified via multiple flags.
   -loggerDisableTimestamps
-    	Whether to disable writing timestamps in logs
+     Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
-    	Format for logs. Possible values: default, json (default "default")
+     Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+     Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
-    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+     Output for the logs. Supported values: stderr, stdout (default "stderr")
   -loggerTimezone string
-    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+     Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
-    	The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16)
+     The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16)
   -maxInsertRequestSize size
-    	The maximum size in bytes of a single Prometheus remote_write API request
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
+     The maximum size in bytes of a single Prometheus remote_write API request
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-    	TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
+     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
   -opentsdbListenAddr string
-    	TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty
+     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty
   -opentsdbTrimTimestamp duration
-    	Trim timestamps for OpenTSDB 'telnet put' data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
+     Trim timestamps for OpenTSDB 'telnet put' data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -opentsdbhttp.maxInsertRequestSize size
-    	The maximum size of OpenTSDB HTTP put request
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
+     The maximum size of OpenTSDB HTTP put request
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
   -opentsdbhttpTrimTimestamp duration
-    	Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+     Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
-  -promscrape.cluster.memberNum int
-    	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
+     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -promscrape.cluster.memberNum string
+     The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
   -promscrape.cluster.membersCount int
-    	The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
+     The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
   -promscrape.cluster.replicationFactor int
-    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
+     The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+     Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
-    	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
+     Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
   -promscrape.config.strictParse
-    	Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
+     Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
   -promscrape.configCheckInterval duration
-    	Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+     Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
   -promscrape.consul.waitTime duration
-    	Wait time used by Consul service discovery. Default value is used if not set
+     Wait time used by Consul service discovery. Default value is used if not set
   -promscrape.consulSDCheckInterval duration
-    	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
   -promscrape.digitaloceanSDCheckInterval duration
-    	Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config for details (default 1m0s)
+     Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config for details (default 1m0s)
   -promscrape.disableCompression
-    	Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.disableKeepAlive
-    	Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets
+     Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets
   -promscrape.discovery.concurrency int
-    	The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
+     The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
   -promscrape.discovery.concurrentWaitTime duration
-    	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
+     The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
   -promscrape.dnsSDCheckInterval duration
-    	Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
+     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
   -promscrape.dockerSDCheckInterval duration
-    	Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details (default 30s)
+     Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details (default 30s)
   -promscrape.dockerswarmSDCheckInterval duration
-    	Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details (default 30s)
+     Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details (default 30s)
   -promscrape.dropOriginalLabels
-    	Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
+     Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
   -promscrape.ec2SDCheckInterval duration
-    	Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config for details (default 1m0s)
+     Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config for details (default 1m0s)
   -promscrape.eurekaSDCheckInterval duration
-    	Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
+     Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config for details (default 30s)
   -promscrape.fileSDCheckInterval duration
-    	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
+     Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 5m0s)
   -promscrape.gceSDCheckInterval duration
-    	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+     Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
-    	Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)
+     Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
-    	How frequently to reload the full state from Kuberntes API server (default 30m0s)
+     How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
-    	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
+     Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets int
-    	The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+     The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
-    	The maximum size of http response headers from Prometheus scrape targets
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
+     The maximum size of http response headers from Prometheus scrape targets
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
   -promscrape.maxScrapeSize size
-    	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
+     The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
   -promscrape.minResponseSizeForStreamParse size
-    	The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
+     The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
   -promscrape.noStaleMarkers
-    	Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
+     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
-    	Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
+     Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
   -promscrape.seriesLimitPerTarget int
-    	Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
+     Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
-    	Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
-    	Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
+     Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
-    	Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
+     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
+  -remoteWrite.aws.accessKey array
+     Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.aws.region array
+     Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.aws.roleARN array
+     Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.aws.secretKey array
+     Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.aws.serice array
+     Optional AWS Service to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url. Defaults to "aps".
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.aws.useSigv4 array
+     Enables SigV4 request signing for -remoteWrite.url. It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.basicAuth.password array
-    	Optional basic auth password to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth password to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.basicAuth.passwordFile array
-    	Optional path to basic auth password to use for -remoteWrite.url. The file is re-read every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to basic auth password to use for -remoteWrite.url. The file is re-read every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.basicAuth.username array
-    	Optional basic auth username to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth username to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.bearerToken array
-    	Optional bearer auth token to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional bearer auth token to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.bearerTokenFile array
-    	Optional path to bearer token file to use for -remoteWrite.url. The token is re-read from the file every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to bearer token file to use for -remoteWrite.url. The token is re-read from the file every second. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.flushInterval duration
-    	Interval for flushing the data to remote storage. This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url (default 1s)
+     Interval for flushing the data to remote storage. This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url (default 1s)
   -remoteWrite.label array
-    	Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.maxBlockSize size
-    	The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
+     The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
   -remoteWrite.maxDailySeries int
-    	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
   -remoteWrite.maxDiskUsagePerURL size
-    	The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. Disk usage is unlimited if the value is set to 0
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. Disk usage is unlimited if the value is set to 0
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -remoteWrite.maxHourlySeries int
-    	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
   -remoteWrite.maxRowsPerBlock int
-    	The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
+     The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
   -remoteWrite.multitenantURL array
-    	Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.clientID array
-    	Optional OAuth2 clientID to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional OAuth2 clientID to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.clientSecret array
-    	Optional OAuth2 clientSecret to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional OAuth2 clientSecret to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.clientSecretFile array
-    	Optional OAuth2 clientSecretFile to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional OAuth2 clientSecretFile to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.scopes array
-    	Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.tokenUrl array
-    	Optional OAuth2 tokenURL to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional OAuth2 tokenURL to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.proxyURL array
-    	Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.queues int
-    	The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs (default 8)
+     The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs (default 8)
   -remoteWrite.rateLimit array
-    	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
-    	Supports array of values separated by comma or specified via multiple flags.
+     Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
-    	Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+     Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
   -remoteWrite.relabelDebug
-    	Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
+     Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
-    	Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
-    	Supports array of values separated by comma or specified via multiple flags.
+     Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.sendTimeout array
-    	Timeout for sending a single block of data to -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+     Timeout for sending a single block of data to -remoteWrite.url
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.showURL
-    	Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
+     Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteWrite.significantFigures array
-    	The number of significant figures to leave in metric values before writing them to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits
-    	Supports array of values separated by comma or specified via multiple flags.
+     The number of significant figures to leave in metric values before writing them to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCAFile array
-    	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCertFile array
-    	Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsInsecureSkipVerify array
-    	Whether to skip tls verification when connecting to -remoteWrite.url
-    	Supports array of values separated by comma or specified via multiple flags.
+     Whether to skip tls verification when connecting to -remoteWrite.url
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsKeyFile array
-    	Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsServerName array
-    	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.tmpDataPath string
-    	Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
+     Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
-    	Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelDebug array
-    	Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
-    	Supports array of values separated by comma or specified via multiple flags.
+     Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs
+     Supports array of values separated by comma or specified via multiple flags.
   -sortLabels
-    	Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
+     Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls
-    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+     Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+  -tlsCipherSuites array
+     Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
+     Supports an array of values separated by comma or specified via multiple flags.
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
-    	Show VictoriaMetrics version
+     Show VictoriaMetrics version
 ```

app/vmagent/main.go

@@ -43,7 +43,7 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
@@ -154,6 +154,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")

app/vmagent/native/request_handler.go

@@ -30,8 +30,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(block *parser.Block) error {
+		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
 			return insertRows(at, block, extraLabels)
 		})
 	})

app/vmagent/remotewrite/client.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -59,6 +60,20 @@ var (
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 	oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
 		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+
+	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	awsService = flagutil.NewArray("remoteWrite.aws.serice", "Optional AWS Service to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url. Defaults to \"aps\".")
+	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 )
 
 type client struct {
@@ -69,6 +84,7 @@ type client struct {
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
+	awsCfg    *awsapi.Config
 
 	rl rateLimiter
 
@@ -78,6 +94,7 @@ type client struct {
 	requestsOKCount *metrics.Counter
 	errorsCount     *metrics.Counter
 	packetsDropped  *metrics.Counter
+	rateLimit       *metrics.Gauge
 	retriesCount    *metrics.Counter
 	sendDuration    *metrics.FloatCounter
 
@@ -88,13 +105,17 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tlsCfg := authCfg.NewTLSConfig()
+	awsCfg, err := getAWSAPIConfig(argIdx)
+	if err != nil {
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
 	tr := &http.Transport{
-		Dial:                statDial,
+		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 5 * time.Second,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -115,6 +136,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
+		awsCfg:         awsCfg,
 		fq:             fq,
 		hc: &http.Client{
 			Transport: tr,
@@ -131,10 +153,13 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
-	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remote_write_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
+	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
+	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
 	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
@@ -201,6 +226,22 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	return authCfg, nil
 }
 
+func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
+	if !awsUseSigv4.GetOptionalArg(argIdx) {
+		return nil, nil
+	}
+	region := awsRegion.GetOptionalArg(argIdx)
+	roleARN := awsRoleARN.GetOptionalArg(argIdx)
+	accessKey := awsAccessKey.GetOptionalArg(argIdx)
+	secretKey := awsSecretKey.GetOptionalArg(argIdx)
+	service := awsService.GetOptionalArg(argIdx)
+	cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, service)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
 func (c *client) runWorker() {
 	var ok bool
 	var block []byte
@@ -250,6 +291,10 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 	retriesCount := 0
 	c.bytesSent.Add(len(block))
 	c.blocksSent.Inc()
+	sigv4Hash := ""
+	if c.awsCfg != nil {
+		sigv4Hash = awsapi.HashHex(block)
+	}
 
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
@@ -264,7 +309,12 @@ again:
 	if ah := c.authCfg.GetAuthHeader(); ah != "" {
 		req.Header.Set("Authorization", ah)
 	}
-
+	if c.awsCfg != nil {
+		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
+		}
+	}
 	startTime := time.Now()
 	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)

app/vmagent/remotewrite/pendingseries.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -210,7 +211,22 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		writeRequestBufPool.Put(bb)
 	}
 
-	// Too big block. Recursively split it into smaller parts.
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
+		samples := wr.Timeseries[0].Samples
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return
+		}
+		n := len(samples) / 2
+		wr.Timeseries[0].Samples = samples[:n]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples[n:]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples
+		return
+	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]

app/vmagent/remotewrite/remotewrite.go

@@ -40,7 +40,7 @@ var (
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
-		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
 	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+
 		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
@@ -274,6 +274,8 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
+	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
@@ -298,9 +300,10 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			tss = nil
 		}
 		if rctx != nil {
-			tssBlockLen := len(tssBlock)
+			rowsCountBeforeRelabel := getRowsCount(tssBlock)
 			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
-			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+			rowsCountAfterRelabel := getRowsCount(tssBlock)
+			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
@@ -414,7 +417,10 @@ func labelsToString(labels []prompbmarshal.Label) string {
 	return string(b)
 }
 
-var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+var (
+	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
+	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+)
 
 type remoteWriteCtx struct {
 	idx        int
@@ -423,7 +429,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
-	relabelMetricsDropped *metrics.Counter
+	rowsPushedAfterRelabel *metrics.Counter
+	rowsDroppedByRelabel   *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -467,7 +474,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 
@@ -483,7 +491,8 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
-	rwctx.relabelMetricsDropped = nil
+	rwctx.rowsPushedAfterRelabel = nil
+	rwctx.rowsDroppedByRelabel = nil
 }
 
 func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
@@ -499,12 +508,15 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
 		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
-		tssLen := len(tss)
+		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, nil, pcs)
-		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+		rowsCountAfterRelabel := getRowsCount(tss)
+		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 	pss[idx].Push(tss)
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
@@ -519,3 +531,11 @@ var tssRelabelPool = &sync.Pool{
 		return &a
 	},
 }
+
+func getRowsCount(tss []prompbmarshal.TimeSeries) int {
+	rowsCount := 0
+	for _, ts := range tss {
+		rowsCount += len(ts.Samples)
+	}
+	return rowsCount
+}

app/vmagent/remotewrite/statconn.go

@@ -1,7 +1,9 @@
 package remotewrite
 
 import (
+	"context"
 	"net"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -9,9 +11,26 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func statDial(networkUnused, addr string) (conn net.Conn, err error) {
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
-	conn, err = net.DialTimeout(network, addr, 5*time.Second)
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
 	dialsTotal.Inc()
 	if err != nil {
 		dialErrors.Inc()

app/vmalert/Makefile

@@ -62,13 +62,14 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-notifier.url=http://localhost:9093 \
 		-notifier.url=http://127.0.0.1:9093 \
@@ -77,13 +78,13 @@ run-vmalert: vmalert
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
 		-evaluationInterval=3s \
-		-rule.configCheckInterval=10s
+		-configCheckInterval=10s
 
 run-vmalert-sd: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
-		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-notifier.config=app/vmalert/notifier/testdata/mixed.good.yaml \
 		-configCheckInterval=10s
 
 replay-vmalert: vmalert

app/vmalert/README.md

@@ -3,54 +3,61 @@
 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
 rules against configured `-datasource.url`. For sending alerting notifications
-vmalert relies on [Alertmanager]((https://github.com/prometheus/alertmanager)) configured via `-notifier.url` flag.
+vmalert relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag.
 Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
 protocol and require `-remoteWrite.url` to be configured.
 Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
 ## Features
+
 * Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
 * VictoriaMetrics [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html)
- support and expressions validation;
+  support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
- support;
+  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-aplha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
 * Keeps the alerts [state on restarts](#alerts-state-on-restarts);
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight without extra dependencies.
+* Supports [reusable templates](#reusable-templates) for annotations.
 
 ## Limitations
+
 * `vmalert` execute queries against remote datasource which has reliability risks because of the network.
-It is recommended to configure alerts thresholds and rules expressions with the understanding that network 
-requests may fail;
+  It is recommended to configure alerts thresholds and rules expressions with the understanding that network
+  requests may fail;
 * by default, rules execution is sequential within one group, but persistence of execution results to remote
-storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
-recording rule is reused in the next one;
+  storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
+  recording rule is reused in the next one;
 
 ## QuickStart
 
 To build `vmalert` from sources:
-```
+
+```bash
 git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
 ```
+
 The build binary will be placed in `VictoriaMetrics/bin` folder.
 
 To start using `vmalert` you will need the following things:
+
 * list of rules - PromQL/MetricsQL expressions to execute;
 * datasource address - reachable MetricsQL endpoint to run queries against;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts, and sending notifications. Please note, notifier address also supports Consul Service Discovery via 
-[config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
+  aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
+  [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
   compatible storage to persist rules and alerts state info;
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
-```
+
+```bash
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
@@ -77,6 +84,7 @@ and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerti
 similar to Prometheus rules and configured using YAML. Configuration examples may be found
 in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
 Every `rule` belongs to a `group` and every configuration file may contain arbitrary number of groups:
+
 ```yaml
 groups:
   [ - <rule_group> ]
@@ -85,6 +93,7 @@ groups:
 ### Groups
 
 Each group has the following attributes:
+
 ```yaml
 # The name of the group. Must be unique within a file.
 name: <string>
@@ -136,13 +145,14 @@ or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. Vmal
 expression and then act according to the Rule type.
 
 There are two types of Rules:
+
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allow defining alert conditions via `expr` field and to send notifications to
-[Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
+  Alerting rules allow defining alert conditions via `expr` field and to send notifications to
+  [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allow defining `expr` which result will be then backfilled to configured
-`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
-expensive expressions and save their result as a new set of time series.
+  Recording rules allow defining `expr` which result will be then backfilled to configured
+  `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
+  expensive expressions and save their result as a new set of time series.
 
 `vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.
@@ -150,6 +160,7 @@ within one group.
 #### Alerting rules
 
 The syntax for alerting rule is the following:
+
 ```yaml
 # The name of the alert. Must be a valid metric name.
 alert: <string>
@@ -174,14 +185,57 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ```
 
-It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations
-to format data, iterate over it or execute expressions.
+It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations to format data, iterate over it or execute expressions.
 Additionally, `vmalert` provides some extra templating functions
-listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go).
+listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go) and [reusable templates](#reusable-templates).
+
+#### Reusable templates
+
+Like in Alertmanager you can define [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates)
+to share same templates across annotations. Just define the templates in a file and 
+set the path via `-rule.templates` flag.
+
+For example, template `grafana.filter` can be defined as following:
+
+{% raw  %}
+```
+{{ define "grafana.filter" -}}
+  {{- $labels := .arg0 -}}
+  {{- range $name, $label := . -}}
+    {{- if (ne $name "arg0") -}}
+      {{- ( or (index $labels $label) "All" ) | printf "&var-%s=%s" $label -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+```
+{% endraw %}
+
+And then used in annotations:
+
+{% raw  %}
+```yaml
+groups:
+  - name: AlertGroupName
+    rules:
+      - alert: AlertName
+        expr: any_metric > 100
+        for: 30s
+        labels:
+          alertname: 'Any metric is too high'
+          severity: 'warning'
+        annotations:
+          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1{{ template "grafana.filter" (args .CommonLabels "account_id" "any_label") }}'
+```
+{% endraw %}
+
+The `-rule.templates` flag supports wildcards so multiple files with templates can be loaded.
+The content of `-rule.templates` can be also [hot reloaded](#hot-config-reload).
+
 
 #### Recording rules
 
 The syntax for recording rules is following:
+
 ```yaml
 # The name of the time series to output to. Must be a valid metric name.
 record: <string>
@@ -198,23 +252,22 @@ labels:
 
 For recording rules to work `-remoteWrite.url` must be specified.
 
-
 ### Alerts state on restarts
 
 `vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after restart of `vmalert`
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
+
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
-into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and maybe queried from VM just as any other time series.
-The state is stored to the configured address on every rule evaluation.
+  into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
+  These are regular time series and maybe queried from VM just as any other time series.
+  The state is stored to the configured address on every rule evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
-from configured address by querying time series with name `ALERTS_FOR_STATE`.
+  from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
 Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
 or received state doesn't match current `vmalert` rules configuration.
 
-
 ### Multitenancy
 
 There are the following approaches exist for alerting and recording rules across
@@ -259,12 +312,13 @@ tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
 ### Topology examples
 
-The following sections are showing how `vmalert` may be used and configured 
-for different scenarios. 
+The following sections are showing how `vmalert` may be used and configured
+for different scenarios.
+
+Please note, not all flags in examples are required:
 
-Please note, not all flags in examples are required: 
 * `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
-you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+  you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
 * `-notifier.url` is optional and is needed only if you have alerting rules.
 
 #### Single-node VictoriaMetrics
@@ -273,6 +327,7 @@ The simplest configuration where one single-node VM server is used for
 rules execution, storing recording rules results and alerts state.
 
 `vmalert` configuration flags:
+
 ```
 ./bin/vmalert -rule=rules.yml  \                    # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
@@ -283,20 +338,20 @@ rules execution, storing recording rules results and alerts state.
 
 <img alt="vmalert single" width="500" src="vmalert_single.png">
 
-
 #### Cluster VictoriaMetrics
 
 In [cluster mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html)
 VictoriaMetrics has separate components for writing and reading path:
 `vminsert` and `vmselect` components respectively. `vmselect` is used for executing rules expressions
 and `vminsert` is used to persist recording rules results and alerts state.
-Cluster mode could have multiple `vminsert` and `vmselect` components. 
+Cluster mode could have multiple `vminsert` and `vmselect` components.
 
 `vmalert` configuration flags:
+
 ```
 ./bin/vmalert -rule=rules.yml  \                                # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for executing rules expressions
-    -remoteWrite.url=http://vminsert:8480/insert/0/prometheuss  # vminsert addr to persist alerts state and recording rules results
+    -remoteWrite.url=http://vminsert:8480/insert/0/prometheus   # vminsert addr to persist alerts state and recording rules results
     -remoteRead.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for restoring alerts state after restart
     -notifier.url=http://alertmanager:9093                      # AlertManager addr to send alerts when they trigger
 ```
@@ -315,6 +370,7 @@ the same destinations, and send alert notifications to multiple configured
 Alertmanagers.
 
 `vmalert` configuration flags:
+
 ```
 ./bin/vmalert -rule=rules.yml \                   # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
@@ -328,6 +384,7 @@ Alertmanagers.
 
 To avoid recording rules results and alerts state duplication in VictoriaMetrics server
 don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert's `evaluation_interval`.
 
 Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
 all `vmalert`s are having the same config.
@@ -335,9 +392,8 @@ all `vmalert`s are having the same config.
 Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
 for Alertmanagers for better reliability.
 
-This example uses single-node VM server for the sake of simplicity. 
-Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. 
-
+This example uses single-node VM server for the sake of simplicity.
+Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed.
 
 #### Downsampling and aggregation via vmalert
 
@@ -349,10 +405,11 @@ recording rules to process raw data from "hot" cluster (by applying additional t
 or reducing resolution) and push results to "cold" cluster.
 
 `vmalert` configuration flags:
+
 ```
 ./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
-    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheuss   # vminsert addr to persist recording rules results
+    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus    # vminsert addr to persist recording rules results
 ```
 
 <img alt="vmalert multi cluster" src="vmalert_multicluster.png">
@@ -363,19 +420,18 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec
 
 See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
 
-
 ### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+
 * `http://<vmalert-addr>` - UI;
-* `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
+* `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
-* `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status" ` - get alert status by ID.
-Used as alert source in AlertManager.
+* `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status"` - get alert status by ID.
+  Used as alert source in AlertManager.
 * `http://<vmalert-addr>/metrics` - application metrics.
 * `http://<vmalert-addr>/-/reload` - hot configuration reload.
 
-
 ## Graphite
 
 vmalert sends requests to `<-datasource.url>/render?format=json` during evaluation of alerting and recording rules
@@ -395,6 +451,7 @@ data source for backfilling.
 
 In `replay` mode vmalert works as a cli-tool and exits immediately after work is done.
 To run vmalert in `replay` mode:
+
 ```
 ./bin/vmalert -rule=path/to/your.rules \        # path to files with rules you usually use with vmalert
     -datasource.url=http://localhost:8428 \     # PromQL/MetricsQL compatible datasource
@@ -404,6 +461,7 @@ To run vmalert in `replay` mode:
 ```
 
 The output of the command will look like the following:
+
 ```
 Replay mode:
 from:   2021-05-11 07:21:43 +0000 UTC   # set by -replay.timeFrom
@@ -447,9 +505,11 @@ The result of recording rules `replay` should match with results of normal rules
 
 The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
 To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
+
 ```
 ALERTS{alertname="your_alertname", alertstate="firing"}
 ```
+
 Execute the query against storage which was used for `-remoteWrite.url` during the `replay`.
 
 ### Additional configuration
@@ -457,14 +517,17 @@ Execute the query against storage which was used for `-remoteWrite.url` during t
 There are following non-required `replay` flags:
 
 * `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
-In two words, it affects the max time range for every `/query_range` request. The higher the value,
-the fewer requests will be issued during `replay`.
+  In two words, it affects the max time range for every `/query_range` request. The higher the value,
+  the fewer requests will be issued during `replay`.
 * `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
-per rule before giving up.
+  per rule before giving up.
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
-(rules which depend on each other) rules. It is expected, that remote storage will be able to persist
-previously accepted data during the delay, so data will be available for the subsequent queries.
-Keep it equal or bigger than `-remoteWrite.flushInterval`.
+  (rules which depend on each other) rules. It is expected, that remote storage will be able to persist
+  previously accepted data during the delay, so data will be available for the subsequent queries.
+  Keep it equal or bigger than `-remoteWrite.flushInterval`.
+* `replay.disableProgressBar` - whether to disable progress bar which shows progress work.
+  Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
+  It could break logs parsing by external system and generate additional load on it.
 
 See full description for these flags in `./vmalert --help`.
 
@@ -473,7 +536,6 @@ See full description for these flags in `./vmalert --help`.
 * Graphite engine isn't supported yet;
 * `query` template function is disabled for performance reasons (might be changed in future);
 
-
 ## Monitoring
 
 `vmalert` exports various metrics in Prometheus exposition format at `http://vmalert-host:8880/metrics` page.
@@ -484,7 +546,6 @@ Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/1495
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.
 
-
 ## Configuration
 
 ### Flags
@@ -493,254 +554,329 @@ Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.
 
 The shortlist of configuration flags is the following:
+
 ```
   -clusterMode
-    	If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -configCheckInterval duration
-    	Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
+     Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
   -datasource.appendTypePrefix
-    	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
+     Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
   -datasource.basicAuth.password string
-    	Optional basic auth password for -datasource.url
+     Optional basic auth password for -datasource.url
   -datasource.basicAuth.passwordFile string
-    	Optional path to basic auth password to use for -datasource.url
+     Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
-    	Optional basic auth username for -datasource.url
+     Optional basic auth username for -datasource.url
   -datasource.bearerToken string
-    	Optional bearer auth token to use for -datasource.url.
+     Optional bearer auth token to use for -datasource.url.
   -datasource.bearerTokenFile string
-    	Optional path to bearer token file to use for -datasource.url.
+     Optional path to bearer token file to use for -datasource.url.
+  -datasource.disableKeepAlive
+     Whether to disable long-lived connections to the datasource. If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.
   -datasource.lookback duration
-    	Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+     Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
   -datasource.maxIdleConnections int
-    	Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
+     Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
+  -datasource.oauth2.clientID string
+     Optional OAuth2 clientID to use for -datasource.url. 
+  -datasource.oauth2.clientSecret string
+     Optional OAuth2 clientSecret to use for -datasource.url.
+  -datasource.oauth2.clientSecretFile string
+     Optional OAuth2 clientSecretFile to use for -datasource.url. 
+  -datasource.oauth2.scopes string
+     Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
+  -datasource.oauth2.tokenUrl string
+     Optional OAuth2 tokenURL to use for -datasource.url.
   -datasource.queryStep duration
-    	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+     queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+  -datasource.queryTimeAlignment
+     Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
-    	Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+     Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
   -datasource.tlsCAFile string
-    	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
+     Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
-    	Optional path to client-side TLS certificate file to use when connecting to -datasource.url
+     Optional path to client-side TLS certificate file to use when connecting to -datasource.url
   -datasource.tlsInsecureSkipVerify
-    	Whether to skip tls verification when connecting to -datasource.url
+     Whether to skip tls verification when connecting to -datasource.url
   -datasource.tlsKeyFile string
-    	Optional path to client-side TLS certificate key to use when connecting to -datasource.url
+     Optional path to client-side TLS certificate key to use when connecting to -datasource.url
   -datasource.tlsServerName string
-    	Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
+     Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-    	VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+     VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend
   -defaultTenant.graphite string
-    	Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -defaultTenant.prometheus string
-    	Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -disableAlertgroupLabel
-    	Whether to disable adding group's Name as label to generated alerts and time series.
+     Whether to disable adding group's Name as label to generated alerts and time series.
   -dryRun -rule
-    	Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
+     Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
-    	Prefix for environment variables if -envflag.enable is set
+     Prefix for environment variables if -envflag.enable is set
   -eula
-    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -evaluationInterval duration
-    	How often to evaluate the rules (default 1m0s)
+     How often to evaluate the rules (default 1m0s)
   -external.alert.source string
-    	External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-    	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
+     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
+     eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
   -external.label array
-    	Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
+     Supports an array of values separated by comma or specified via multiple flags.
   -external.url string
-    	External URL is used as alert's source for sent alerts to the notifier
+     External URL is used as alert's source for sent alerts to the notifier
   -fs.disableMmap
-    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+     Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+     Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
-    	Timeout for incoming idle http connections (default 1m0s)
+     Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
-    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+     An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+     Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-    	Address to listen for http connections (default ":8880")
+     Address to listen for http connections (default ":8880")
   -loggerDisableTimestamps
-    	Whether to disable writing timestamps in logs
+     Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
-    	Format for logs. Possible values: default, json (default "default")
+     Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+     Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
-    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+     Output for the logs. Supported values: stderr, stdout (default "stderr")
   -loggerTimezone string
-    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+     Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
-    	Optional basic auth password for -notifier.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth password for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.basicAuth.passwordFile array
-    	Optional path to basic auth password file for -notifier.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to basic auth password file for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.basicAuth.username array
-    	Optional basic auth username for -notifier.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional basic auth username for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.bearerToken array
+     Optional bearer token for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.bearerTokenFile array
+     Optional path to bearer token file for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.config string
-    	Path to configuration file for notifiers
+     Path to configuration file for notifiers
+  -notifier.oauth2.clientID array
+     Optional OAuth2 clientID to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.clientSecret array
+     Optional OAuth2 clientSecret to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.clientSecretFile array
+     Optional OAuth2 clientSecretFile to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.scopes array
+     Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.oauth2.tokenUrl array
+     Optional OAuth2 tokenURL to use for -notifier.url. If multiple args are set, then they are applied independently for the corresponding -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.suppressDuplicateTargetErrors
-    	Whether to suppress 'duplicate target' errors during discovery
+     Whether to suppress 'duplicate target' errors during discovery
   -notifier.tlsCAFile array
-    	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsCertFile array
-    	Optional path to client-side TLS certificate file to use when connecting to -notifier.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to client-side TLS certificate file to use when connecting to -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsInsecureSkipVerify array
-    	Whether to skip tls verification when connecting to -notifier.url
-    	Supports array of values separated by comma or specified via multiple flags.
+     Whether to skip tls verification when connecting to -notifier.url
+     Supports array of values separated by comma or specified via multiple flags.
   -notifier.tlsKeyFile array
-    	Optional path to client-side TLS certificate key to use when connecting to -notifier.url
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional path to client-side TLS certificate key to use when connecting to -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsServerName array
-    	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
+     Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-    	Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
+     Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.consul.waitTime duration
-    	Wait time used by Consul service discovery. Default value is used if not set
+     Wait time used by Consul service discovery. Default value is used if not set
   -promscrape.consulSDCheckInterval duration
-    	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
   -promscrape.discovery.concurrency int
-    	The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
+     The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
   -promscrape.discovery.concurrentWaitTime duration
-    	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
+     The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
+  -promscrape.dnsSDCheckInterval duration
+     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
   -remoteRead.basicAuth.password string
-    	Optional basic auth password for -remoteRead.url
+     Optional basic auth password for -remoteRead.url
   -remoteRead.basicAuth.passwordFile string
-    	Optional path to basic auth password to use for -remoteRead.url
+     Optional path to basic auth password to use for -remoteRead.url
   -remoteRead.basicAuth.username string
-    	Optional basic auth username for -remoteRead.url
+     Optional basic auth username for -remoteRead.url
   -remoteRead.bearerToken string
-    	Optional bearer auth token to use for -remoteRead.url.
+     Optional bearer auth token to use for -remoteRead.url.
   -remoteRead.bearerTokenFile string
-    	Optional path to bearer token file to use for -remoteRead.url.
+     Optional path to bearer token file to use for -remoteRead.url.
   -remoteRead.disablePathAppend
-    	Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.
+     Whether to disable automatic appending of '/api/v1/query' path to the configured -datasource.url and -remoteRead.url
   -remoteRead.ignoreRestoreErrors
-    	Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
+     Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
   -remoteRead.lookback duration
-    	Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+     Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteRead.oauth2.clientID string
+     Optional OAuth2 clientID to use for -remoteRead.url.
+  -remoteRead.oauth2.clientSecret string
+     Optional OAuth2 clientSecret to use for -remoteRead.url.
+  -remoteRead.oauth2.clientSecretFile string
+     Optional OAuth2 clientSecretFile to use for -remoteRead.url.
+  -remoteRead.oauth2.scopes string
+     Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
+  -remoteRead.oauth2.tokenUrl string
+     Optional OAuth2 tokenURL to use for -remoteRead.url. 
   -remoteRead.tlsCAFile string
-    	Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
+     Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
-    	Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url
+     Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url
   -remoteRead.tlsInsecureSkipVerify
-    	Whether to skip tls verification when connecting to -remoteRead.url
+     Whether to skip tls verification when connecting to -remoteRead.url
   -remoteRead.tlsKeyFile string
-    	Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url
+     Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url
   -remoteRead.tlsServerName string
-    	Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
+     Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-    	Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
+     Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
   -remoteWrite.basicAuth.password string
-    	Optional basic auth password for -remoteWrite.url
+     Optional basic auth password for -remoteWrite.url
   -remoteWrite.basicAuth.passwordFile string
-    	Optional path to basic auth password to use for -remoteWrite.url
+     Optional path to basic auth password to use for -remoteWrite.url
   -remoteWrite.basicAuth.username string
-    	Optional basic auth username for -remoteWrite.url
+     Optional basic auth username for -remoteWrite.url
   -remoteWrite.bearerToken string
-    	Optional bearer auth token to use for -remoteWrite.url.
+     Optional bearer auth token to use for -remoteWrite.url.
   -remoteWrite.bearerTokenFile string
-    	Optional path to bearer token file to use for -remoteWrite.url.
+     Optional path to bearer token file to use for -remoteWrite.url.
   -remoteWrite.concurrency int
-    	Defines number of writers for concurrent writing into remote querier (default 1)
+     Defines number of writers for concurrent writing into remote querier (default 1)
   -remoteWrite.disablePathAppend
-    	Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
+     Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
   -remoteWrite.flushInterval duration
-    	Defines interval of flushes to remote write endpoint (default 5s)
+     Defines interval of flushes to remote write endpoint (default 5s)
   -remoteWrite.maxBatchSize int
-    	Defines defines max number of timeseries to be flushed at once (default 1000)
+     Defines defines max number of timeseries to be flushed at once (default 1000)
   -remoteWrite.maxQueueSize int
-    	Defines the max number of pending datapoints to remote write endpoint (default 100000)
+     Defines the max number of pending datapoints to remote write endpoint (default 100000)
+  -remoteWrite.oauth2.clientID string
+     Optional OAuth2 clientID to use for -remoteWrite.url.
+  -remoteWrite.oauth2.clientSecret string
+     Optional OAuth2 clientSecret to use for -remoteWrite.url.
+  -remoteWrite.oauth2.clientSecretFile string
+     Optional OAuth2 clientSecretFile to use for -remoteWrite.url.
+  -remoteWrite.oauth2.scopes string
+     Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
+  -remoteWrite.oauth2.tokenUrl string
+     Optional OAuth2 tokenURL to use for -notifier.url.
   -remoteWrite.tlsCAFile string
-    	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
+     Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
-    	Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url
+     Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url
   -remoteWrite.tlsInsecureSkipVerify
-    	Whether to skip tls verification when connecting to -remoteWrite.url
+     Whether to skip tls verification when connecting to -remoteWrite.url
   -remoteWrite.tlsKeyFile string
-    	Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url
+     Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url
   -remoteWrite.tlsServerName string
-    	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
+     Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-    	Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+  -replay.disableProgressBar
+     Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int
-    	Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
+     Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
-    	Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
+     Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration
-    	Delay between rules evaluation within the group. Could be important if there are chained rules inside of the groupand processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule.Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
+     Delay between rules evaluation within the group. Could be important if there are chained rules inside of the groupand processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule.Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
   -replay.timeFrom string
-    	The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+     The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
   -replay.timeTo string
-    	The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+     The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
-    	Path to the file with alert rules.
-    	Supports patterns. Flag can be specified multiple times.
-    	Examples:
-    	 -rule="/path/to/file". Path to a single file with alerting rules
-    	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-    	absolute path to all .yaml files in root.
-    	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
-    	Supports an array of values separated by comma or specified via multiple flags.
+     Path to the file with alert rules.
+     Supports patterns. Flag can be specified multiple times.
+     Examples:
+      -rule="/path/to/file". Path to a single file with alerting rules
+      -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
+     absolute path to all .yaml files in root.
+     Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+     Supports an array of values separated by comma or specified via multiple flags.
   -rule.configCheckInterval duration
-    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
+     Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
   -rule.maxResolveDuration duration
-    	Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
+     Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
+  -rule.resendDelay duration
+     Minimum amount of time to wait before resending an alert to notifier
+  -rule.templates array
+     Path or glob pattern to location with go template definitions
+      for rules annotations templating. Flag can be specified multiple times.
+     Examples:
+      -rule.templates="/path/to/file". Path to a single file with go templates
+      -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+     absolute path to all .tpl files in root.
+     Supports an array of values separated by comma or specified via multiple flags.
   -rule.validateExpressions
-    	Whether to validate rules expressions via MetricsQL engine (default true)
+     Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
-    	Whether to validate annotation and label templates (default true)
+     Whether to validate annotation and label templates (default true)
   -tls
-    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+     Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+  -tlsCipherSuites array
+     Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
+     Supports an array of values separated by comma or specified via multiple flags.
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
-    	Show VictoriaMetrics version
+     Show VictoriaMetrics version
 ```
 
 ### Hot config reload
+
 `vmalert` supports "hot" config reload via the following methods:
+
 * send SIGHUP signal to `vmalert` process;
 * send GET request to `/-/reload` endpoint;
 * configure `-configCheckInterval` flag for periodic reload
-on config change.
+  on config change.
 
 ### URL params
 
@@ -749,6 +885,7 @@ just add them in address: `-datasource.url=http://localhost:8428?nocache=1`.
 
 To set additional URL params for specific [group of rules](#Groups) modify
 the `params` group:
+
 ```yaml
 groups:
   - name: TestGroup
@@ -756,6 +893,7 @@ groups:
       denyPartialResponse: ["true"]
       extra_label: ["env=dev"]
 ```
+
 Please note, `params` are used only for executing rules expressions (requests to `datasource.url`).
 If there would be a conflict between URL params set in `datasource.url` flag and params in group definition
 the latter will have higher priority.
@@ -763,15 +901,18 @@ the latter will have higher priority.
 ### Notifier configuration file
 
 Notifier also supports configuration via file specified with flag `notifier.config`:
+
 ```
 ./bin/vmalert -rule=app/vmalert/config/testdata/rules.good.rules \
-		-datasource.url=http://localhost:8428 \
-		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
+  -datasource.url=http://localhost:8428 \
+  -notifier.config=app/vmalert/notifier/testdata/consul.good.yaml
 ```
 
-The configuration file allows to configure static notifiers or discover notifiers via 
-[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config).
+The configuration file allows to configure static notifiers, discover notifiers via
+[Consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
+and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config):
 For example:
+
 ```
 static_configs: 
   - targets:
@@ -782,12 +923,19 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - alertmanager
+      
+dns_sd_configs:
+  - names:
+      - my.domain.com
+    type: 'A'
+    port: 9093
 ```
 
 The list of configured or discovered Notifiers can be explored via [UI](#Web).
 
 The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go)
 is the following:
+
 ```
 # Per-target Notifier timeout when pushing alerts.
 [ timeout: <duration> | default = 10s ]
@@ -832,17 +980,26 @@ static_configs:
 consul_sd_configs:
   [ - <consul_sd_config> ... ]
 
-# List of relabel configurations.
+# List of DNS service discovery configurations.
+# See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+dns_sd_configs:
+  [ - <dns_sd_config> ... ]
+
+# List of relabel configurations for entities discovered via service discovery.
 # Supports the same relabeling features as the rest of VictoriaMetrics components.
 # See https://docs.victoriametrics.com/vmagent.html#relabeling
 relabel_configs:
   [ - <relabel_config> ... ]
 
+# List of relabel configurations for alert labels sent via Notifier.
+# Supports the same relabeling features as the rest of VictoriaMetrics components.
+# See https://docs.victoriametrics.com/vmagent.html#relabeling
+alert_relabel_configs:
+  [ - <relabel_config> ... ]
 ```
 
 The configuration file can be [hot-reloaded](#hot-config-reload).
 
-
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.
@@ -853,8 +1010,8 @@ software. Please keep simplicity as the main priority.
 
 It is recommended using
 [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-- `vmalert` is located in `vmutils-*` archives there.
 
+* `vmalert` is located in `vmutils-*` archives there.
 
 ### Development build
 
@@ -868,7 +1025,6 @@ It is recommended using
 2. Run `make vmalert-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-prod` binary and puts it into the `bin` folder.
 
-
 ### ARM build
 
 ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).

app/vmalert/alerting.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -38,6 +39,8 @@ type AlertingRule struct {
 	alerts map[uint64]*notifier.Alert
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -139,6 +142,53 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+type labelSet struct {
+	// origin labels from series
+	// used for templating
+	origin map[string]string
+	// processed labels with additional data
+	// used as Alert labels
+	processed map[string]string
+}
+
+// toLabels converts labels from given Metric
+// to labelSet which contains original and processed labels.
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
+	ls := &labelSet{
+		origin:    make(map[string]string, len(m.Labels)),
+		processed: make(map[string]string),
+	}
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		ls.origin[l.Name] = l.Value
+		ls.processed[l.Name] = l.Value
+	}
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+		Labels: ls.origin,
+		Value:  m.Values[0],
+		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
+	}
+	for k, v := range extraLabels {
+		ls.processed[k] = v
+	}
+
+	// set additional labels to identify group and rule name
+	if ar.Name != "" {
+		ls.processed[alertNameLabel] = ar.Name
+	}
+	if !*disableAlertGroupLabel && ar.GroupName != "" {
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+	}
+	return ls, nil
+}
+
 // ExecRange executes alerting rule on the given time range similarly to Exec.
 // It doesn't update internal states of the Rule and meant to be used just
 // to get time series for backfilling.
@@ -153,24 +203,7 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
-		// set additional labels to identify group and rule Name
-		if ar.Name != "" {
-			s.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			s.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(s, qFn, ar)
-		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
-		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			s.SetLabel(k, v)
-		}
-		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
 		}
@@ -189,9 +222,10 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
-				a.Start = at
-			} else if at.Sub(a.Start) >= ar.For {
+				a.ActiveAt = at
+			} else if at.Sub(a.ActiveAt) >= ar.For {
 				a.State = notifier.StateFiring
+				a.Start = at
 			}
 			prevT = at
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
@@ -200,15 +234,21 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 	return result, nil
 }
 
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequently repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := ar.q.Query(ctx, ar.Expr)
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
 	ar.lastExecError = err
-	ar.lastExecTime = time.Now()
 	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
@@ -216,59 +256,55 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
-		if a.State == notifier.StateInactive {
+		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
 			delete(ar.alerts, h)
 		}
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			m.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			m.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(m, qFn, ar)
+		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
 			return nil, fmt.Errorf("failed to expand labels: %s", err)
 		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			m.SetLabel(k, v)
-		}
-		h := hash(m)
+		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
 			// conflicting with the metric labels
-			return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
+			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, ar.lastExecError
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
+			if a.State == notifier.StateInactive {
+				// alert could be in inactive state for resolvedRetention
+				// so when we again receive metrics for it - we switch it
+				// back to notifier.StatePending
+				a.State = notifier.StatePending
+				a.ActiveAt = ts
+			}
 			if a.Value != m.Values[0] {
 				// update Value field with latest value
 				a.Value = m.Values[0]
 				// and re-exec template since Value can be used
 				// in annotations
-				a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 				if err != nil {
 					return nil, err
 				}
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ar.lastExecTime, qFn)
+		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
 		if err != nil {
 			ar.lastExecError = err
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = h
 		a.State = notifier.StatePending
+		a.ActiveAt = ts
 		ar.alerts[h] = a
 	}
 
@@ -282,28 +318,19 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 				delete(ar.alerts, h)
 				continue
 			}
-			a.State = notifier.StateInactive
+			if a.State == notifier.StateFiring {
+				a.State = notifier.StateInactive
+				a.ResolvedAt = ts
+			}
 			continue
 		}
-		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
+			a.Start = ts
 			alertsFired.Inc()
 		}
 	}
-	return ar.toTimeSeries(ar.lastExecTime.Unix()), nil
-}
-
-func expandLabels(m datasource.Metric, q notifier.QueryFn, ar *AlertingRule) (map[string]string, error) {
-	metricLabels := make(map[string]string)
-	for _, l := range m.Labels {
-		metricLabels[l.Name] = l.Value
-	}
-	tpl := notifier.AlertTplData{
-		Labels: metricLabels,
-		Value:  m.Values[0],
-		Expr:   ar.Expr,
-	}
-	return notifier.ExecTemplate(q, ar.Labels, tpl)
+	return ar.toTimeSeries(ts.Unix()), nil
 }
 
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
@@ -336,42 +363,43 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 }
 
 // TODO: consider hashing algorithm in VM
-func hash(m datasource.Metric) uint64 {
+func hash(labels map[string]string) uint64 {
 	hash := fnv.New64a()
-	labels := m.Labels
-	sort.Slice(labels, func(i, j int) bool {
-		return labels[i].Name < labels[j].Name
-	})
-	for _, l := range labels {
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
 		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
+		if k == "__name__" {
 			continue
 		}
-		hash.Write([]byte(l.Name))
-		hash.Write([]byte(l.Value))
+		name, value := k, labels[k]
+		hash.Write([]byte(name))
+		hash.Write([]byte(value))
 		hash.Write([]byte("\xff"))
 	}
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
-	a := &notifier.Alert{
-		GroupID: ar.GroupID,
-		Name:    ar.Name,
-		Labels:  map[string]string{},
-		Value:   m.Values[0],
-		Start:   start,
-		Expr:    ar.Expr,
-	}
-	for _, l := range m.Labels {
-		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
-			continue
-		}
-		a.Labels[l.Name] = l.Value
-	}
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
 	var err error
-	a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+	}
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 	return a, err
 }
 
@@ -386,34 +414,54 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// RuleAPI returns Rule representation in form
-// of APIAlertingRule
-func (ar *AlertingRule) RuleAPI() APIAlertingRule {
-	var lastErr string
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
 	if ar.lastExecError != nil {
-		lastErr = ar.lastExecError.Error()
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
 	}
-	return APIAlertingRule{
-		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", ar.ID()),
-		GroupID:     fmt.Sprintf("%d", ar.GroupID),
-		Type:        ar.Type.String(),
-		Name:        ar.Name,
-		Expression:  ar.Expr,
-		For:         ar.For.String(),
-		LastError:   lastErr,
-		LastSamples: ar.lastExecSamples,
-		LastExec:    ar.lastExecTime,
-		Labels:      ar.Labels,
-		Annotations: ar.Annotations,
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
 	}
+	return r
 }
 
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
 	ar.mu.RLock()
 	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
 		alerts = append(alerts, ar.newAlertAPI(*a))
 	}
 	ar.mu.RUnlock()
@@ -432,7 +480,7 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Labels:      a.Labels,
 		Annotations: a.Annotations,
 		State:       a.State.String(),
-		ActiveAt:    a.Start,
+		ActiveAt:    a.ActiveAt,
 		Restored:    a.Restored,
 		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
@@ -458,7 +506,7 @@ const (
 	alertGroupNameLabel = "alertgroup"
 )
 
-// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+// alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	tss = append(tss, alertToTimeSeries(a, timestamp))
@@ -486,11 +534,11 @@ func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.Time
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
+	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
 // Restore restores the state of active alerts basing on previously written time series.
-// Restore restores only Start field. Field State will be always Pending and supposed
+// Restore restores only ActiveAt field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
 func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
@@ -498,7 +546,8 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		return fmt.Errorf("querier is nil")
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	ts := time.Now()
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 
 	// account for external labels in filter
 	var labelsFilter string
@@ -511,21 +560,61 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr)
+	qMetrics, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return err
 	}
 
 	for _, m := range qMetrics {
-		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
+		ls := &labelSet{
+			origin:    make(map[string]string, len(m.Labels)),
+			processed: make(map[string]string, len(m.Labels)),
+		}
+		for _, l := range m.Labels {
+			if l.Name == "__name__" {
+				continue
+			}
+			ls.origin[l.Name] = l.Value
+			ls.processed[l.Name] = l.Value
+		}
+		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
-		a.ID = hash(m)
+		a.ID = hash(ls.processed)
 		a.State = notifier.StatePending
 		a.Restored = true
 		ar.alerts[a.ID] = a
-		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
 	return nil
 }
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	needsSending := func(a *notifier.Alert) bool {
+		if a.State == notifier.StatePending {
+			return false
+		}
+		if a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(ts)
+	}
+
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		if !needsSending(a) {
+			continue
+		}
+		a.End = ts.Add(resolveDuration)
+		if a.State == notifier.StateInactive {
+			a.End = a.ResolvedAt
+		}
+		a.LastSent = ts
+		alerts = append(alerts, *a)
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -60,7 +61,7 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 		},
 		{
 			newTestAlertingRule("for", time.Second),
-			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -75,7 +76,7 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 		},
 		{
 			newTestAlertingRule("for pending", 10*time.Second),
-			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StatePending, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -168,7 +169,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -176,7 +177,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			nil,
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -216,8 +219,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 			},
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
-			// 3: fire third alert, set second inactive, delete first one
+			// 3: fire third alert, set second inactive
 			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
@@ -300,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -311,10 +315,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 			}
 			expAlerts := make(map[uint64]*notifier.Alert)
 			for _, ta := range tc.expAlerts {
-				labels := ta.labels
-				labels = append(labels, alertNameLabel)
-				labels = append(labels, tc.rule.Name)
-				h := hash(metricWithLabels(t, labels...))
+				labels := make(map[string]string)
+				for i := 0; i < len(ta.labels); i += 2 {
+					k, v := ta.labels[i], ta.labels[i+1]
+					labels[k] = v
+				}
+				labels[alertNameLabel] = tc.rule.Name
+				h := hash(labels)
 				expAlerts[h] = ta.alert
 			}
 			for key, exp := range expAlerts {
@@ -379,9 +386,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(3, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
 			},
 		},
 		{
@@ -390,9 +397,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 		},
 		{
@@ -401,11 +408,11 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(5, 0)},
-				{State: notifier.StatePending, Start: time.Unix(20, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
 			},
 		},
 		{
@@ -417,15 +424,15 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 				//
-				{State: notifier.StatePending, Start: time.Unix(1, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
-				{State: notifier.StatePending, Start: time.Unix(5, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
@@ -478,7 +485,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 						a.Labels = make(map[string]string)
 					}
 					a.Labels[alertNameLabel] = tc.rule.Name
-					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
+					expTS = append(expTS, tc.rule.alertToTimeSeries(a, timestamp)...)
 					j++
 				}
 			}
@@ -509,8 +516,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(nil): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -525,13 +532,13 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					alertNameLabel, "metric labels",
-					alertGroupNameLabel, "groupID",
-					"foo", "bar",
-					"namespace", "baz",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					alertNameLabel:      "metric labels",
+					alertGroupNameLabel: "groupID",
+					"foo":               "bar",
+					"namespace":         "baz",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -546,12 +553,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					"foo", "bar",
-					"namespace", "baz",
-					"source", "vm",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					"foo":       "bar",
+					"namespace": "baz",
+					"source":    "vm",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -571,12 +578,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(2 * time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(3 * time.Hour)},
+				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(2 * time.Hour)},
+				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(3 * time.Hour)},
 			},
 		},
 	}
@@ -601,8 +608,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 				if got.State != exp.State {
 					t.Fatalf("expected state %d; got %d", exp.State, got.State)
 				}
-				if got.Start != exp.Start {
-					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				if got.ActiveAt != exp.ActiveAt {
+					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
 				}
 			}
 		})
@@ -617,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO())
+	_, err := ar.Exec(context.TODO(), time.Now())
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now())
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -633,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -655,7 +662,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
 						alertNameLabel: "common",
@@ -663,7 +670,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
 						alertNameLabel: "common",
@@ -678,77 +685,70 @@ func TestAlertingRule_Template(t *testing.T) {
 				Name: "override label",
 				Labels: map[string]string{
 					"instance": "{{ $labels.instance }}",
-					"region":   "east",
 				},
 				Annotations: map[string]string{
-					"summary":     `Too high connection number for "{{ $labels.instance }}" for region {{ $labels.region }}`,
-					"description": `It is {{ $value }} connections for "{{ $labels.instance }}"`,
+					"summary":     `Too high connection number for "{{ $labels.instance }}"`,
+					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "instance", "foo"),
-				metricWithValueAndLabels(t, 10, "instance", "bar"),
+				metricWithValueAndLabels(t, 2, "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
 					Labels: map[string]string{
 						alertNameLabel: "override label",
 						"instance":     "foo",
-						"region":       "east",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "foo" for region east`,
-						"description": `It is 2 connections for "foo"`,
+						"summary":     `Too high connection number for "foo"`,
+						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
 					Labels: map[string]string{
 						alertNameLabel: "override label",
 						"instance":     "bar",
-						"region":       "east",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "bar" for region east`,
-						"description": `It is 10 connections for "bar"`,
+						"summary":     `Too high connection number for "bar"`,
+						"description": `override: It is 10 connections for "bar"`,
 					},
 				},
 			},
 		},
 		{
 			&AlertingRule{
-				Name:      "ExtraTemplating",
+				Name:      "OriginLabels",
 				GroupName: "Testing",
 				Labels: map[string]string{
-					"name":     "alert_{{ $labels.alertname }}",
-					"group":    "group_{{ $labels.alertgroup }}",
 					"instance": "{{ $labels.instance }}",
 				},
 				Annotations: map[string]string{
-					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
-					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 1, "instance", "foo"),
+				metricWithValueAndLabels(t, 1,
+					alertNameLabel, "originAlertname",
+					alertGroupNameLabel, "originGroupname",
+					"instance", "foo"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
-					"name", "alert_ExtraTemplating",
-					alertGroupNameLabel, "Testing",
-					"group", "group_Testing",
-					"instance", "foo")): {
+				hash(map[string]string{
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo"}): {
 					Labels: map[string]string{
-						alertNameLabel:      "ExtraTemplating",
-						"name":              "alert_ExtraTemplating",
+						alertNameLabel:      "OriginLabels",
 						alertGroupNameLabel: "Testing",
-						"group":             "group_Testing",
 						"instance":          "foo",
 					},
 					Annotations: map[string]string{
-						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
-						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
 					},
 				},
 			},
@@ -761,7 +761,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -781,6 +781,86 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 }
 
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+}
+
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)

app/vmalert/config/config.go

@@ -25,10 +25,10 @@ import (
 type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
-	Name        string             `yaml:"name"`
-	Interval    promutils.Duration `yaml:"interval"`
-	Rules       []Rule             `yaml:"rules"`
-	Concurrency int                `yaml:"concurrency"`
+	Name        string              `yaml:"name"`
+	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Rules       []Rule              `yaml:"rules"`
+	Concurrency int                 `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
@@ -127,12 +127,12 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Record      string             `yaml:"record,omitempty"`
-	Alert       string             `yaml:"alert,omitempty"`
-	Expr        string             `yaml:"expr"`
-	For         promutils.Duration `yaml:"for"`
-	Labels      map[string]string  `yaml:"labels,omitempty"`
-	Annotations map[string]string  `yaml:"annotations,omitempty"`
+	Record      string              `yaml:"record,omitempty"`
+	Alert       string              `yaml:"alert,omitempty"`
+	Expr        string              `yaml:"expr"`
+	For         *promutils.Duration `yaml:"for,omitempty"`
+	Labels      map[string]string   `yaml:"labels,omitempty"`
+	Annotations map[string]string   `yaml:"annotations,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`

app/vmalert/config/config_test.go

@@ -10,18 +10,19 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
 func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
 		t.Errorf("error parsing files %s", err)
 	}
 }
@@ -32,7 +33,7 @@ func TestParseBad(t *testing.T) {
 		expErr string
 	}{
 		{
-			[]string{"testdata/rules0-bad.rules"},
+			[]string{"testdata/rules/rules0-bad.rules"},
 			"unexpected token",
 		},
 		{
@@ -56,7 +57,7 @@ func TestParseBad(t *testing.T) {
 			"either `record` or `alert` must be set",
 		},
 		{
-			[]string{"testdata/rules1-bad.rules"},
+			[]string{"testdata/rules/rules1-bad.rules"},
 			"bad graphite expr",
 		},
 	}

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -2,8 +2,6 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
-    extra_filter_labels: # deprecated param, use `params` instead
-        job: victoriametrics
     params:
       denyPartialResponse: ["true"]
       extra_label: ["env=dev"]
@@ -49,4 +47,12 @@ groups:
         expr: |2
             sum(code:requests:rate5m{code="200"})
           /
-            sum(code:requests:rate5m)
+            sum(code:requests:rate5m)
+      - record: code:requests:slo
+        labels:
+           recording: true
+        expr: 0.95
+      - record: time:current
+        labels:
+           recording: true
+        expr: time()

app/vmalert/config/testdata/rules/rules_interval_good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupTest
+    interval: 1s
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/config/testdata/templates/templates0-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template0" }}
+Visit {{ externalURL }}
+{{ end }}

app/vmalert/config/testdata/templates/templates1-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template1" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates2-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template2" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates3-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ printf "%s to %s!" "welcome" "hell" | toUpper }}
+{{ end }}

app/vmalert/config/testdata/templates/templates4-good-tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ 1230912039102391023.0 | humanizeDuration }}
+{{ end }}

app/vmalert/datasource/datasource.go

@@ -8,7 +8,7 @@ import (
 
 // Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string) ([]Metric, error)
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, error)
 	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
 }
 

app/vmalert/datasource/init.go

@@ -12,13 +12,15 @@ import (
 
 var (
 	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
-		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix      = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend")
+	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
 	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
-	bearerToken           = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
-	bearerTokenFile       = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
+
+	bearerToken     = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
@@ -26,12 +28,22 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
+	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
+	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+
 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
 		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
 		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
-	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
@@ -52,6 +64,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
@@ -64,7 +77,10 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}
 
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/datasource/vm.go

@@ -24,20 +24,18 @@ type VMStorage struct {
 	dataSourceType     Type
 	evaluationInterval time.Duration
 	extraParams        url.Values
-	disablePathAppend  bool
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                 s.c,
-		authCfg:           s.authCfg,
-		datasourceURL:     s.datasourceURL,
-		lookBack:          s.lookBack,
-		queryStep:         s.queryStep,
-		appendTypePrefix:  s.appendTypePrefix,
-		dataSourceType:    s.dataSourceType,
-		disablePathAppend: s.disablePathAppend,
+		c:                s.c,
+		authCfg:          s.authCfg,
+		datasourceURL:    s.datasourceURL,
+		lookBack:         s.lookBack,
+		queryStep:        s.queryStep,
+		appendTypePrefix: s.appendTypePrefix,
+		dataSourceType:   s.dataSourceType,
 	}
 }
 
@@ -57,27 +55,25 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
 	return &VMStorage{
-		c:                 c,
-		authCfg:           authCfg,
-		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix:  appendTypePrefix,
-		lookBack:          lookBack,
-		queryStep:         queryStep,
-		dataSourceType:    NewPrometheusType(),
-		disablePathAppend: disablePathAppend,
+		c:                c,
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix: appendTypePrefix,
+		lookBack:         lookBack,
+		queryStep:        queryStep,
+		dataSourceType:   NewPrometheusType(),
 	}
 }
 
 // Query executes the given query and returns parsed response
-func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, error) {
 	req, err := s.newRequestPOST()
 	if err != nil {
 		return nil, err
 	}
 
-	ts := time.Now()
 	switch s.dataSourceType.String() {
 	case "prometheus":
 		s.setPrometheusInstantReqParams(req, query, ts)

app/vmalert/datasource/vm_prom_api.go

@@ -2,12 +2,18 @@ package datasource
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"strconv"
 	"time"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+
+		"to the configured -datasource.url and -remoteRead.url")
+)
+
 type promResponse struct {
 	Status    string `json:"status"`
 	ErrorType string `json:"errorType"`
@@ -25,13 +31,6 @@ type promInstant struct {
 	} `json:"result"`
 }
 
-type promRange struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
-	} `json:"result"`
-}
-
 func (r promInstant) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -50,6 +49,13 @@ func (r promInstant) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promRange struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TVs    [][2]interface{}  `json:"values"`
+	} `json:"result"`
+}
+
 func (r promRange) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -74,9 +80,22 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promScalar [2]interface{}
+
+func (r promScalar) metrics() ([]Metric, error) {
+	var m Metric
+	f, err := strconv.ParseFloat(r[1].(string), 64)
+	if err != nil {
+		return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", r, r[1], err)
+	}
+	m.Values = append(m.Values, f)
+	m.Timestamps = append(m.Timestamps, int64(r[0].(float64)))
+	return []Metric{m}, nil
+}
+
 const (
-	statusSuccess, statusError = "success", "error"
-	rtVector, rtMatrix         = "vector", "matrix"
+	statusSuccess, statusError  = "success", "error"
+	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
 )
 
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
@@ -103,29 +122,29 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric,
 			return nil, err
 		}
 		return pr.metrics()
+	case rScalar:
+		var ps promScalar
+		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
+			return nil, err
+		}
+		return ps.metrics()
 	default:
 		return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
 	}
 }
 
-const (
-	prometheusInstantPath = "/api/v1/query"
-	prometheusRangePath   = "/api/v1/query_range"
-	prometheusPrefix      = "/prometheus"
-)
-
 func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusInstantPath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
 	}
-	if s.evaluationInterval > 0 {
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
@@ -136,10 +155,10 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 
 func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusRangePath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))

app/vmalert/datasource/vm_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -36,7 +37,7 @@ func TestVMInstantQuery(t *testing.T) {
 	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
 		c++
 		switch c {
-		case 7:
+		case 8:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -74,6 +75,8 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
 		case 6:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests"},"value":[1583786140,"2000"]}]}}`))
+		case 7:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 
@@ -84,30 +87,26 @@ func TestVMInstantQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	ts := time.Now()
 
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected connection error got nil")
+	expErr := func(err string) {
+		if _, err := pq.Query(ctx, query, ts); err == nil {
+			t.Fatalf("expected %q got nil", err)
+		}
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected invalid response status error got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected response body error got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected error status got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected unknown status got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected non-vector resultType error  got nil")
-	}
-	m, err := pq.Query(ctx, query)
+
+	expErr("connection error")              // 0
+	expErr("invalid response status error") // 1
+	expErr("response body error")           // 2
+	expErr("error status")                  // 3
+	expErr("unknown status")                // 4
+	expErr("non-vector resultType error")   // 5
+
+	m, err := pq.Query(ctx, query, ts) // 6 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -130,10 +129,27 @@ func TestVMInstantQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m, expected)
 	}
 
+	m, err = pq.Query(ctx, query, ts) // 7 - scalar
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metrics got %d in %+v", len(m), m)
+	}
+	expected = []Metric{
+		{
+			Timestamps: []int64{1583786142},
+			Values:     []float64{1},
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m, expected)
+	}
+
 	g := NewGraphiteType()
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
 
-	m, err = gq.Query(ctx, queryRender)
+	m, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -194,7 +210,7 @@ func TestVMRangeQuery(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
@@ -250,18 +266,7 @@ func TestRequestParams(t *testing.T) {
 				dataSourceType: NewPrometheusType(),
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus path with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
 			},
 		},
 		{
@@ -272,19 +277,7 @@ func TestRequestParams(t *testing.T) {
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus prefix with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
 			},
 		},
 		{
@@ -294,18 +287,7 @@ func TestRequestParams(t *testing.T) {
 				dataSourceType: NewPrometheusType(),
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range path with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
@@ -316,19 +298,7 @@ func TestRequestParams(t *testing.T) {
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range prefix with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
@@ -517,6 +487,59 @@ func TestRequestParams(t *testing.T) {
 	}
 }
 
+func TestAuthConfig(t *testing.T) {
+	var testCases = []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
+}
+
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {

app/vmalert/group.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -13,20 +15,23 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu          sync.RWMutex
-	Name        string
-	File        string
-	Rules       []Rule
-	Type        datasource.Type
-	Interval    time.Duration
-	Concurrency int
-	Checksum    string
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           datasource.Type
+	Interval       time.Duration
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
 
 	Labels map[string]string
 	Params url.Values
@@ -43,13 +48,22 @@ type Group struct {
 type groupMetrics struct {
 	iterationTotal    *utils.Counter
 	iterationDuration *utils.Summary
+	iterationMissed   *utils.Counter
+	iterationInterval *utils.Gauge
 }
 
-func newGroupMetrics(name, file string) *groupMetrics {
+func newGroupMetrics(g *Group) *groupMetrics {
 	m := &groupMetrics{}
-	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
+	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
 	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		i := g.Interval.Seconds()
+		g.mu.RUnlock()
+		return i
+	})
 	return m
 }
 
@@ -85,13 +99,13 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		finishedCh: make(chan struct{}),
 		updateCh:   make(chan *Group),
 	}
-	g.metrics = newGroupMetrics(g.Name, g.File)
 	if g.Interval == 0 {
 		g.Interval = defaultInterval
 	}
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
+	g.metrics = newGroupMetrics(g)
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
 		var extraLabels map[string]string
@@ -215,6 +229,8 @@ func (g *Group) close() {
 
 	g.metrics.iterationDuration.Unregister()
 	g.metrics.iterationTotal.Unregister()
+	g.metrics.iterationMissed.Unregister()
+	g.metrics.iterationInterval.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -225,6 +241,11 @@ var skipRandSleepOnGroupStart bool
 func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
 
+	e := &executor{
+		rw:                       rw,
+		notifiers:                nts,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
+
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
 		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
@@ -245,8 +266,34 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 	}
 
+	evalTS := time.Now()
+
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{rw: rw, notifiers: nts}
+
+	eval := func(ts time.Time) {
+		g.metrics.iterationTotal.Inc()
+
+		start := time.Now()
+
+		if len(g.Rules) < 1 {
+			g.metrics.iterationDuration.UpdateDuration(start)
+			g.LastEvaluation = start
+			return
+		}
+
+		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration)
+		for err := range errs {
+			if err != nil {
+				logger.Errorf("group %q: %s", g.Name, err)
+			}
+		}
+		g.metrics.iterationDuration.UpdateDuration(start)
+		g.LastEvaluation = start
+	}
+
+	eval(evalTS)
+
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -265,6 +312,10 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				g.mu.Unlock()
 				continue
 			}
+
+			// ensure that staleness is tracked or existing rules only
+			e.purgeStaleSeries(g.Rules)
+
 			if g.Interval != ng.Interval {
 				g.Interval = ng.Interval
 				t.Stop()
@@ -273,44 +324,48 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			g.mu.Unlock()
 			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
-			g.metrics.iterationTotal.Inc()
-			iterationStart := time.Now()
-			if len(g.Rules) > 0 {
-				resolveDuration := getResolveDuration(g.Interval)
-				errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
-				for err := range errs {
-					if err != nil {
-						logger.Errorf("group %q: %s", g.Name, err)
-					}
-				}
+			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed > 0 {
+				g.metrics.iterationMissed.Inc()
 			}
-			g.metrics.iterationDuration.UpdateDuration(iterationStart)
+			evalTS = evalTS.Add((missed + 1) * g.Interval)
+
+			eval(evalTS)
 		}
 	}
 }
 
-// resolveDuration for alerts is equal to 3 interval evaluations
-// so in case if vmalert stops sending updates for some reason,
-// notifier could automatically resolve the alert.
-func getResolveDuration(groupInterval time.Duration) time.Duration {
-	resolveInterval := groupInterval * 3
-	if *maxResolveDuration > 0 && (resolveInterval > *maxResolveDuration) {
-		return *maxResolveDuration
+// getResolveDuration returns the duration after which firing alert
+// can be considered as resolved.
+func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Duration {
+	if groupInterval > delta {
+		delta = groupInterval
 	}
-	return resolveInterval
+	resolveDuration := delta * 4
+	if maxDuration > 0 && resolveDuration > maxDuration {
+		resolveDuration = maxDuration
+	}
+	return resolveDuration
 }
 
 type executor struct {
 	notifiers func() []notifier.Notifier
 	rw        *remotewrite.Client
+
+	previouslySentSeriesToRWMu sync.Mutex
+	// previouslySentSeriesToRW stores series sent to RW on previous iteration
+	// map[ruleID]map[ruleLabels][]prompb.Label
+	// where `ruleID` is ID of the Rule within a Group
+	// and `ruleLabels` is []prompb.Label marshalled to a string
+	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, resolveDuration)
+			res <- e.exec(ctx, rule, ts, resolveDuration)
 		}
 		close(res)
 		return res
@@ -323,7 +378,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, resolveDuration)
+				res <- e.exec(ctx, r, ts, resolveDuration)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -344,47 +399,41 @@ var (
 	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration) error {
 	execTotal.Inc()
 
-	tss, err := rule.Exec(ctx)
+	tss, err := rule.Exec(ctx, ts)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
 	}
 
-	if len(tss) > 0 && e.rw != nil {
-		for _, ts := range tss {
-			remoteWriteTotal.Inc()
-			if err := e.rw.Push(ts); err != nil {
-				remoteWriteErrors.Inc()
-				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
+	errGr := new(utils.ErrGroup)
+	if e.rw != nil {
+		pushToRW := func(tss []prompbmarshal.TimeSeries) {
+			for _, ts := range tss {
+				remoteWriteTotal.Inc()
+				if err := e.rw.Push(ts); err != nil {
+					remoteWriteErrors.Inc()
+					errGr.Add(fmt.Errorf("rule %q: remote write failure: %w", rule, err))
+				}
 			}
 		}
+		pushToRW(tss)
+		staleSeries := e.getStaleSeries(rule, tss, ts)
+		pushToRW(staleSeries)
 	}
 
 	ar, ok := rule.(*AlertingRule)
 	if !ok {
 		return nil
 	}
-	var alerts []notifier.Alert
-	for _, a := range ar.alerts {
-		switch a.State {
-		case notifier.StateFiring:
-			a.End = time.Now().Add(resolveDuration)
-			alerts = append(alerts, *a)
-		case notifier.StateInactive:
-			// set End to execStart to notify
-			// that it was just resolved
-			a.End = time.Now()
-			alerts = append(alerts, *a)
-		}
-	}
+
+	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
 
-	errGr := new(utils.ErrGroup)
 	for _, nt := range e.notifiers() {
 		if err := nt.Send(ctx, alerts); err != nil {
 			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
@@ -392,3 +441,74 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur
 	}
 	return errGr.Err()
 }
+
+// getStaledSeries checks whether there are stale series from previously sent ones.
+func (e *executor) getStaleSeries(rule Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
+	for _, ts := range tss {
+		// convert labels to strings so we can compare with previously sent series
+		key := labelsToString(ts.Labels)
+		ruleLabels[key] = ts.Labels
+	}
+
+	rID := rule.ID()
+	var staleS []prompbmarshal.TimeSeries
+	// check whether there are series which disappeared and need to be marked as stale
+	e.previouslySentSeriesToRWMu.Lock()
+	for key, labels := range e.previouslySentSeriesToRW[rID] {
+		if _, ok := ruleLabels[key]; ok {
+			continue
+		}
+		// previously sent series are missing in current series, so we mark them as stale
+		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
+		staleS = append(staleS, ss)
+	}
+	// set previous series to current
+	e.previouslySentSeriesToRW[rID] = ruleLabels
+	e.previouslySentSeriesToRWMu.Unlock()
+
+	return staleS
+}
+
+// purgeStaleSeries deletes references in tracked
+// previouslySentSeriesToRW list to Rules which aren't present
+// in the given activeRules list. The method is used when the list
+// of loaded rules has changed and executor has to remove
+// references to non-existing rules.
+func (e *executor) purgeStaleSeries(activeRules []Rule) {
+	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
+
+	e.previouslySentSeriesToRWMu.Lock()
+
+	for _, rule := range activeRules {
+		id := rule.ID()
+		prev, ok := e.previouslySentSeriesToRW[id]
+		if ok {
+			// keep previous series for staleness detection
+			newPreviouslySentSeriesToRW[id] = prev
+		}
+	}
+	e.previouslySentSeriesToRW = nil
+	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
+
+	e.previouslySentSeriesToRWMu.Unlock()
+}
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b strings.Builder
+	b.WriteRune('{')
+	for i, label := range labels {
+		if len(label.Name) == 0 {
+			b.WriteString("__name__")
+		} else {
+			b.WriteString(label.Name)
+		}
+		b.WriteRune('=')
+		b.WriteString(strconv.Quote(label.Value))
+		if i < len(labels)-1 {
+			b.WriteRune(',')
+		}
+	}
+	b.WriteRune('}')
+	return b.String()
+}

app/vmalert/group_test.go

@@ -3,6 +3,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -154,14 +157,15 @@ func TestUpdateWith(t *testing.T) {
 
 func TestGroupStart(t *testing.T) {
 	// TODO: make parsing from string instead of file
-	groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, true, true)
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
-	const evalInterval = time.Millisecond
+
 	fs := &fakeQuerier{}
 	fn := &fakeNotifier{}
 
+	const evalInterval = time.Millisecond
 	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
 	g.Concurrency = 2
 
@@ -170,7 +174,7 @@ func TestGroupStart(t *testing.T) {
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
 	r := g.Rules[0].(*AlertingRule)
-	alert1, err := r.newAlert(m1, time.Now(), nil)
+	alert1, err := r.newAlert(m1, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -183,13 +187,9 @@ func TestGroupStart(t *testing.T) {
 	// add service labels
 	alert1.Labels[alertNameLabel] = alert1.Name
 	alert1.Labels[alertGroupNameLabel] = g.Name
-	var labels1 []string
-	for k, v := range alert1.Labels {
-		labels1 = append(labels1, k, v)
-	}
-	alert1.ID = hash(metricWithLabels(t, labels1...))
+	alert1.ID = hash(alert1.Labels)
 
-	alert2, err := r.newAlert(m2, time.Now(), nil)
+	alert2, err := r.newAlert(m2, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -202,11 +202,7 @@ func TestGroupStart(t *testing.T) {
 	// add service labels
 	alert2.Labels[alertNameLabel] = alert2.Name
 	alert2.Labels[alertGroupNameLabel] = g.Name
-	var labels2 []string
-	for k, v := range alert2.Labels {
-		labels2 = append(labels2, k, v)
-	}
-	alert2.ID = hash(metricWithLabels(t, labels2...))
+	alert2.ID = hash(alert2.Labels)
 
 	finished := make(chan struct{})
 	fs.add(m1)
@@ -223,6 +219,12 @@ func TestGroupStart(t *testing.T) {
 	expectedAlerts := []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
+	gotAlertsNum := fn.getCounter()
+	if gotAlertsNum < len(expectedAlerts)*2 {
+		t.Fatalf("expected to receive at least %d alerts; got %d instead",
+			len(expectedAlerts)*2, gotAlertsNum)
+	}
+
 	// reset previous data
 	fs.reset()
 	// and set only one datapoint for response
@@ -232,7 +234,8 @@ func TestGroupStart(t *testing.T) {
 	time.Sleep(20 * evalInterval)
 
 	gotAlerts = fn.getAlerts()
-	expectedAlerts = []notifier.Alert{*alert1}
+	alert2.State = notifier.StateInactive
+	expectedAlerts = []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
 	g.close()
@@ -243,22 +246,170 @@ func TestResolveDuration(t *testing.T) {
 	testCases := []struct {
 		groupInterval time.Duration
 		maxDuration   time.Duration
+		resendDelay   time.Duration
 		expected      time.Duration
 	}{
-		{time.Minute, 0, 3 * time.Minute},
-		{3 * time.Minute, 0, 9 * time.Minute},
-		{time.Minute, 2 * time.Minute, 2 * time.Minute},
-		{0, 0, 0},
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
 	}
-	defaultResolveDuration := *maxResolveDuration
-	defer func() { *maxResolveDuration = defaultResolveDuration }()
+
 	for _, tc := range testCases {
 		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
-			*maxResolveDuration = tc.maxDuration
-			got := getResolveDuration(tc.groupInterval)
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
 			if got != tc.expected {
 				t.Errorf("expected to have %v; got %v", tc.expected, got)
 			}
 		})
 	}
 }
+
+func TestGetStaleSeries(t *testing.T) {
+	ts := time.Now()
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	f := func(rule Rule, labels, expLabels [][]prompbmarshal.Label) {
+		t.Helper()
+		var tss []prompbmarshal.TimeSeries
+		for _, l := range labels {
+			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
+		}
+		staleS := e.getStaleSeries(rule, tss, ts)
+		if staleS == nil && expLabels == nil {
+			return
+		}
+		if len(staleS) != len(expLabels) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expLabels), len(staleS))
+		}
+		for i, exp := range expLabels {
+			got := staleS[i]
+			if !reflect.DeepEqual(exp, got.Labels) {
+				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
+					exp, got.Labels)
+			}
+			if len(got.Samples) != 1 {
+				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
+			}
+			if !decimal.IsStaleNaN(got.Samples[0].Value) {
+				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
+			}
+		}
+	}
+
+	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
+
+	// single series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		nil)
+
+	// multiple series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
+
+	// multiple rules and series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 2},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+}
+
+func TestPurgeStaleSeries(t *testing.T) {
+	ts := time.Now()
+	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
+	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
+
+	f := func(curRules, newRules, expStaleRules []Rule) {
+		t.Helper()
+		e := &executor{
+			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		}
+		// seed executor with series for
+		// current rules
+		for _, rule := range curRules {
+			e.getStaleSeries(rule, tss, ts)
+		}
+
+		e.purgeStaleSeries(newRules)
+
+		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expStaleRules), len(e.previouslySentSeriesToRW))
+		}
+
+		for _, exp := range expStaleRules {
+			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
+				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
+			}
+		}
+	}
+
+	f(nil, nil, nil)
+	f(
+		nil,
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+	)
+}

app/vmalert/helpers_test.go

@@ -44,10 +44,10 @@ func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Qu
 }
 
 func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
-	return fq.Query(ctx, q)
+	return fq.Query(ctx, q, time.Now())
 }
 
-func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
@@ -61,6 +61,8 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric,
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert
+	// records number of received alerts in total
+	counter int
 }
 
 func (*fakeNotifier) Close()       {}
@@ -68,10 +70,17 @@ func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
+	fn.counter += len(alerts)
 	fn.alerts = alerts
 	return nil
 }
 
+func (fn *fakeNotifier) getCounter() int {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.counter
+}
+
 func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	fn.Lock()
 	defer fn.Unlock()
@@ -107,6 +116,21 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
+func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	var ls []prompbmarshal.Label
+	for i := 0; i < len(labels); i += 2 {
+		ls = append(ls, prompbmarshal.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return ls
+}
+
 func compareGroups(t *testing.T, a, b *Group) {
 	t.Helper()
 	if a.Name != b.Name {

app/vmalert/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -34,6 +35,13 @@ Examples:
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
+	ruleTemplatesPath = flagutil.NewArray("rule.templates", `Path or glob pattern to location with go template definitions
+	for rules annotations templating. Flag can be specified multiple times.
+Examples:
+ -rule.templates="/path/to/file". Path to a single file with go templates
+ -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+absolute path to all .tpl files in root.`)
+
 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
 		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
 
@@ -47,6 +55,8 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
 	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
 		"which is by default equal to 3 evaluation intervals of the parent group.")
+	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
@@ -71,10 +81,12 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	err := templates.Load(*ruleTemplatesPath, true)
+	if err != nil {
+		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
+	}
 
 	if *dryRun {
-		u, _ := url.Parse("https://victoriametrics.com/")
-		notifier.InitTemplateFunc(u)
 		groups, err := config.Parse(*rulePath, true, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
@@ -89,7 +101,7 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init `external.url`: %s", err)
 	}
-	notifier.InitTemplateFunc(eu)
+
 	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
 	if err != nil {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
@@ -103,7 +115,6 @@ func main() {
 		if rw == nil {
 			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
-		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
@@ -125,7 +136,6 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
-
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 	if err != nil {
@@ -168,7 +178,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	labels := make(map[string]string, 0)
+	labels := make(map[string]string)
 	for _, s := range *externalLabels {
 		if len(s) == 0 {
 			continue
@@ -241,7 +251,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, m)
+		templated, err := alert.ExecTemplate(nil, nil, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
@@ -279,7 +289,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		case <-ctx.Done():
 			return
 		case <-sighupCh:
-			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
+			tmplMsg := ""
+			if len(*ruleTemplatesPath) > 0 {
+				tmplMsg = fmt.Sprintf("and templates %q ", *ruleTemplatesPath)
+			}
+			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
@@ -289,6 +303,13 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
+		err := templates.Load(*ruleTemplatesPath, false)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to load new templates: %s", err)
+			continue
+		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
@@ -297,6 +318,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
 			// set success to 1 since previous reload
 			// could have been unsuccessful
 			configSuccess.Set(1)
@@ -309,6 +331,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		templates.Reload()
 		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())

app/vmalert/manager.go

@@ -37,7 +37,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 
 	g, ok := m.groups[gID]
 	if !ok {
-		return nil, fmt.Errorf("can't find group with id %q", gID)
+		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
 	for _, rule := range g.Rules {
 		ar, ok := rule.(*AlertingRule)
@@ -48,7 +48,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 			return apiAlert, nil
 		}
 	}
-	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
+	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)
 }
 
 func (m *manager) start(ctx context.Context, groupsCfg []config.Group) error {
@@ -163,21 +163,17 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:        g.Name,
-		Type:        g.Type.String(),
-		File:        g.File,
-		Interval:    g.Interval.String(),
-		Concurrency: g.Concurrency,
-		Params:      urlValuesToStrings(g.Params),
-		Labels:      g.Labels,
+		Name:           g.Name,
+		Type:           g.Type.String(),
+		File:           g.File,
+		Interval:       g.Interval.Seconds(),
+		LastEvaluation: g.LastEvaluation,
+		Concurrency:    g.Concurrency,
+		Params:         urlValuesToStrings(g.Params),
+		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
-		switch v := r.(type) {
-		case *AlertingRule:
-			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
-		case *RecordingRule:
-			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
-		}
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return ag
 }

app/vmalert/manager_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"math/rand"
-	"net/url"
 	"os"
 	"strings"
 	"sync"
@@ -14,11 +13,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
@@ -47,9 +48,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 		"config/testdata/dir/rules0-bad.rules",
 		"config/testdata/dir/rules1-good.rules",
 		"config/testdata/dir/rules1-bad.rules",
-		"config/testdata/rules0-good.rules",
-		"config/testdata/rules1-good.rules",
-		"config/testdata/rules2-good.rules",
+		"config/testdata/rules/rules0-good.rules",
+		"config/testdata/rules/rules1-good.rules",
+		"config/testdata/rules/rules2-good.rules",
 	}
 	evalInterval := *evaluationInterval
 	defer func() { *evaluationInterval = evalInterval }()
@@ -125,7 +126,7 @@ func TestManagerUpdate(t *testing.T) {
 	}{
 		{
 			name:       "update good rules",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules1-good.rules",
 			want: []*Group{
 				{
@@ -150,18 +151,18 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update good rules from 1 to 2 groups",
-			initPath:   "config/testdata/dir/rules1-good.rules",
-			updatePath: "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/dir/rules/rules1-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     datasource.NewPrometheusType(),
 					Rules:    []Rule{VMRows},
 					Interval: defaultEvalInterval,
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Type:     datasource.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
@@ -172,18 +173,18 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update with one bad rule file",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules2-bad.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     datasource.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Name:     "TestGroup",
 					Type:     datasource.NewPrometheusType(),
@@ -196,17 +197,17 @@ func TestManagerUpdate(t *testing.T) {
 		{
 			name:       "update empty dir rules from 0 to 2 groups",
 			initPath:   "config/testdata/empty/*",
-			updatePath: "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     datasource.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Type:     datasource.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{

app/vmalert/notifier/alert.go

@@ -5,10 +5,13 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"text/template"
+	textTpl "text/template"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // Alert the triggered alert
@@ -26,10 +29,16 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
-	// Start defines the moment of time when Alert has triggered
+	// ActiveAt defines the moment of time when Alert has become active
+	ActiveAt time.Time
+	// Start defines the moment of time when Alert has become firing
 	Start time.Time
 	// End defines the moment of time when Alert supposed to expire
 	End time.Time
+	// ResolvedAt defines the moment when Alert was switched from Firing to Inactive
+	ResolvedAt time.Time
+	// LastSent defines the moment when Alert was sent last time
+	LastSent time.Time
 	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
 	// ID is the unique identifer for the Alert
@@ -82,26 +91,38 @@ var tplHeaders = []string{
 // map of annotations.
 // Every alert could have a different datasource, so function
 // requires a queryFunction as an argument.
-func (a *Alert) ExecTemplate(q QueryFn, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
-	return templateAnnotations(annotations, tplData, funcsWithQuery(q))
+func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error getting a template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ExecTemplate executes the given template for given annotations map.
-func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) {
-	return templateAnnotations(annotations, tpl, funcsWithQuery(q))
+func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error cloning template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ValidateTemplates validate annotations for possible template error, uses empty data for template population
 func ValidateTemplates(annotations map[string]string) error {
-	_, err := templateAnnotations(annotations, AlertTplData{
+	tmpl, err := templates.Get()
+	if err != nil {
+		return err
+	}
+	_, err = templateAnnotations(annotations, AlertTplData{
 		Labels: map[string]string{},
 		Value:  0,
-	}, tmplFunc)
+	}, tmpl, false)
 	return err
 }
 
-func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap) (map[string]string, error) {
+func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
@@ -114,7 +135,7 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 		builder.Grow(len(header) + len(text))
 		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, tmpl, execute); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -130,14 +151,38 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
-	t := template.New("").Funcs(funcs).Option("missingkey=zero")
-	tpl, err := t.Parse(text)
+func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
+	tpl, err := tmpl.Clone()
 	if err != nil {
-		return fmt.Errorf("error parsing annotation: %w", err)
+		return fmt.Errorf("error cloning template before parse annotation: %w", err)
+	}
+	tpl, err = tpl.Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation template: %w", err)
+	}
+	if !execute {
+		return nil
+	}
+	if !execute {
+		return nil
 	}
 	if err = tpl.Execute(dst, data); err != nil {
 		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}
 	return nil
 }
+
+func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
+	for k, v := range a.Labels {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	promrelabel.SortLabels(labels)
+	if relabelCfg != nil {
+		return relabelCfg.Apply(labels, 0, false)
+	}
+	return labels
+}

app/vmalert/notifier/alert_test.go

@@ -2,13 +2,16 @@ package notifier
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
-	extLabels := make(map[string]string, 0)
+	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
 		extDC      = "east"
@@ -130,7 +133,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			tpl, err := tc.alert.ExecTemplate(qFn, tc.annotations)
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -146,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestAlert_toPromLabels(t *testing.T) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
+		t.Helper()
+		a := Alert{Labels: labels}
+		got := a.toPromLabels(relabel)
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
+				exp, got)
+		}
+	}
+
+	fn(nil, nil, nil)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		nil,
+	)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	fn(
+		map[string]string{"a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		pcs,
+	)
+}

app/vmalert/notifier/alertmanager.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
@@ -21,6 +23,8 @@ type AlertManager struct {
 	timeout time.Duration
 
 	authCfg *promauth.Config
+	// stores already parsed RelabelConfigs object
+	relabelConfigs *promrelabel.ParsedConfigs
 
 	metrics *metrics
 }
@@ -58,7 +62,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 
 func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
-	writeamRequest(b, alerts, am.argFunc)
+	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
 	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
@@ -102,7 +106,8 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
+	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
@@ -112,21 +117,30 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
-	ba := &promauth.BasicAuthConfig{}
+	ba := new(promauth.BasicAuthConfig)
+	oauth := new(promauth.OAuth2Config)
 	if authCfg.BasicAuth != nil {
 		ba = authCfg.BasicAuth
 	}
-	aCfg, err := utils.AuthConfig(ba.Username, ba.Password.String(), ba.PasswordFile, authCfg.BearerToken.String(), authCfg.BearerTokenFile)
+	if authCfg.OAuth2 != nil {
+		oauth = authCfg.OAuth2
+	}
+
+	aCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
 	return &AlertManager{
-		addr:    alertManagerURL,
-		argFunc: fn,
-		authCfg: aCfg,
-		client:  &http.Client{Transport: tr},
-		timeout: timeout,
-		metrics: newMetrics(alertManagerURL),
+		addr:           alertManagerURL,
+		argFunc:        fn,
+		authCfg:        aCfg,
+		relabelConfigs: relabelCfg,
+		client:         &http.Client{Transport: tr},
+		timeout:        timeout,
+		metrics:        newMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,9 +1,11 @@
 {% import (
     "time"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
 {
@@ -14,8 +16,9 @@
     {% endif %}
     "labels": {
         "alertname":{%q= alert.Name %}
-        {% for k,v := range alert.Labels %}
-            ,{%q= k %}:{%q= v %}
+        {% code lbls := alert.toPromLabels(relabelCfg) %}
+        {% for _, l := range lbls %}
+            ,{%q= l.Name %}:{%q= l.Value %}
         {% endfor %}
     },
     "annotations": {

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -7,124 +7,129 @@ package notifier
 //line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		qw422016.N().S(`"labels": {"alertname":`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-		for k, v := range alert.Labels {
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().Q(alert.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:19
-		qw422016.N().S(`},"annotations": {`)
+		lbls := alert.toPromLabels(relabelCfg)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		for _, l := range lbls {
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Value)
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 		c := len(alert.Annotations)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		for k, v := range alert.Annotations {
-//line app/vmalert/notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
 			c = c - 1
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	streamamRequest(qw422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	writeamRequest(qb422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

app/vmalert/notifier/alertmanager_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -89,7 +89,7 @@ func TestAlertManager_Send(t *testing.T) {
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, aCfg, 0)
+	}, aCfg, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}

app/vmalert/notifier/config.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -29,16 +30,21 @@ type Config struct {
 	// ConsulSDConfigs contains list of settings for service discovery via Consul
 	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// DNSSDConfigs ontains list of settings for service discovery via DNS.
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
+
 	// StaticConfigs contains list of static targets
 	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
 
 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
-	// RelabelConfigs contains list of relabeling rules
+	// RelabelConfigs contains list of relabeling rules for entities discovered via SD
 	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
-
+	// AlertRelabelConfigs contains list of relabeling rules alert labels
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
 	// The timeout used when sending alerts.
-	Timeout promutils.Duration `yaml:"timeout,omitempty"`
+	Timeout *promutils.Duration `yaml:"timeout,omitempty"`
 
 	// Checksum stores the hash of yaml definition for the config.
 	// May be used to detect any changes to the config file.
@@ -52,6 +58,8 @@ type Config struct {
 
 	// stores already parsed RelabelConfigs object
 	parsedRelabelConfigs *promrelabel.ParsedConfigs
+	// stores already parsed AlertRelabelConfigs object
+	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }
 
 // StaticConfig contains list of static targets in the following form:
@@ -78,6 +86,11 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("failed to parse relabeling config: %w", err)
 	}
 	cfg.parsedRelabelConfigs = rCfg
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
+	}
+	cfg.parsedAlertRelabelConfigs = arCfg
 
 	b, err := yaml.Marshal(cfg)
 	if err != nil {

app/vmalert/notifier/config_test.go

@@ -12,6 +12,7 @@ func TestConfigParseGood(t *testing.T) {
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
+	f("testdata/dns.good.yaml")
 	f("testdata/static.good.yaml")
 }
 

app/vmalert/notifier/config_watcher.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
 
 // configWatcher supports dynamic reload of Notifier objects
@@ -141,7 +142,7 @@ func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator)
 		}
 		duplicates[u] = struct{}{}
 
-		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			errors = append(errors, err)
 			continue
@@ -165,7 +166,7 @@ func (cw *configWatcher) start() error {
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
@@ -195,6 +196,24 @@ func (cw *configWatcher) start() error {
 			return fmt.Errorf("failed to start consulSD discovery: %s", err)
 		}
 	}
+
+	if len(cw.cfg.DNSSDConfigs) > 0 {
+		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.DNSSDConfigs {
+				sdc := &cw.cfg.DNSSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start DNSSD discovery: %s", err)
+		}
+	}
 	return nil
 }
 

app/vmalert/notifier/init.go

@@ -3,9 +3,11 @@ package notifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -15,11 +17,15 @@ var (
 	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
 	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
 
-	addrs                 = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+
 	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
+	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
+
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
 	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
 	tlsKeyFile            = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
@@ -27,6 +33,17 @@ var (
 		"By default system CA is used")
 	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")
+
+	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )
 
 // cw holds a configWatcher for configPath configuration file
@@ -68,6 +85,12 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 
 	externalURL = extURL
 	externalLabels = extLabels
+	eu, err := url.Parse(externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse external URL: %s", err)
+	}
+
+	templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
 
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
@@ -87,7 +110,6 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 		return staticNotifiersFn, nil
 	}
 
-	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %s", err)
@@ -111,9 +133,19 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
 				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
 			},
+			BearerToken:     promauth.NewSecret(bearerToken.GetOptionalArg(i)),
+			BearerTokenFile: bearerTokenFile.GetOptionalArg(i),
+			OAuth2: &promauth.OAuth2Config{
+				ClientID:         oauth2ClientID.GetOptionalArg(i),
+				ClientSecret:     promauth.NewSecret(oauth2ClientSecret.GetOptionalArg(i)),
+				ClientSecretFile: oauth2ClientSecretFile.GetOptionalArg(i),
+				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
+				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
+			},
 		}
+
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
 		if err != nil {
 			return nil, err
 		}
@@ -137,6 +169,8 @@ const (
 	TargetStatic TargetType = "static"
 	// TargetConsul is for targets discovered via Consul
 	TargetConsul TargetType = "consulSD"
+	// TargetDNS is for targets discovered via DNS
+	TargetDNS TargetType = "DNSSD"
 )
 
 // GetTargets returns list of static or discovered targets

app/vmalert/notifier/package_test.go

@@ -1,13 +1,14 @@
 package notifier
 
 import (
-	"net/url"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"os"
 	"testing"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }

app/vmalert/notifier/testdata/consul.good.yaml

@@ -10,4 +10,7 @@ relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/dns.good.yaml

@@ -0,0 +1,12 @@
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+relabel_configs:
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -11,8 +11,18 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - consul
+
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+
 relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name

app/vmalert/notifier/testdata/static.good.yaml

@@ -2,3 +2,6 @@ static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/recording.go

@@ -31,6 +31,8 @@ type RecordingRule struct {
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -122,12 +124,13 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }
 
 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := rr.q.Query(ctx, rr.Expr)
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
 
-	rr.lastExecTime = time.Now()
+	rr.lastExecTime = ts
+	rr.lastExecDuration = time.Since(ts)
 	rr.lastExecError = err
 	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
@@ -193,23 +196,27 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	return nil
 }
 
-// RuleAPI returns Rule representation in form
-// of APIRecordingRule
-func (rr *RecordingRule) RuleAPI() APIRecordingRule {
-	var lastErr string
-	if rr.lastExecError != nil {
-		lastErr = rr.lastExecError.Error()
-	}
-	return APIRecordingRule{
+// ToAPI returns Rule's representation in form
+// of APIRule
+func (rr *RecordingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "recording",
+		DatasourceType: rr.Type.String(),
+		Name:           rr.Name,
+		Query:          rr.Expr,
+		Labels:         rr.Labels,
+		LastEvaluation: rr.lastExecTime,
+		EvaluationTime: rr.lastExecDuration.Seconds(),
+		Health:         "ok",
+		LastSamples:    rr.lastExecSamples,
 		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", rr.ID()),
-		GroupID:     fmt.Sprintf("%d", rr.GroupID),
-		Name:        rr.Name,
-		Type:        rr.Type.String(),
-		Expression:  rr.Expr,
-		LastError:   lastErr,
-		LastSamples: rr.lastExecSamples,
-		LastExec:    rr.lastExecTime,
-		Labels:      rr.Labels,
+		ID:      fmt.Sprintf("%d", rr.ID()),
+		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
+
+	if rr.lastExecError != nil {
+		r.LastError = rr.lastExecError.Error()
+		r.Health = "err"
+	}
+	return r
 }

app/vmalert/recording_test.go

@@ -77,7 +77,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now())
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -178,7 +178,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO())
+	_, err := rr.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +193,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO())
+	_, err = rr.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remoteread/init.go

@@ -13,11 +13,13 @@ var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
 		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
 		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
 	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
-	bearerToken           = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
-	bearerTokenFile       = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
+	bearerToken     = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
@@ -26,7 +28,12 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
-	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
+
+	oauth2ClientID         = flag.String("remoteRead.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteRead.url.")
+	oauth2ClientSecret     = flag.String("remoteRead.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteRead.url.")
+	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
+	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
+	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
 // Init creates a Querier from provided flag values.
@@ -39,10 +46,14 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c), nil
 }

app/vmalert/remotewrite/init.go

@@ -13,11 +13,13 @@ var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
 	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
-	bearerToken           = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
-	bearerTokenFile       = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
+
+	bearerToken     = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -31,7 +33,12 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
-	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
+
+	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url.")
+	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url.")
+	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
+	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
+	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
 // Init creates Client object from given flags.
@@ -46,7 +53,10 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -19,17 +20,20 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
+)
+
 // Client is an asynchronous HTTP client for writing
 // timeseries via remote write protocol.
 type Client struct {
-	addr              string
-	c                 *http.Client
-	authCfg           *promauth.Config
-	input             chan prompbmarshal.TimeSeries
-	flushInterval     time.Duration
-	maxBatchSize      int
-	maxQueueSize      int
-	disablePathAppend bool
+	addr          string
+	c             *http.Client
+	authCfg       *promauth.Config
+	input         chan prompbmarshal.TimeSeries
+	flushInterval time.Duration
+	maxBatchSize  int
+	maxQueueSize  int
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
@@ -70,8 +74,6 @@ const (
 	defaultWriteTimeout  = 30 * time.Second
 )
 
-const writePath = "/api/v1/write"
-
 // NewClient returns asynchronous client for
 // writing timeseries via remotewrite protocol.
 func NewClient(ctx context.Context, cfg Config) (*Client, error) {
@@ -102,14 +104,13 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
-		addr:              strings.TrimSuffix(cfg.Addr, "/"),
-		authCfg:           cfg.AuthCfg,
-		flushInterval:     cfg.FlushInterval,
-		maxBatchSize:      cfg.MaxBatchSize,
-		maxQueueSize:      cfg.MaxQueueSize,
-		doneCh:            make(chan struct{}),
-		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
-		disablePathAppend: cfg.DisablePathAppend,
+		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		authCfg:       cfg.AuthCfg,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		maxQueueSize:  cfg.MaxQueueSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 	}
 
 	for i := 0; i < cc; i++ {
@@ -225,7 +226,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 	droppedRows.Add(len(wr.Timeseries))
 	droppedBytes.Add(len(b))
-	logger.Errorf("all %d attempts to send request failed - dropping %d timeseries",
+	logger.Errorf("all %d attempts to send request failed - dropping %d time series",
 		attempts, len(wr.Timeseries))
 }
 
@@ -240,8 +241,8 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 			req.Header.Set("Authorization", auth)
 		}
 	}
-	if !c.disablePathAppend {
-		req.URL.Path = path.Join(req.URL.Path, writePath)
+	if !*disablePathAppend {
+		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {

app/vmalert/replay.go

@@ -7,13 +7,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cheggaaa/pb/v3"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/dmitryk-dk/pb/v3"
 )
 
 var (
@@ -29,6 +28,8 @@ var (
 		"Max number of data points expected in one request. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
+		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw *remotewrite.Client) error {
@@ -88,7 +89,10 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 		g.Name, g.Interval, iterations, step)
 	for _, rule := range g.Rules {
 		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
-		bar := pb.StartNew(iterations)
+		var bar *pb.ProgressBar
+		if !*disableProgressBar {
+			bar = pb.StartNew(iterations)
+		}
 		ri.reset()
 		for ri.next() {
 			n, err := replayRule(rule, ri.s, ri.e, rw)
@@ -96,9 +100,13 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 				logger.Fatalf("rule %q: %s", rule, err)
 			}
 			total += n
-			bar.Increment()
+			if bar != nil {
+				bar.Increment()
+			}
+		}
+		if bar != nil {
+			bar.Finish()
 		}
-		bar.Finish()
 		// sleep to let remote storage to flush data on-disk
 		// so chained rules could be calculated correctly
 		time.Sleep(*replayRulesDelay)

app/vmalert/rule.go

@@ -3,8 +3,9 @@ package main
 import (
 	"context"
 	"errors"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Rule represents alerting or recording rule
@@ -14,13 +15,15 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context
-	Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error)
+	// Exec executes the rule with given context at the given timestamp
+	Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error)
 	// ExecRange executes the rule on the given time range
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error
+	// ToAPI converts Rule into APIRule
+	ToAPI() APIRule
 	// Close performs the shutdown procedures for rule
 	// such as metrics unregister
 	Close()

app/vmalert/templates/template.go

@@ -11,25 +11,118 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package notifier
+package templates
 
 import (
 	"errors"
 	"fmt"
+	htmlTpl "html/template"
+	"io/ioutil"
 	"math"
 	"net"
 	"net/url"
+	"path/filepath"
 	"regexp"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	htmlTpl "html/template"
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
+// go template execution fails when it's tree is empty
+const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
+
+var tplMu sync.RWMutex
+
+type textTemplate struct {
+	current     *textTpl.Template
+	replacement *textTpl.Template
+}
+
+var masterTmpl textTemplate
+
+func newTemplate() *textTpl.Template {
+	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(defaultTemplate))
+}
+
+// Load func loads templates from multiple globs specified in pathPatterns and either
+// sets them directly to current template if it's undefined or with overwrite=true
+// or sets replacement templates and adds templates with new names to a current
+func Load(pathPatterns []string, overwrite bool) error {
+	var err error
+	tmpl := newTemplate()
+	for _, tp := range pathPatterns {
+		p, err := filepath.Glob(tp)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
+		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseGlob(tp)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	if len(tmpl.Templates()) > 0 {
+		err := tmpl.Execute(ioutil.Discard, nil)
+		if err != nil {
+			return fmt.Errorf("failed to execute template: %w", err)
+		}
+	}
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.current == nil || overwrite {
+		masterTmpl.replacement = nil
+		masterTmpl.current = newTemplate()
+	} else {
+		masterTmpl.replacement = newTemplate()
+		if err = copyTemplates(tmpl, masterTmpl.replacement, overwrite); err != nil {
+			return err
+		}
+	}
+	return copyTemplates(tmpl, masterTmpl.current, overwrite)
+}
+
+func copyTemplates(from *textTpl.Template, to *textTpl.Template, overwrite bool) error {
+	if from == nil {
+		return nil
+	}
+	if to == nil {
+		to = newTemplate()
+	}
+	tmpl, err := from.Clone()
+	if err != nil {
+		return err
+	}
+	for _, t := range tmpl.Templates() {
+		if to.Lookup(t.Name()) == nil || overwrite {
+			to, err = to.AddParseTree(t.Name(), t.Tree)
+			if err != nil {
+				return fmt.Errorf("failed to add template %q: %w", t.Name(), err)
+			}
+		}
+	}
+	return nil
+}
+
+// Reload func replaces current template with a replacement template
+// which was set by Load with override=false
+func Reload() {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.replacement != nil {
+		masterTmpl.current = masterTmpl.replacement
+		masterTmpl.replacement = nil
+	}
+}
+
 // metric is private copy of datasource.Metric,
 // it is used for templating annotations,
 // Labels as map simplifies templates evaluation.
@@ -59,12 +152,62 @@ func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
 // for templating functions.
 type QueryFn func(query string) ([]datasource.Metric, error)
 
-var tmplFunc textTpl.FuncMap
+// UpdateWithFuncs updates existing or sets a new function map for a template
+func UpdateWithFuncs(funcs textTpl.FuncMap) {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	masterTmpl.current = masterTmpl.current.Funcs(funcs)
+}
 
-// InitTemplateFunc initiates template helper functions
-func InitTemplateFunc(externalURL *url.URL) {
+// GetWithFuncs returns a copy of current template with additional FuncMap
+// provided with funcs argument
+func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	tmpl, err := masterTmpl.current.Clone()
+	if err != nil {
+		return nil, err
+	}
+	return tmpl.Funcs(funcs), nil
+}
+
+// Get returns a copy of a template
+func Get() (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	return masterTmpl.current.Clone()
+}
+
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
+}
+
+// FuncsWithExternalURL returns a function map that depends on externalURL value
+func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
+}
+
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
 	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	tmplFunc = textTpl.FuncMap{
+	return textTpl.FuncMap{
 		/* Strings */
 
 		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
@@ -116,9 +259,13 @@ func InitTemplateFunc(externalURL *url.URL) {
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(v float64) string {
+		"humanize": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				prefix := ""
@@ -129,7 +276,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 					prefix = p
 					v /= 1000
 				}
-				return fmt.Sprintf("%.4g%s", v, prefix)
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -139,13 +286,17 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(v float64) string {
+		"humanize1024": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
@@ -155,16 +306,20 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v /= 1024
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanizeDuration converts given seconds to a human readable duration
-		"humanizeDuration": func(v float64) string {
+		"humanizeDuration": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if v == 0 {
-				return fmt.Sprintf("%.4gs", v)
+				return fmt.Sprintf("%.4gs", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				sign := ""
@@ -178,16 +333,16 @@ func InitTemplateFunc(externalURL *url.URL) {
 				days := int64(v) / 60 / 60 / 24
 				// For days to minutes, we display seconds as an integer.
 				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
 				}
 				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
 				}
 				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
 				}
 				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v)
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -197,33 +352,51 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%ss", v, prefix)
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(v float64) string {
-			return fmt.Sprintf("%.4g%%", v*100)
+		"humanizePercentage": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(v float64) string {
+		"humanizeTimestamp": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
-			return fmt.Sprint(t)
+			return fmt.Sprint(t), nil
 		},
 
 		/* URLs */
 
 		// externalURL returns value of `external.url` flag
 		"externalURL": func() string {
-			return externalURL.String()
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathPrefix returns a Path segment from the URL value in `external.url` flag
 		"pathPrefix": func() string {
-			return externalURL.Path
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathEscape escapes the string so it can be safely placed inside a URL path segment,
@@ -258,7 +431,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// execute "/api/v1/query?query=foo" request and will return
 		// the first value in response.
 		"query": func(q string) ([]metric, error) {
-			// query function supposed to be substituted at funcsWithQuery().
+			// query function supposed to be substituted at FuncsWithQuery().
 			// it is present here only for validation purposes, when there is no
 			// provided datasource.
 			//
@@ -282,6 +455,14 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return m.Labels[label]
 		},
 
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
 		// value returns the value of the given metric.
 		// usually used alongside with `query` template function.
 		"value": func(m metric) float64 {
@@ -307,21 +488,6 @@ func InitTemplateFunc(externalURL *url.URL) {
 	}
 }
 
-func funcsWithQuery(query QueryFn) textTpl.FuncMap {
-	fm := make(textTpl.FuncMap)
-	for k, fn := range tmplFunc {
-		fm[k] = fn
-	}
-	fm["query"] = func(q string) ([]metric, error) {
-		result, err := query(q)
-		if err != nil {
-			return nil, err
-		}
-		return datasourceMetricsToTemplateMetrics(result), nil
-	}
-	return fm
-}
-
 // Time is the number of milliseconds since the epoch
 // (1970-01-01 00:00 UTC) excluding leap seconds.
 type Time int64
@@ -346,3 +512,28 @@ const second = int64(time.Second / minimumTick)
 func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
+
+func toFloat64(v interface{}) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
+}

app/vmalert/templates/template_test.go

@@ -0,0 +1,275 @@
+package templates
+
+import (
+	"strings"
+	"testing"
+	textTpl "text/template"
+)
+
+func mkTemplate(current, replacement interface{}) textTemplate {
+	tmpl := textTemplate{}
+	if current != nil {
+		switch val := current.(type) {
+		case string:
+			tmpl.current = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	if replacement != nil {
+		switch val := replacement.(type) {
+		case string:
+			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	return tmpl
+}
+
+func equalTemplates(tmpls ...*textTpl.Template) bool {
+	var cmp *textTpl.Template
+	for i, tmpl := range tmpls {
+		if i == 0 {
+			cmp = tmpl
+		} else {
+			if cmp == nil || tmpl == nil {
+				if cmp != tmpl {
+					return false
+				}
+				continue
+			}
+			if len(tmpl.Templates()) != len(cmp.Templates()) {
+				return false
+			}
+			for _, t := range tmpl.Templates() {
+				tp := cmp.Lookup(t.Name())
+				if tp == nil {
+					return false
+				}
+				if tp.Root.String() != t.Root.String() {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+func TestTemplates_Load(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		pathPatterns     []string
+		overwrite        bool
+		expectedTemplate textTemplate
+		expErr           string
+	}{
+		{
+			"non existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"non existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`, nil),
+			"",
+		},
+		{
+			"existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				`, `
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`),
+			"",
+		},
+		{
+			"load template with syntax error",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/bad0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			"failed to parse template glob",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			err := Load(tc.pathPatterns, tc.overwrite)
+			if tc.expErr == "" && err != nil {
+				t.Error("happened error that wasn't expected: %w", err)
+			}
+			if tc.expErr != "" && err == nil {
+				t.Error("%+w", err)
+				t.Error("expected error that didn't happend")
+			}
+			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
+				t.Error("%+w", err)
+				t.Error("expected string doesn't exist in error message")
+			}
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}
+
+func TestTemplates_Reload(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		expectedTemplate textTemplate
+	}{
+		{
+			"empty current and replacement templates",
+			mkTemplate(nil, nil),
+			mkTemplate(nil, nil),
+		},
+		{
+			"empty current template only",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"empty replacement template only",
+			mkTemplate(nil, `
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"defined both templates",
+			mkTemplate(`
+				{{- define "test.0" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{- printf "before" -}}
+				{{- end -}}
+			`, `
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`, nil),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			Reload()
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}

app/vmalert/templates/templates/other/nested/bad0-test.tpl

@@ -0,0 +1,3 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL" }}
+{{- end -}}

app/vmalert/templates/templates/other/nested/good0-test.tpl

@@ -0,0 +1,9 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.0" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.3" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}

app/vmalert/templates/templates/test/good0-test.tpl

@@ -0,0 +1,9 @@
+{{- define "test.2" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.0" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.3" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}

app/vmalert/tpl/header.qtpl

@@ -9,11 +9,6 @@
           min-height: 75rem;
           padding-top: 4.5rem;
         }
-        pre {
-            overflow: scroll;
-            max-width: 600px;
-            min-height: 30px;
-        }
         .group-heading {
             cursor: pointer;
             padding: 5px;
@@ -32,8 +27,27 @@
          .group-heading:hover {
             background-color: #f8f9fa!important;
         }
+        .table {
+            table-layout: fixed;
+        }
         .table .error-cell{
             word-break: break-word;
+            font-size: 14px;
+        }
+        pre {
+            overflow: scroll;
+            min-height: 30px;
+            max-width: 100%;
+        }
+        pre::-webkit-scrollbar {
+          -webkit-appearance: none;
+          width: 0px;
+          height: 5px;
+        }
+        pre::-webkit-scrollbar-thumb {
+          border-radius: 5px;
+          background-color: rgba(0,0,0,.5);
+          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
         }
     </style>
 </head>

app/vmalert/tpl/header.qtpl.go

@@ -41,11 +41,6 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
           min-height: 75rem;
           padding-top: 4.5rem;
         }
-        pre {
-            overflow: scroll;
-            max-width: 600px;
-            min-height: 30px;
-        }
         .group-heading {
             cursor: pointer;
             padding: 5px;
@@ -64,44 +59,63 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
          .group-heading:hover {
             background-color: #f8f9fa!important;
         }
+        .table {
+            table-layout: fixed;
+        }
         .table .error-cell{
             word-break: break-word;
+            font-size: 14px;
+        }
+        pre {
+            overflow: scroll;
+            min-height: 30px;
+            max-width: 100%;
+        }
+        pre::-webkit-scrollbar {
+          -webkit-appearance: none;
+          width: 0px;
+          height: 5px;
+        }
+        pre::-webkit-scrollbar-thumb {
+          border-radius: 5px;
+          background-color: rgba(0,0,0,.5);
+          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
         }
     </style>
 </head>
 <body>
     `)
-//line app/vmalert/tpl/header.qtpl:41
+//line app/vmalert/tpl/header.qtpl:55
 	StreamPrintNavItems(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:41
+//line app/vmalert/tpl/header.qtpl:55
 	qw422016.N().S(`
     <main class="px-2">
 `)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 }
 
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	StreamHeader(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 }
 
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 func Header(title string, pages []NavItem) string {
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	WriteHeader(qb422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:43
+//line app/vmalert/tpl/header.qtpl:57
 }

app/vmalert/utils.go

@@ -30,3 +30,20 @@ func newTimeSeries(values []float64, timestamps []int64, labels map[string]strin
 	}
 	return ts
 }
+
+// newTimeSeriesPB creates prompbmarshal.TimeSeries with given
+// values, timestamps and labels.
+// It expects that labels are already sorted.
+func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{
+		Samples: make([]prompbmarshal.Sample, len(values)),
+	}
+	for i := range values {
+		ts.Samples[i] = prompbmarshal.Sample{
+			Value:     values[i],
+			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
+		}
+	}
+	ts.Labels = labels
+	return ts
+}

app/vmalert/utils/auth.go

@@ -1,18 +1,60 @@
 package utils
 
 import (
+	"strings"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
+// AuthConfigOptions options which helps build promauth.Config
+type AuthConfigOptions func(config *promauth.HTTPClientConfig)
+
 // AuthConfig returns promauth.Config based on the given params
-func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*promauth.Config, error) {
-	var baCfg *promauth.BasicAuthConfig
-	if baUser != "" || baPass != "" || baFile != "" {
-		baCfg = &promauth.BasicAuthConfig{
-			Username:     baUser,
-			Password:     promauth.NewSecret(baPass),
-			PasswordFile: baFile,
+func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
+	authCfg := &promauth.HTTPClientConfig{}
+	for _, option := range filterOptions {
+		option(authCfg)
+	}
+
+	return authCfg.NewConfig(".")
+}
+
+// WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
+func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if username != "" || password != "" || passwordFile != "" {
+			config.BasicAuth = &promauth.BasicAuthConfig{
+				Username:     username,
+				Password:     promauth.NewSecret(password),
+				PasswordFile: passwordFile,
+			}
+		}
+	}
+}
+
+// WithBearer returns AuthConfigOptions and set BearerToken or BearerTokenFile based on given params
+func WithBearer(token, tokenFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if token != "" {
+			config.BearerToken = promauth.NewSecret(token)
+		}
+		if tokenFile != "" {
+			config.BearerTokenFile = tokenFile
+		}
+	}
+}
+
+// WithOAuth returns AuthConfigOptions and set OAuth params based on given params
+func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if clientSecretFile != "" || clientSecret != "" {
+			config.OAuth2 = &promauth.OAuth2Config{
+				ClientID:         clientID,
+				ClientSecret:     promauth.NewSecret(clientSecret),
+				ClientSecretFile: clientSecretFile,
+				TokenURL:         tokenURL,
+				Scopes:           strings.Split(scopes, ";"),
+			}
 		}
 	}
-	return promauth.NewConfig(".", nil, baCfg, bearerToken, bearerTokenFile, nil, nil)
 }

app/vmalert/web.go

@@ -29,7 +29,7 @@ func initLinks() {
 		pathPrefix = "/"
 	}
 	apiLinks = [][2]string{
-		{path.Join(pathPrefix, "api/v1/groups"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/rules"), "list all loaded groups and rules"},
 		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
 		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
 		{path.Join(pathPrefix, "flags"), "command-line flags"},
@@ -69,13 +69,13 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/alerts":
 		WriteListAlerts(w, pathPrefix, rh.groupAlerts())
 		return true
-	case "/groups":
+	case "/groups", "/rules":
 		WriteListGroups(w, rh.groups())
 		return true
 	case "/notifiers":
 		WriteListTargets(w, notifier.GetTargets())
 		return true
-	case "/api/v1/groups":
+	case "/api/v1/rules":
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
@@ -127,10 +127,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 type listGroupsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Groups []APIGroup `json:"groups"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
 func (rh *requestHandler) groups() []APIGroup {
@@ -149,6 +149,7 @@ func (rh *requestHandler) groups() []APIGroup {
 
 	return groups
 }
+
 func (rh *requestHandler) listGroups() ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
 	lr.Data.Groups = rh.groups()
@@ -163,10 +164,10 @@ func (rh *requestHandler) listGroups() ([]byte, error) {
 }
 
 type listAlertsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Alerts []*APIAlert `json:"alerts"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
 func (rh *requestHandler) groupAlerts() []GroupAlerts {
@@ -181,7 +182,7 @@ func (rh *requestHandler) groupAlerts() []GroupAlerts {
 			if !ok {
 				continue
 			}
-			alerts = append(alerts, a.AlertsAPI()...)
+			alerts = append(alerts, a.AlertsToAPI()...)
 		}
 		if len(alerts) > 0 {
 			groupAlerts = append(groupAlerts, GroupAlerts{
@@ -204,7 +205,7 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 			if !ok {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
 		}
 	}
 

app/vmalert/web.qtpl

@@ -31,14 +31,7 @@
             rOk := make(map[string]int)
             rNotOk := make(map[string]int)
             for _, g := range groups {
-                for _, r := range g.AlertingRules{
-                    if r.LastError != "" {
-                        rNotOk[g.Name]++
-                    } else {
-                        rOk[g.Name]++
-                    }
-                }
-                 for _, r := range g.RecordingRules{
+                for _, r := range g.Rules {
                     if r.LastError != "" {
                         rNotOk[g.Name]++
                     } else {
@@ -52,7 +45,7 @@
         {% for _, g := range groups  %}
               <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
-                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%s g.Interval %})</a>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
                  {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
                 <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>
@@ -68,41 +61,44 @@
                 <table class="table table-striped table-hover table-sm">
                     <thead>
                         <tr>
-                            <th scope="col">Rule</th>
-                            <th scope="col" title="Shows if rule's execution ended with error">Error</th>
-                            <th scope="col" title="How many samples were produced by the rule">Samples</th>
-                            <th scope="col" title="How many seconds ago rule was executed">Updated</th>
+                            <th scope="col" style="width: 60%">Rule</th>
+                            <th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
+                            <th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                         </tr>
                     </thead>
                     <tbody>
-                    {% for _, ar := range g.AlertingRules %}
-                        <tr{% if ar.LastError != "" %} class="alert-danger"{% endif %}>
+                    {% for _, r := range g.Rules %}
+                        <tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
                             <td>
-                                <b>alert:</b> {%s ar.Name %} (for: {%v ar.For %})<br>
-                                <code><pre>{%s ar.Expression %}</pre></code><br>
-                                {% if len(ar.Labels) > 0 %} <b>Labels:</b>{% endif %}
-                                {% for k, v := range ar.Labels %}
-                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
-                                {% endfor %}
+                                <div class="row">
+                                    <div class="col-12 mb-2">
+                                        {% if r.Type == "alerting" %}
+                                        <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
+                                        {% else %}
+                                        <b>record:</b> {%s r.Name %}
+                                        {% endif %}
+                                    </div>
+                                    <div class="col-12">
+                                        <code><pre>{%s r.Query %}</pre></code>
+                                    </div>
+                                    <div class="col-12 mb-2">
+                                        {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                        {% for k, v := range r.Labels %}
+                                                <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                        {% endfor %}
+                                    </div>
+                                    {% if r.LastError != "" %}
+                                    <div class="col-12">
+                                        <b>Error:</b>
+                                        <div class="error-cell">
+                                        {%s r.LastError %}
+                                        </div>
+                                    </div>
+                                    {% endif %}
+                                </div>
                             </td>
-                            <td><div class="error-cell">{%s ar.LastError %}</div></td>
-                            <td>{%d ar.LastSamples %}</td>
-                            <td>{%f.3 time.Since(ar.LastExec).Seconds() %}s ago</td>
-                        </tr>
-                    {% endfor %}
-                    {% for _, rr := range g.RecordingRules  %}
-                        <tr>
-                            <td>
-                                <b>record:</b> {%s rr.Name %}<br>
-                                <code><pre>{%s rr.Expression %}</pre></code>
-                                {% if len(rr.Labels) > 0 %} <b>Labels:</b>{% endif %}
-                                {% for k, v := range rr.Labels %}
-                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
-                                {% endfor %}
-                            </td>
-                            <td><div class="error-cell">{%s rr.LastError %}</div></td>
-                            <td>{%d rr.LastSamples %}</td>
-                            <td>{%f.3 time.Since(rr.LastExec).Seconds() %}s ago</td>
+                            <td class="text-center">{%d r.LastSamples %}</td>
+                            <td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
                         </tr>
                     {% endfor %}
                  </tbody>
@@ -360,4 +356,4 @@
 
 {% func badgeRestored() %}
 <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
-{% endfunc %}
+{% endfunc %}

app/vmalert/web_test.go

@@ -14,7 +14,7 @@ func TestHandler(t *testing.T) {
 	ar := &AlertingRule{
 		Name: "alert",
 		alerts: map[uint64]*notifier.Alert{
-			0: {},
+			0: {State: notifier.StateFiring},
 		},
 	}
 	g := &Group{
@@ -54,9 +54,9 @@ func TestHandler(t *testing.T) {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 	})
-	t.Run("/api/v1/groups", func(t *testing.T) {
+	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
-		getResp(ts.URL+"/api/v1/groups", &lr, 200)
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}

app/vmalert/web_types.go

@@ -4,63 +4,61 @@ import (
 	"time"
 )
 
-// APIAlert represents an notifier.AlertingRule state
+// APIAlert represents a notifier.AlertingRule state
 // for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIAlert struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	RuleID      string            `json:"rule_id"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
 	State       string            `json:"state"`
+	Name        string            `json:"name"`
 	Value       string            `json:"value"`
-	Labels      map[string]string `json:"labels"`
+	Labels      map[string]string `json:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations"`
 	ActiveAt    time.Time         `json:"activeAt"`
-	SourceLink  string            `json:"source"`
-	Restored    bool              `json:"restored"`
+
+	// Additional fields
+
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// RuleID is an unique Rule's ID within a group
+	RuleID string `json:"rule_id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+	// Expression contains the PromQL/MetricsQL expression
+	// for Rule's evaluation
+	Expression string `json:"expression"`
+	// SourceLink contains a link to a system which should show
+	// why Alert was generated
+	SourceLink string `json:"source"`
+	// Restored shows whether Alert's state was restored on restart
+	Restored bool `json:"restored"`
 }
 
 // APIGroup represents Group for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIGroup struct {
-	Name           string             `json:"name"`
-	Type           string             `json:"type"`
-	ID             string             `json:"id"`
-	File           string             `json:"file"`
-	Interval       string             `json:"interval"`
-	Concurrency    int                `json:"concurrency"`
-	Params         []string           `json:"params"`
-	Labels         map[string]string  `json:"labels,omitempty"`
-	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
-	RecordingRules []APIRecordingRule `json:"recording_rules"`
-}
+	// Name is the group name as present in the config
+	Name string `json:"name"`
+	// Rules contains both recording and alerting rules
+	Rules []APIRule `json:"rules"`
+	// Interval is the Group's evaluation interval in float seconds as present in the file.
+	Interval float64 `json:"interval"`
+	// LastEvaluation is the timestamp of the last time the Group was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
 
-// APIAlertingRule represents AlertingRule for WEB view
-type APIAlertingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	For         string            `json:"for"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
-	Annotations map[string]string `json:"annotations"`
-}
+	// Additional fields
 
-// APIRecordingRule represents RecordingRule for WEB view
-type APIRecordingRule struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Type        string            `json:"type"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	LastError   string            `json:"last_error"`
-	LastSamples int               `json:"last_samples"`
-	LastExec    time.Time         `json:"last_exec"`
-	Labels      map[string]string `json:"labels"`
+	// Type shows the datasource type (prometheus or graphite) of the Group
+	Type string `json:"type"`
+	// ID is a unique Group ID
+	ID string `json:"id"`
+	// File contains a path to the file with Group's config
+	File string `json:"file"`
+	// Concurrency shows how many rules may be evaluated simultaneously
+	Concurrency int `json:"concurrency"`
+	// Params contains HTTP URL parameters added to each Rule's request
+	Params []string `json:"params,omitempty"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 // GroupAlerts represents a group of alerts for WEB view
@@ -68,3 +66,43 @@ type GroupAlerts struct {
 	Group  APIGroup
 	Alerts []*APIAlert
 }
+
+// APIRule represents a Rule for WEB view
+// see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
+type APIRule struct {
+	// State must be one of these under following scenarios
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state.
+	//  "firing": at least 1 alert in the rule in firing state.
+	//  "inactive": no alert in the rule in firing or pending state.
+	State string `json:"state"`
+	Name  string `json:"name"`
+	// Query represents Rule's `expression` field
+	Query string `json:"query"`
+	// Duration represents Rule's `for` field
+	Duration    float64           `json:"duration"`
+	Labels      map[string]string `json:"labels,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// LastError contains the error faced while executing the rule.
+	LastError string `json:"lastError"`
+	// EvaluationTime is the time taken to completely evaluate the rule in float seconds.
+	EvaluationTime float64 `json:"evaluationTime"`
+	// LastEvaluation is the timestamp of the last time the rule was executed
+	LastEvaluation time.Time `json:"lastEvaluation"`
+	// Alerts  is the list of all the alerts in this rule that are currently pending or firing
+	Alerts []*APIAlert `json:"alerts,omitempty"`
+	// Health is the health of rule evaluation.
+	// It MUST be one of "ok", "err", "unknown"
+	Health string `json:"health"`
+	// Type of the rule: recording or alerting
+	Type string `json:"type"`
+
+	// Additional fields
+
+	// Type of the rule: recording or alerting
+	DatasourceType string `json:"datasourceType"`
+	LastSamples    int    `json:"lastSamples"`
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+}

app/vmauth/README.md

@@ -1,7 +1,7 @@
 # vmauth
 
 `vmauth` is a simple auth proxy, router and [load balancer](#load-balancing) for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
-It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) and `Bearer token` is supported),
+It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), `Bearer token` and [InfluxDB authorization](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1897) is supported),
 matches them against configs pointed by [-auth.config](#auth-config) command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
 The `-auth.config` can point to either local file or to http url.
 
@@ -10,7 +10,7 @@ The `-auth.config` can point to either local file or to http url.
 Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and pass the following flag to `vmauth` binary in order to start authorizing and routing requests:
 
-```
+```bash
 /path/to/vmauth -auth.config=/path/to/auth/config.yml
 ```
 
@@ -39,7 +39,8 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl
 # Username and bearer_token values must be unique.
 
 users:
-  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
+  # Requests with the 'Authorization: Bearer XXXX' and 'Authorization: Token XXXX'
+  # header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
   # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
@@ -128,13 +129,13 @@ It is expected that all the backend services protected by `vmauth` are located i
 
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
 
-```
+```bash
   -tls
-    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+     Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+     Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+     Path to file with TLS key. Used only if -tls is set
 ```
 
 Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.
@@ -216,7 +217,7 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
 
 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
 
-```
+```bash
 ./vmauth -help
 
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
@@ -224,70 +225,73 @@ vmauth authenticates and authorizes incoming requests and proxies them to Victor
 See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
-    	Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+     Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
-    	Prefix for environment variables if -envflag.enable is set
+     Prefix for environment variables if -envflag.enable is set
   -eula
-    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -fs.disableMmap
-    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+     Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+     Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
-    	Timeout for incoming idle http connections (default 1m0s)
+     Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
-    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+     An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+     Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-    	TCP address to listen for http connections (default ":8427")
+     TCP address to listen for http connections (default ":8427")
   -logInvalidAuthTokens
-    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
+     Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
-    	Whether to disable writing timestamps in logs
+     Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
-    	Format for logs. Possible values: default, json (default "default")
+     Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+     Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
-    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+     Output for the logs. Supported values: stderr, stdout (default "stderr")
   -loggerTimezone string
-    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+     Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxIdleConnsPerBackend int
-    	The maximum number of idle connections vmauth can open per each backend host (default 100)
+     The maximum number of idle connections vmauth can open per each backend host (default 100)
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -reloadAuthKey string
-    	Auth key for /-/reload http endpoint. It must be passed as authKey=...
+     Auth key for /-/reload http endpoint. It must be passed as authKey=...
   -tls
-    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+     Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+  -tlsCipherSuites array
+     Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
+     Supports an array of values separated by comma or specified via multiple flags.
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+     Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
   -version
-    	Show VictoriaMetrics version
+     Show VictoriaMetrics version
 ```

app/vmauth/example_config.yml

@@ -2,7 +2,8 @@
 # Username and bearer_token values must be unique.
 
 users:
-  # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
+  # Requests with the 'Authorization: Bearer XXXX' and 'Authorization: Token XXXX'
+  # header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
   # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"

app/vmauth/main.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -72,6 +73,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
+	if strings.HasPrefix(authToken, "Token ") {
+		// Handle InfluxDB's proprietary token authentication scheme as a bearer token authentication
+		// See https://docs.influxdata.com/influxdb/v2.0/api/
+		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
+	}
 	ac := authConfig.Load().(map[string]*UserInfo)
 	ui := ac[authToken]
 	if ui == nil {

app/vmbackup/README.md

@@ -7,7 +7,7 @@ Supported storage systems for backups:
 * [GCS](https://cloud.google.com/storage/). Example: `gs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
 * Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/en/pacific/radosgw/s3/) or [Swift](https://platform.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
-* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`. Note that `vmbackup` prevents from storing the backup into the directory pointed by `-storageDataPath` command-line flag, since this directory should be managed solely by VictoriaMetrics or `vmstorage`.
 
 `vmbackup` supports incremental and full backups. Incremental backups are created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to an already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
@@ -22,45 +22,41 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 See also [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html) tool built on top of `vmbackup`. This tool simplifies
 creation of hourly, daily, weekly and monthly backups.
 
-
 ## Use cases
 
 ### Regular backups
 
 Regular backup can be performed with the following command:
 
-```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup>
+```bash
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup>
 ```
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups since they are performed from immutable [instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` can create the snapshot on itself if `-snapshot.createURL` command-line flag is set to an url for creating snapshots. In this case `-snapshotName` flag isn't needed.
+* `http://victoriametrics:8428/snapshot/create` is the url for creating snapshots according to [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` creates a snapshot by querying the provided `-snapshot.createURL`, then performs the backup and then automatically removes the created snapshot.
 * `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
-
 ### Regular backups with server-side copy from existing backup
 
 If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
 with the following command:
 
-```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
+```bash
+./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
 ```
 
 It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
-
 ### Incremental backups
 
 Incremental backups are performed if `-dst` points to an already existing backup. In this case only new data is uploaded to remote storage.
 It saves time and network bandwidth costs when working with big backups:
 
+```bash
+./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/existing/backup>
 ```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/existing/backup>
-```
-
 
 ### Smart backups
 
@@ -68,8 +64,8 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati
 
 * Run the following command every hour:
 
-```
-vmbackup -snapshotName=<latest-snapshot> -dst=gs://<bucket>/latest
+```bash
+./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/latest
 ```
 
 Where `<latest-snapshot>` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
@@ -77,47 +73,46 @@ The command will upload only changed data to `gs://<bucket>/latest`.
 
 * Run the following command once a day:
 
-```
-vmbackup -snapshotName=<daily-snapshot> -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
+```bash
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
 ```
 
 Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
 
-
 This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
 or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
 
-Do not forget to remove old snapshots and backups when they are no longer needed in order to save storage costs.
+Do not forget to remove old backups when they are no longer needed in order to save storage costs.
 
 See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
 
-
 ## How does it work?
 
 The backup algorithm is the following:
 
-1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
-2. Determine which files in `-dst` are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
-3. Determine which files in `-snapshotName` are missing in `-dst`. These are usually small new files and bigger merged files.
-4. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+1. Create a snapshot by querying the provided `-snapshot.createURL`
+2. Collect information about files in the created snapshot, in the `-dst` and in the `-origin`.
+3. Determine which files in `-dst` are missing in the created snapshot, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+4. Determine which files in the created snapshot are missing in `-dst`. These are usually small new files and bigger merged files.
+5. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
    These are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
+6. Upload the remaining files from step 3 from the created snapshot to `-dst`.
+7. Delete the created snapshot.
 
 The algorithm splits source files into 1 GiB chunks in the backup. Each chunk is stored as a separate file in the backup.
-Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+Such splitting balances between the number of files in the backup and the amounts of data that needs to be re-transfered after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
-- All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
-- Smaller files have higher probability to be merged.
-- Consecutive snapshots share many identical files.
+* All the files in the snapshot are immutable.
+* Old files are periodically merged into new files.
+* Smaller files have higher probability to be merged.
+* Consecutive snapshots share many identical files.
 
 These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
 See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
 `vmbackup` can work improperly or slowly when these properties are violated.
 
-
 ## Troubleshooting
 
 * If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
@@ -126,15 +121,14 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 * Backups created from [single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) cannot be restored
   at [cluster VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) and vice versa.
 
-
 ## Advanced usage
 
-
 * Obtaining credentials from a file.
 
   Add flag `-credsFilePath=/etc/credentials` with the following content:
 
     for s3 (aws, minio or other s3 compatible storages):
+
      ```bash
      [default]
      aws_access_key_id=theaccesskey
@@ -142,6 +136,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
     ```
 
     for gce cloud storage:
+
     ```json
     {
            "type": "service_account",
@@ -159,7 +154,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 * Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
   You have to add a custom url endpoint via flag:
-```
+
+```bash
   # for minio
   -customS3Endpoint=http://localhost:9000
 
@@ -169,102 +165,100 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 * Run `vmbackup -help` in order to see all the available options:
 
-```
+```bash
   -concurrency int
-    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+     The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
   -configFilePath string
-    	Path to file with S3 configs. Configs are loaded from default location if not set.
-    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+     Path to file with S3 configs. Configs are loaded from default location if not set.
+     See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
+     Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
-    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
-    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+     Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+     See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -customS3Endpoint string
-    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+     Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
   -dst string
-    	Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
-    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+     Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+     -dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
   -enableTCP6
-    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
-    	Prefix for environment variables if -envflag.enable is set
+     Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
-    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+     Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+     Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
-    	Timeout for incoming idle http connections (default 1m0s)
+     Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
-    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+     An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+     Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-    	TCP address for exporting metrics at /metrics page (default ":8420")
+     TCP address for exporting metrics at /metrics page (default ":8420")
   -loggerDisableTimestamps
-    	Whether to disable writing timestamps in logs
+     Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
-    	Format for logs. Possible values: default, json (default "default")
+     Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+     Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
-    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+     Output for the logs. Supported values: stderr, stdout (default "stderr")
   -loggerTimezone string
-    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+     Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+     Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxBytesPerSecond size
-    	The maximum upload speed. There is no limit if it is set to 0
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     The maximum upload speed. There is no limit if it is set to 0
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedBytes size
-    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -origin string
-    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+     Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -s3ForcePathStyle
-    	Prefixing endpoint with bucket name when set false, true by default. (default true)
+     Prefixing endpoint with bucket name when set false, true by default. (default true)
   -snapshot.createURL string
-    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
+     VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
   -snapshot.deleteURL string
-    	VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete
+     VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete
   -snapshotName string
-    	Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set
+     Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set
   -storageDataPath string
-    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+     Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
   -tls
-    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+     Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+     Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
   -tlsKeyFile string
-    	Path to file with TLS key. Used only if -tls is set
+     Path to file with TLS key. Used only if -tls is set
   -version
-    	Show VictoriaMetrics version
+     Show VictoriaMetrics version
 ```
 
-
 ## How to build from sources
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
 
-
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.

app/vmbackup/main.go

@@ -4,10 +4,10 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
@@ -17,6 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
 )
 
 var (
@@ -71,6 +72,11 @@ func main() {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
 		}()
+	} else if len(*snapshotName) == 0 {
+		logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
+	}
+	if err := snapshot.Validate(*snapshotName); err != nil {
+		logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
 	}
 
 	go httpserver.Serve(*httpListenAddr, nil)
@@ -119,9 +125,6 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
 }
 
 func newSrcFS() (*fslocal.FS, error) {
-	if len(*snapshotName) == 0 {
-		return nil, fmt.Errorf("`-snapshotName` or `-snapshot.createURL` must be provided")
-	}
 	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
 
 	// Verify the snapshot exists.
@@ -153,9 +156,28 @@ func newDstFS() (common.RemoteFS, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
+	if hasFilepathPrefix(*dst, *storageDataPath) {
+		return nil, fmt.Errorf("-dst=%q can not point to the directory with VictoriaMetrics data (aka -storageDataPath=%q)", *dst, *storageDataPath)
+	}
 	return fs, nil
 }
 
+func hasFilepathPrefix(path, prefix string) bool {
+	if !strings.HasPrefix(path, "fs://") {
+		return false
+	}
+	path = path[len("fs://"):]
+	pathAbs, err := filepath.Abs(path)
+	if err != nil {
+		return false
+	}
+	prefixAbs, err := filepath.Abs(prefix)
+	if err != nil {
+		return false
+	}
+	return strings.HasPrefix(pathAbs, prefixAbs)
+}
+
 func newOriginFS() (common.OriginFS, error) {
 	if len(*origin) == 0 {
 		return &fsnil.FS{}, nil

app/vmbackup/main_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func TestHasFilepathPrefix(t *testing.T) {
+	f := func(dst, storageDataPath string, resultExpected bool) {
+		t.Helper()
+		result := hasFilepathPrefix(dst, storageDataPath)
+		if result != resultExpected {
+			t.Errorf("unexpected hasFilepathPrefix(%q, %q); got: %v; want: %v", dst, storageDataPath, result, resultExpected)
+		}
+	}
+	pwd, err := filepath.Abs("")
+	if err != nil {
+		t.Fatalf("cannot determine working directory: %s", err)
+	}
+	f("s3://foo/bar", "foo", false)
+	f("fs://"+pwd+"/foo", "foo", true)
+	f("fs://"+pwd+"/foo", "foo/bar", false)
+	f("fs://"+pwd+"/foo/bar", "foo", true)
+	f("fs://"+pwd+"/foo", "bar", false)
+	f("fs://"+pwd+"/foo", pwd+"/foo", true)
+	f("fs://"+pwd+"/foo", pwd+"/foo/bar", false)
+	f("fs://"+pwd+"/foo/bar", pwd+"/foo", true)
+	f("fs://"+pwd+"/foo", pwd+"/bar", false)
+}

app/vmbackupmanager/README.md

@@ -9,11 +9,10 @@ The required flags for running the service are as follows:
 
 * -eula - should be true and means that you have the legal right to run a backup manager. That can either be a signed contract or an email with confirmation to run the service in a trial period
 * -storageDataPath - path to VictoriaMetrics or vmstorage data path to make backup from
-* -snapshot.createURL - VictoriaMetrics creates snapshot URL which will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create
+* -snapshot.createURL - VictoriaMetrics creates snapshot URL which will automatically be created during backup. Example: <http://victoriametrics:8428/snapshot/create>
 * -dst - backup destination at s3, gcs or local filesystem
 * -credsFilePath - path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See [https://cloud.google.com/iam/docs/creating-managing-service-account-keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) and [https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html](https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html)
 
-
 Backup schedule is controlled by the following flags:
 
 * -disableHourly - disable hourly run. Default false
@@ -23,7 +22,6 @@ Backup schedule is controlled by the following flags:
 
 By default, all flags are turned on and Backup Manager backups data every hour for every interval (hourly, daily, weekly and monthly).
 
-
 The backup manager creates the following directory hierarchy at **-dst**:
 
 * /latest/ - contains the latest backup
@@ -32,7 +30,6 @@ The backup manager creates the following directory hierarchy at **-dst**:
 * /weekly/ - contains weekly backups. Each backup is named as *YYYY-WW*
 * /monthly/ - contains monthly backups. Each backup is named as *YYYY-MM*
 
-
 To get the full list of supported flags please run the following command:
 
 ```console
@@ -48,7 +45,6 @@ There are two flags which could help with performance tuning:
 * -maxBytesPerSecond - the maximum upload speed. There is no limit if it is set to 0
 * -concurrency - The number of concurrent workers. Higher concurrency may improve upload speed (default 10)
 
-
 ## Example of Usage
 
 GCS and cluster version. You need to have a credentials file in json format with following structure
@@ -96,11 +92,11 @@ info    VictoriaMetrics/lib/storage/storage.go:319    deleted snapshot "/vmstora
 
 The result on the GCS bucket
 
-- The root folder
+* The root folder
   
   ![root](vmbackupmanager_root_folder.png)
 
-- The latest folder
+* The latest folder
 
   ![latest](vmbackupmanager_latest_folder.png)
 
@@ -119,7 +115,6 @@ Let’s assume we have a backup manager collecting daily backups for the past 10
 
 ![daily](vmbackupmanager_rp_daily_1.png)
 
-
 We enable backup retention policy for backup manager by using following configuration:
 
 ```console
@@ -144,3 +139,122 @@ info    app/vmbackupmanager/retention.go:106    daily backups to delete [daily/2
 The result on the GCS bucket. We see only 3 daily backups:
 
 ![daily](vmbackupmanager_rp_daily_2.png)
+
+
+## Configuration
+
+### Flags
+
+Pass `-help` to `vmbackupmanager` in order to see the full list of supported
+command-line flags with their descriptions.
+
+The shortlist of configuration flags is the following:
+
+```
+vmbackupmanager performs regular backups according to the provided configs.
+
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -disableDaily
+    	Disable daily run. Default false
+  -disableHourly
+    	Disable hourly run. Default false
+  -disableMonthly
+    	Disable monthly run. Default false
+  -disableWeekly
+    	Disable weekly run. Default false
+  -dst string
+    	The root folder of Victoria Metrics backups. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	Address to listen for http connections (default ":8300")
+  -keepLastDaily int
+    	Keep last N daily backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
+  -keepLastHourly int
+    	Keep last N hourly backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
+  -keepLastMonthly int
+    	Keep last N monthly backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
+  -keepLastWeekly int
+    	Keep last N weekly backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
+  -loggerDisableTimestamps
+    	Whether to disable writing timestamps in logs
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -loggerTimezone string
+    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+  -loggerWarnsPerSecondLimit int
+    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedBytes size
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -runOnStart
+    	Upload backups immediately after start of the service. Otherwise the backup starts on new hour
+  -s3ForcePathStyle
+    	Prefixing endpoint with bucket name when set false, true by default. (default true)
+  -snapshot.createURL string
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snapshot/create
+  -snapshot.deleteURL string
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshot.createURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snapshot/delete
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -tls
+    	Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated
+  -tlsCipherSuites array
+    	Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
+    	Supports an array of values separated by comma or specified via multiple flags.
+  -tlsKeyFile string
+    	Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+  -version
+    	Show VictoriaMetrics version
+```

app/vmctl/README.md

@@ -2,20 +2,21 @@
 
 VictoriaMetrics command-line tool
 
-Features:
-- [x] Prometheus: migrate data from Prometheus to VictoriaMetrics using snapshot API
-- [x] Thanos: migrate data from Thanos to VictoriaMetrics
-- [ ] ~~Prometheus: migrate data from Prometheus to VictoriaMetrics by query~~(discarded)
-- [x] InfluxDB: migrate data from InfluxDB to VictoriaMetrics
-- [x] OpenTSDB: migrate data from OpenTSDB to VictoriaMetrics
-- [ ] Storage Management: data re-balancing between nodes 
+vmctl provides various useful actions with VictoriaMetrics components.
 
-vmctl acts as a proxy between data source ([Prometheus](#migrating-data-from-prometheus), 
-[InfluxDB](#migrating-data-from-influxdb-1x), [VictoriaMetrics](##migrating-data-from-victoriametrics), etc.)
-and destination - VictoriaMetrics single or cluster version. To see the full list of supported modes 
+Features:
+- migrate data from [Prometheus](#migrating-data-from-prometheus) to VictoriaMetrics using snapshot API
+- migrate data from [Thanos](#migrating-data-from-thanos) to VictoriaMetrics
+- migrate data from [InfluxDB](#migrating-data-from-influxdb-1x) to VictoriaMetrics
+- migrate data from [OpenTSDB](#migrating-data-from-opentsdb) to VictoriaMetrics
+- migrate data between [VictoriaMetrics](#migrating-data-from-victoriametrics) single or cluster version.
+- [verify](#verifying-exported-blocks-from-victoriametrics) exported blocks from VictoriaMetrics single or cluster version.
+
+To see the full list of supported modes
 run the following command:
-```
-./vmctl --help                                        
+
+```bash
+$ ./vmctl --help                                        
 NAME:
    vmctl - VictoriaMetrics command-line tool
 
@@ -27,12 +28,14 @@ COMMANDS:
    influx      Migrate timeseries from InfluxDB
    prometheus  Migrate timeseries from Prometheus
    vm-native   Migrate time series between VictoriaMetrics installations via native binary format
+   verify-block  Verifies correctness of data blocks exported via VictoriaMetrics Native format. See https://docs.victoriametrics.com/#how-to-export-data-in-native-format
 ```
 
 Each mode has its own unique set of flags specific (e.g. prefixed with `influx` for influx mode)
 to the data source and common list of flags for destination (prefixed with `vm` for VictoriaMetrics):
+
 ```
-./vmctl influx --help
+$ ./vmctl influx --help
 OPTIONS:
    --influx-addr value              InfluxDB server addr (default: "http://localhost:8086")
    --influx-user value              InfluxDB user [$INFLUX_USERNAME]
@@ -46,12 +49,13 @@ Please note, that vmctl performs initial readiness check for the given address b
 ```
 
 When doing a migration user needs to specify flags for source (where and how to fetch data) and for
-destination (where to migrate data). Every mode has additional details and nuances, please see 
+destination (where to migrate data). Every mode has additional details and nuances, please see
 them below in corresponding sections.
 
 For the destination flags see the full description by running the following command:
+
 ```
-./vmctl influx --help | grep vm-
+$ ./vmctl influx --help | grep vm-
 ```
 
 Some flags like [--vm-extra-label](#adding-extra-labels) or [--vm-significant-figures](#significant-figures)
@@ -59,14 +63,13 @@ has additional sections with description below. Details about tweaking and adjus
 are explained in [Tuning](#tuning) section.
 
 Please note, that if you're going to import data into VictoriaMetrics cluster do not
-forget to specify the `--vm-account-id` flag. See more details for cluster version 
+forget to specify the `--vm-account-id` flag. See more details for cluster version
 [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 ## Articles
 
-* [How to migrate data from Prometheus](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
-* [How to migrate data from Prometheus. Filtering and modifying time series](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
-
+- [How to migrate data from Prometheus](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
+- [How to migrate data from Prometheus. Filtering and modifying time series](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
 
 ## Migrating data from OpenTSDB
 
@@ -74,26 +77,33 @@ forget to specify the `--vm-account-id` flag. See more details for cluster versi
 
 See `./vmctl opentsdb --help` for details and full list of flags.
 
-*OpenTSDB migration is not possible without a functioning [meta](http://opentsdb.net/docs/build/html/user_guide/metadata.html) table to search for metrics/series.*
+**Important:** OpenTSDB migration is not possible without a functioning [meta](http://opentsdb.net/docs/build/html/user_guide/metadata.html) table to search for metrics/series. Check in OpenTSDB config that appropriate options are [activated]( https://github.com/OpenTSDB/opentsdb/issues/681#issuecomment-177359563) and HBase meta tables are present. W/o them migration won't work.
 
 OpenTSDB migration works like so:
 
-1. Find metrics based on selected filters (or the default filter set ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'])
-  * e.g. `curl -Ss "http://opentsdb:4242/api/suggest?type=metrics&q=sys"`
+1. Find metrics based on selected filters (or the default filter set `['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']`)
+
+- e.g. `curl -Ss "http://opentsdb:4242/api/suggest?type=metrics&q=sys"`
+
 2. Find series associated with each returned metric
-  * e.g. `curl -Ss "http://opentsdb:4242/api/search/lookup?m=system.load5&limit=1000000"`
+
+- e.g. `curl -Ss "http://opentsdb:4242/api/search/lookup?m=system.load5&limit=1000000"`
+
+Here `results` return field should not be empty. Otherwise it means that meta tables are absent and needs to be turned on previously.
+
 3. Download data for each series in chunks defined in the CLI switches
-  * e.g. `-retention=sum-1m-avg:1h:90d` ==
-    * `curl -Ss "http://opentsdb:4242/api/query?start=1h-ago&end=now&m=sum:1m-avg-none:system.load5\{host=host1\}"`
-    * `curl -Ss "http://opentsdb:4242/api/query?start=2h-ago&end=1h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
-    * `curl -Ss "http://opentsdb:4242/api/query?start=3h-ago&end=2h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
-    * ...
-    * `curl -Ss "http://opentsdb:4242/api/query?start=2160h-ago&end=2159h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+
+- e.g. `-retention=sum-1m-avg:1h:90d` means
+  - `curl -Ss "http://opentsdb:4242/api/query?start=1h-ago&end=now&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+  - `curl -Ss "http://opentsdb:4242/api/query?start=2h-ago&end=1h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+  - `curl -Ss "http://opentsdb:4242/api/query?start=3h-ago&end=2h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+  - ...
+  - `curl -Ss "http://opentsdb:4242/api/query?start=2160h-ago&end=2159h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
 
 This means that we must stream data from OpenTSDB to VictoriaMetrics in chunks. This is where concurrency for OpenTSDB comes in. We can query multiple chunks at once, but we shouldn't perform too many chunks at a time to avoid overloading the OpenTSDB cluster.
 
 ```
-$ bin/vmctl opentsdb --otsdb-addr http://opentsdb:4242/ --otsdb-retentions sum-1m-avg:1h:1d --otsdb-filters system --otsdb-normalize --vm-addr http://victoria/
+$ ./vmctl opentsdb --otsdb-addr http://opentsdb:4242/ --otsdb-retentions sum-1m-avg:1h:1d --otsdb-filters system --otsdb-normalize --vm-addr http://victoria:8428/
 OpenTSDB import mode
 2021/04/09 11:52:50 Will collect data starting at TS 1617990770
 2021/04/09 11:52:50 Loading all metrics from OpenTSDB for filters:  [system]
@@ -101,12 +111,21 @@ Found 9 metrics to import. Continue? [Y/n]
 2021/04/09 11:52:51 Starting work on system.load1
 23 / 402200 [>____________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________] 0.01% 2 p/s
 ```
+Where `:8428` is Prometheus port of VictoriaMetrics.
+
+For clustered VictoriaMetrics setup `--vm-account-id` flag needs to be added, for example:
+
+```
+$ ./vmctl opentsdb --otsdb-addr http://opentsdb:4242/ --otsdb-retentions sum-1m-avg:1h:1d --otsdb-filters system --otsdb-normalize --vm-addr http://victoria:8480/ --vm-account-id 0
+```
+This time `:8480` port is vminsert/Prometheus input port.
 
 ### Retention strings
 
 Starting with a relatively simple retention string (`sum-1m-avg:1h:30d`), let's describe how this is converted into actual queries.
 
 There are two essential parts of a retention string:
+
 1. [aggregation](#aggregation)
 2. [windows/time ranges](#windows)
 
@@ -115,8 +134,9 @@ There are two essential parts of a retention string:
 Retention strings essentially define the two levels of aggregation for our collected series.
 
 `sum-1m-avg` would become:
-* First order: `sum`
-* Second order: `1m-avg-none`
+
+- First order: `sum`
+- Second order: `1m-avg-none`
 
 ##### First Order Aggregations
 
@@ -137,6 +157,7 @@ We do not allow for defining the "null value" portion of the rollup window (e.g.
 #### Windows
 
 There are two important windows we define in a retention string:
+
 1. the "chunk" range of each query
 2. The time range we will be querying on with that "chunk"
 
@@ -182,8 +203,8 @@ See `./vmctl influx --help` for details and full list of flags.
 
 To use migration tool please specify the InfluxDB address `--influx-addr`, the database `--influx-database` and VictoriaMetrics address `--vm-addr`.
 Flag `--vm-addr` for single-node VM is usually equal to `--httpListenAddr`, and for cluster version
-is equal to `--httpListenAddr` flag of vminsert component. Please note, that vmctl performs initial readiness check for the given address 
-by checking `/health` endpoint. For cluster version it is additionally required to specify the `--vm-account-id` flag. 
+is equal to `--httpListenAddr` flag of vminsert component. Please note, that vmctl performs initial readiness check for the given address
+by checking `/health` endpoint. For cluster version it is additionally required to specify the `--vm-account-id` flag.
 See more details for cluster version [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 As soon as required flags are provided and all endpoints are accessible, `vmctl` will start the InfluxDB scheme exploration.
@@ -191,8 +212,9 @@ Basically, it just fetches all fields and timeseries from the provided database
 Then `vmctl` sends fetch requests for each timeseries to InfluxDB one by one and pass results to VM importer.
 VM importer then accumulates received samples in batches and sends import requests to VM.
 
-The importing process example for local installation of InfluxDB(`http://localhost:8086`) 
+The importing process example for local installation of InfluxDB(`http://localhost:8086`)
 and single-node VictoriaMetrics(`http://localhost:8428`):
+
 ```
 ./vmctl influx --influx-database benchmark
 InfluxDB import mode
@@ -212,25 +234,27 @@ Found 40000 timeseries to import. Continue? [Y/n] y
   bytes/s: 5.4 MB;
   import requests: 40001;
 2020/01/18 21:19:00 Total time: 31m48.467044016s
-``` 
+```
 
 ### Data mapping
 
 Vmctl maps InfluxDB data the same way as VictoriaMetrics does by using the following rules:
 
-* `influx-database` arg is mapped into `db` label value unless `db` tag exists in the InfluxDB line.
-* Field names are mapped to time series names prefixed with {measurement}{separator} value, 
-where {separator} equals to _ by default. 
+- `influx-database` arg is mapped into `db` label value unless `db` tag exists in the InfluxDB line.
+- Field names are mapped to time series names prefixed with {measurement}{separator} value,
+where {separator} equals to _ by default.
 It can be changed with `--influx-measurement-field-separator` command-line flag.
-* Field values are mapped to time series values.
-* Tags are mapped to Prometheus labels format as-is.
+- Field values are mapped to time series values.
+- Tags are mapped to Prometheus labels format as-is.
 
 For example, the following InfluxDB line:
+
 ```
 foo,tag1=value1,tag2=value2 field1=12,field2=40
 ```
 
 is converted into the following Prometheus format data points:
+
 ```
 foo_field1{tag1="value1", tag2="value2"} 12
 foo_field2{tag1="value1", tag2="value2"} 40
@@ -238,7 +262,7 @@ foo_field2{tag1="value1", tag2="value2"} 40
 
 ### Configuration
 
-The configuration flags should contain self-explanatory descriptions. 
+The configuration flags should contain self-explanatory descriptions.
 
 ### Filtering
 
@@ -246,6 +270,7 @@ The filtering consists of two parts: timeseries and time.
 The first step of application is to select all available timeseries
 for given database and retention. User may specify additional filtering
 condition via `--influx-filter-series` flag. For example:
+
 ```
 ./vmctl influx --influx-database benchmark \
   --influx-filter-series "on benchmark from cpu where hostname='host_1703'"
@@ -256,13 +281,15 @@ InfluxDB import mode
 2020/01/26 14:23:29 fetching series: command: "show series on benchmark from cpu where hostname='host_1703'"; database: "benchmark"; retention: "autogen"
 Found 10 timeseries to import. Continue? [Y/n] 
 ```
+
 The timeseries select query would be following:
  `fetching series: command: "show series on benchmark from cpu where hostname='host_1703'"; database: "benchmark"; retention: "autogen"`
- 
+
 The second step of filtering is a time filter and it applies when fetching the datapoints from Influx.
 Time filtering may be configured with two flags:
-* --influx-filter-time-start 
-* --influx-filter-time-end 
+
+- --influx-filter-time-start
+- --influx-filter-time-end
 Here's an example of importing timeseries for one day only:
 `./vmctl influx --influx-database benchmark --influx-filter-series "where hostname='host_1703'" --influx-filter-time-start "2020-01-01T10:07:00Z" --influx-filter-time-end "2020-01-01T15:07:00Z"`
 
@@ -271,36 +298,36 @@ Please see more about time filtering [here](https://docs.influxdata.com/influxdb
 ## Migrating data from InfluxDB (2.x)
 
 Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
-You may find useful a 3rd party solution for this - https://github.com/jonppe/influx_to_victoriametrics.
-
+You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
 
 ## Migrating data from Prometheus
 
 `vmctl` supports the `prometheus` mode for migrating data from Prometheus to VictoriaMetrics time-series database.
-Migration is based on reading Prometheus snapshot, which is basically a hard-link to Prometheus data files. 
+Migration is based on reading Prometheus snapshot, which is basically a hard-link to Prometheus data files.
 
 See `./vmctl prometheus --help` for details and full list of flags. Also see Prometheus related articles [here](#articles).
 
 To use migration tool please specify the file path to Prometheus snapshot `--prom-snapshot` (see how to make a snapshot [here](https://www.robustperception.io/taking-snapshots-of-prometheus-data)) and VictoriaMetrics address `--vm-addr`.
 Please note, that `vmctl` *do not make a snapshot from Prometheus*, it uses an already prepared snapshot. More about Prometheus snapshots may be found [here](https://www.robustperception.io/taking-snapshots-of-prometheus-data) and [here](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043).
 Flag `--vm-addr` for single-node VM is usually equal to `--httpListenAddr`, and for cluster version
-is equal to `--httpListenAddr` flag of vminsert component. Please note, that vmctl performs initial readiness check for the given address 
-by checking `/health` endpoint. For cluster version it is additionally required to specify the `--vm-account-id` flag. 
+is equal to `--httpListenAddr` flag of vminsert component. Please note, that vmctl performs initial readiness check for the given address
+by checking `/health` endpoint. For cluster version it is additionally required to specify the `--vm-account-id` flag.
 See more details for cluster version [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 As soon as required flags are provided and all endpoints are accessible, `vmctl` will start the Prometheus snapshot exploration.
 Basically, it just fetches all available blocks in provided snapshot and read the metadata. It also does initial filtering by time
 if flags `--prom-filter-time-start` or `--prom-filter-time-end` were set. The exploration procedure prints some stats from read blocks.
 Please note that stats are not taking into account timeseries or samples filtering. This will be done during importing process.
- 
+
 The importing process takes the snapshot blocks revealed from Explore procedure and processes them one by one
 accumulating timeseries and samples. Please note, that `vmctl` relies on responses from InfluxDB on this stage,
-so ensure that Explore queries are executed without errors or limits. Please see this 
+so ensure that Explore queries are executed without errors or limits. Please see this
 [issue](https://github.com/VictoriaMetrics/vmctl/issues/30) for details.
 The data processed in chunks and then sent to VM.
 
-The importing process example for local installation of Prometheus 
+The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
+
 ```
 ./vmctl prometheus --prom-snapshot=/path/to/snapshot \
   --vm-concurrency=1 \
@@ -327,7 +354,7 @@ Found 14 blocks to import. Continue? [Y/n] y
   import requests: 323;
   import requests retries: 0;
 2020/02/23 15:50:03 Total time: 51.077451066s
-``` 
+```
 
 ### Data mapping
 
@@ -336,7 +363,7 @@ So no data changes will be applied.
 
 ### Configuration
 
-The configuration flags should contain self-explanatory descriptions. 
+The configuration flags should contain self-explanatory descriptions.
 
 ### Filtering
 
@@ -347,6 +374,7 @@ in in RFC3339 format. This filter applied twice: to drop blocks out of range and
 overlapping time range.
 
 Example of applying time filter:
+
 ```
 ./vmctl prometheus --prom-snapshot=/path/to/snapshot \
   --prom-filter-time-start=2020-02-07T00:07:01Z \
@@ -366,12 +394,13 @@ Please notice, that total amount of blocks in provided snapshot is 14, but only
 time range. So other 12 blocks were marked as `skipped`. The amount of samples and series is not taken into account,
 since this is heavy operation and will be done during import process.
 
+Filtering by timeseries is configured with following flags:
 
-Filtering by timeseries is configured with following flags: 
-* `--prom-filter-label` - the label name, e.g. `__name__` or `instance`;
-* `--prom-filter-label-value` - the regular expression to filter the label value. By default matches all `.*`
+- `--prom-filter-label` - the label name, e.g. `__name__` or `instance`;
+- `--prom-filter-label-value` - the regular expression to filter the label value. By default matches all `.*`
 
 For example:
+
 ```
 ./vmctl prometheus --prom-snapshot=/path/to/snapshot \
   --prom-filter-label="__name__" \
@@ -405,38 +434,44 @@ Found 2 blocks to import. Continue? [Y/n] y
 
 Thanos uses the same storage engine as Prometheus and the data layout on-disk should be the same. That means
 `vmctl` in mode `prometheus` may be used for Thanos historical data migration as well.
-These instructions may vary based on the details of your Thanos configuration. 
-Please read carefully and verify as you go. We assume you're using Thanos Sidecar on your Prometheus pods, 
+These instructions may vary based on the details of your Thanos configuration.
+Please read carefully and verify as you go. We assume you're using Thanos Sidecar on your Prometheus pods,
 and that you have a separate Thanos Store installation.
 
 ### Current data
 
-1. For now, keep your Thanos Sidecar and Thanos-related Prometheus configuration, but add this to also stream 
+1. For now, keep your Thanos Sidecar and Thanos-related Prometheus configuration, but add this to also stream
     metrics to VictoriaMetrics:
+
     ```
     remote_write:
     - url: http://victoria-metrics:8428/api/v1/write
     ```
-2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving. 
+
+2. Make sure VM is running, of course. Now check the logs to make sure that Prometheus is sending and VM is receiving.
     In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
+
     ```
-    2020-04-27T18:38:46.474Z	info	VictoriaMetrics/lib/storage/partition.go:207	creating a partition "2020_04" with smallPartsPath="/victoria-metrics-data/data/small/2020_04", bigPartsPath="/victoria-metrics-data/data/big/2020_04"
-    2020-04-27T18:38:46.506Z	info	VictoriaMetrics/lib/storage/partition.go:222	partition "2020_04" has been created
+    2020-04-27T18:38:46.474Z info VictoriaMetrics/lib/storage/partition.go:207 creating a partition "2020_04" with smallPartsPath="/victoria-metrics-data/data/small/2020_04", bigPartsPath="/victoria-metrics-data/data/big/2020_04"
+    2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2020_04" has been created
     ```
+
 3. Now just wait. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
     storage.
 
 ### Historical data
 
-Let's assume your data is stored on S3 served by minio. You first need to copy that out to a local filesystem, 
+Let's assume your data is stored on S3 served by minio. You first need to copy that out to a local filesystem,
 then import it into VM using `vmctl` in `prometheus` mode.
+
 1. Copy data from minio.
     1. Run the `minio/mc` Docker container.
     1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
     1. `mc cp -r minio/prometheus thanos-data`
 1. Import using `vmctl`.
     1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
-    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data: 
+    1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
+
     ```
     vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
     ```
@@ -453,8 +488,8 @@ or higher.
 
 See `./vmctl vm-native --help` for details and full list of flags.
 
-In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`) 
-and processing is done by "destination" (`dst`). Because of that, `vmctl` doesn't actually know how much data will be 
+In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`)
+and processing is done by "destination" (`dst`). Because of that, `vmctl` doesn't actually know how much data will be
 processed and can't show the progress bar. It will show the current processing speed and total number of processed bytes:
 
 ```
@@ -468,20 +503,39 @@ Initing export pipe from "http://localhost:8528" with filters:
 Initing import process to "http://localhost:8428":
 Total: 336.75 KiB ↖ Speed: 454.46 KiB p/s                                                                                                               
 2020/10/13 17:04:59 Total time: 952.143376ms
-``` 
+```
 
 Importing tips:
-1. Migrating all the metrics from one VM to another may collide with existing application metrics 
-(prefixed with `vm_`) at destination and lead to confusion when using 
-[official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards). 
+
+1. Migrating big volumes of data may result in reaching the safety limits on `src` side.
+Please verify that `-search.maxExportDuration` and `-search.maxExportSeries` were set with
+proper values for `src`. If hitting the limits, follow the recommendations [here](https://docs.victoriametrics.com/#how-to-export-data-in-native-format).
+2. Migrating all the metrics from one VM to another may collide with existing application metrics
+(prefixed with `vm_`) at destination and lead to confusion when using
+[official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
 To avoid such situation try to filter out VM process metrics via `--vm-native-filter-match` flag.
-2. Migration is a backfilling process, so it is recommended to read 
+3. Migration is a backfilling process, so it is recommended to read
 [Backfilling tips](https://github.com/VictoriaMetrics/VictoriaMetrics#backfilling) section.
-3. `vmctl` doesn't provide relabeling or other types of labels management in this mode.
+4. `vmctl` doesn't provide relabeling or other types of labels management in this mode.
 Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/vmctl/issues/4#issuecomment-683424375).
-4. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
+5. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
 and specify `accountID` param.
 
+## Verifying exported blocks from VictoriaMetrics
+
+In this mode, `vmctl` allows verifying correctness and integrity of data exported via [native format](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-native-format) from VictoriaMetrics. 
+You can verify exported data at disk before uploading it by `vmctl verify-block` command:
+
+```bash
+# export blocks from VictoriaMetrics
+curl localhost:8428/api/v1/export/native -g -d 'match[]={__name__!=""}' -o exported_data_block
+# verify block content
+./vmctl verify-block exported_data_block
+2022/03/30 18:04:50 verifying block at path="exported_data_block"
+2022/03/30 18:04:50 successfully verified block at path="exported_data_block", blockCount=123786
+2022/03/30 18:04:50 Total time: 100.108ms
+```
+
 ## Tuning
 
 ### InfluxDB mode
@@ -491,7 +545,7 @@ timeseries. Please set it wisely to avoid InfluxDB overwhelming.
 
 The flag `--influx-chunk-size` controls the max amount of datapoints to return in single chunk from fetch requests.
 Please see more details [here](https://docs.influxdata.com/influxdb/v1.7/guides/querying_data/#chunking).
-The chunk size is used to control InfluxDB memory usage, so it won't OOM on processing large timeseries with 
+The chunk size is used to control InfluxDB memory usage, so it won't OOM on processing large timeseries with
 billions of datapoints.
 
 ### Prometheus mode
@@ -507,17 +561,18 @@ Please note that each import request can load up to a single vCPU core on Victor
 to allocated CPU resources of your VictoriMetrics installation.
 
 The flag `--vm-batch-size` controls max amount of samples collected before sending the import request.
-For example, if  `--influx-chunk-size=500` and `--vm-batch-size=2000` then importer will process not more 
-than 4 chunks before sending the request. 
+For example, if  `--influx-chunk-size=500` and `--vm-batch-size=2000` then importer will process not more
+than 4 chunks before sending the request.
 
 ### Importer stats
 
-After successful import `vmctl` prints some statistics for details. 
+After successful import `vmctl` prints some statistics for details.
 The important numbers to watch are following:
- - `idle duration` - shows time that importer spent while waiting for data from InfluxDB/Prometheus 
+
+- `idle duration` - shows time that importer spent while waiting for data from InfluxDB/Prometheus
 to fill up `--vm-batch-size` batch size. Value shows total duration across all workers configured
 via `--vm-concurrency`. High value may be a sign of too slow InfluxDB/Prometheus fetches or too
-high `--vm-concurrency` value. Try to improve it by increasing `--<mode>-concurrency` value or 
+high `--vm-concurrency` value. Try to improve it by increasing `--<mode>-concurrency` value or
 decreasing `--vm-concurrency` value.
 - `import requests` - shows how many import requests were issued to VM server.
 The import request is issued once the batch size(`--vm-batch-size`) is full and ready to be sent.
@@ -529,6 +584,7 @@ a sign of network issues or VM being overloaded. See the logs during import for
 
 By default `vmctl` waits confirmation from user before starting the import. If this is unwanted
 behavior and no user interaction required - pass `-s` flag to enable "silence" mode:
+
 ```
     -s Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
 ```
@@ -537,18 +593,18 @@ behavior and no user interaction required - pass `-s` flag to enable "silence" m
 
 `vmctl` allows to limit the number of [significant figures](https://en.wikipedia.org/wiki/Significant_figures)
 before importing. For example, the average value for response size is `102.342305` bytes and it has 9 significant figures.
-If you ask a human to pronounce this value then with high probability value will be rounded to first 4 or 5 figures 
-because the rest aren't really that important to mention. In most cases, such a high precision is too much. 
-Moreover, such values may be just a result of [floating point arithmetic](https://en.wikipedia.org/wiki/Floating-point_arithmetic), 
-create a [false precision](https://en.wikipedia.org/wiki/False_precision) and result into bad compression ratio 
-according to [information theory](https://en.wikipedia.org/wiki/Information_theory). 
+If you ask a human to pronounce this value then with high probability value will be rounded to first 4 or 5 figures
+because the rest aren't really that important to mention. In most cases, such a high precision is too much.
+Moreover, such values may be just a result of [floating point arithmetic](https://en.wikipedia.org/wiki/Floating-point_arithmetic),
+create a [false precision](https://en.wikipedia.org/wiki/False_precision) and result into bad compression ratio
+according to [information theory](https://en.wikipedia.org/wiki/Information_theory).
 
 `vmctl` provides the following flags for improving data compression:
 
-* `--vm-round-digits` flag for rounding processed values to the given number of decimal digits after the point.
+- `--vm-round-digits` flag for rounding processed values to the given number of decimal digits after the point.
   For example, `--vm-round-digits=2` would round `1.2345` to `1.23`. By default the rounding is disabled.
 
-* `--vm-significant-figures` flag for limiting the number of significant figures in processed values. It takes no effect if set
+- `--vm-significant-figures` flag for limiting the number of significant figures in processed values. It takes no effect if set
   to 0 (by default), but set `--vm-significant-figures=5` and `102.342305` will be rounded to `102.34`.
 
 The most common case for using these flags is to improve data compression for time series storing aggregation
@@ -556,7 +612,7 @@ results such as `average`, `rate`, etc.
 
 ### Adding extra labels
 
- `vmctl` allows to add extra labels to all imported series. It can be achived with flag `--vm-extra-label label=value`. 
+ `vmctl` allows to add extra labels to all imported series. It can be achived with flag `--vm-extra-label label=value`.
  If multiple labels needs to be added, set flag for each label, for example, `--vm-extra-label label1=value1 --vm-extra-label label2=value2`.
  If timeseries already have label, that must be added with `--vm-extra-label` flag, flag has priority and will override label value from timeseries.
 
@@ -565,15 +621,13 @@ results such as `average`, `rate`, etc.
 Limiting the rate of data transfer could help to reduce pressure on disk or on destination database.
 The rate limit may be set in bytes-per-second via `--vm-rate-limit` flag.
 
-Please note, you can also use [vmagent](https://docs.victoriametrics.com/vmagent.html) 
+Please note, you can also use [vmagent](https://docs.victoriametrics.com/vmagent.html)
 as a proxy between `vmctl` and destination with `-remoteWrite.rateLimit` flag enabled.
 
-
 ## How to build
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmctl` is located in `vmutils-*` archives there.
 
-
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.

app/vmctl/barpool/pool.go

@@ -0,0 +1,26 @@
+// Package barpool provides access to the global
+// pool of progress bars, so they could be rendered
+// altogether.
+package barpool
+
+import "github.com/dmitryk-dk/pb/v3"
+
+var pool = pb.NewPool()
+
+// Add adds bar to the global pool
+func Add(bar *pb.ProgressBar) { pool.Add(bar) }
+
+// Start starts the global pool
+// Must be called after all progress bars were added
+func Start() error { return pool.Start() }
+
+// Stop stops the global pool
+func Stop() { _ = pool.Stop() }
+
+// AddWithTemplate adds bar with the given template
+// to the global pool
+func AddWithTemplate(format string, total int) *pb.ProgressBar {
+	bar := pb.ProgressBarTemplate(format).New(total)
+	Add(bar)
+	return bar
+}

app/vmctl/flags.go

@@ -36,6 +36,7 @@ const (
 	vmBatchSize          = "vm-batch-size"
 	vmSignificantFigures = "vm-significant-figures"
 	vmRoundDigits        = "vm-round-digits"
+	vmDisableProgressBar = "vm-disable-progress-bar"
 
 	// also used in vm-native
 	vmExtraLabel = "vm-extra-label"
@@ -109,6 +110,10 @@ var (
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
 				"By default the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
 		},
+		&cli.BoolFlag{
+			Name:  vmDisableProgressBar,
+			Usage: "Whether to disable progress bar per each worker during the import.",
+		},
 	}
 )
 
@@ -196,6 +201,8 @@ const (
 	influxFilterTimeStart           = "influx-filter-time-start"
 	influxFilterTimeEnd             = "influx-filter-time-end"
 	influxMeasurementFieldSeparator = "influx-measurement-field-separator"
+	influxSkipDatabaseLabel         = "influx-skip-database-label"
+	influxPrometheusMode            = "influx-prometheus-mode"
 )
 
 var (
@@ -253,6 +260,16 @@ var (
 			Usage: "The {separator} symbol used to concatenate {measurement} and {field} names into series name {measurement}{separator}{field}.",
 			Value: "_",
 		},
+		&cli.BoolFlag{
+			Name:  influxSkipDatabaseLabel,
+			Usage: "Wether to skip adding the label 'db' to timeseries.",
+			Value: false,
+		},
+		&cli.BoolFlag{
+			Name:  influxPrometheusMode,
+			Usage: "Wether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write.",
+			Value: false,
+		},
 	}
 )
 

app/vmctl/influx.go

@@ -6,27 +6,31 @@ import (
 	"log"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/cheggaaa/pb/v3"
 )
 
 type influxProcessor struct {
-	ic        *influx.Client
-	im        *vm.Importer
-	cc        int
-	separator string
+	ic          *influx.Client
+	im          *vm.Importer
+	cc          int
+	separator   string
+	skipDbLabel bool
+	promMode    bool
 }
 
-func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator string) *influxProcessor {
+func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator string, skipDbLabel bool, promMode bool) *influxProcessor {
 	if cc < 1 {
 		cc = 1
 	}
 	return &influxProcessor{
-		ic:        ic,
-		im:        im,
-		cc:        cc,
-		separator: separator,
+		ic:          ic,
+		im:          im,
+		cc:          cc,
+		separator:   separator,
+		skipDbLabel: skipDbLabel,
+		promMode:    promMode,
 	}
 }
 
@@ -44,7 +48,11 @@ func (ip *influxProcessor) run(silent, verbose bool) error {
 		return nil
 	}
 
-	bar := pb.StartNew(len(series))
+	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
+	if err := barpool.Start(); err != nil {
+		return err
+	}
+
 	seriesCh := make(chan *influx.Series)
 	errCh := make(chan error)
 	ip.im.ResetStats()
@@ -78,19 +86,25 @@ func (ip *influxProcessor) run(silent, verbose bool) error {
 	close(seriesCh)
 	wg.Wait()
 	ip.im.Close()
+	close(errCh)
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
 		if vmErr.Err != nil {
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 		}
 	}
-	bar.Finish()
+	for err := range errCh {
+		return fmt.Errorf("import process failed: %s", err)
+	}
+	barpool.Stop()
 	log.Println("Import finished!")
 	log.Print(ip.im.Stats())
 	return nil
 }
 
 const dbLabel = "db"
+const nameLabel = "__name__"
+const valueField = "value"
 
 func (ip *influxProcessor) do(s *influx.Series) error {
 	cr, err := ip.ic.FetchDataPoints(s)
@@ -112,14 +126,15 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 	for i, lp := range s.LabelPairs {
 		if lp.Name == dbLabel {
 			containsDBLabel = true
-			break
+		} else if lp.Name == nameLabel && s.Field == valueField && ip.promMode {
+			name = lp.Value
 		}
 		labels[i] = vm.LabelPair{
 			Name:  lp.Name,
 			Value: lp.Value,
 		}
 	}
-	if !containsDBLabel {
+	if !containsDBLabel && !ip.skipDbLabel {
 		labels = append(labels, vm.LabelPair{
 			Name:  dbLabel,
 			Value: ip.ic.Database(),
@@ -138,11 +153,14 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 		if len(time) < 1 {
 			continue
 		}
-		ip.im.Input() <- &vm.TimeSeries{
+		ts := vm.TimeSeries{
 			Name:       name,
 			LabelPairs: labels,
 			Timestamps: time,
 			Values:     values,
 		}
+		if err := ip.im.Input(&ts); err != nil {
+			return err
+		}
 	}
 }

app/vmctl/main.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
 	"os/signal"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -14,6 +16,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/urfave/cli/v2"
 )
 
@@ -23,6 +27,7 @@ func main() {
 		importer *vm.Importer
 	)
 
+	ctx, cancelCtx := context.WithCancel(context.Background())
 	start := time.Now()
 	app := &cli.App{
 		Name:    "vmctl",
@@ -52,6 +57,9 @@ func main() {
 					}
 
 					vmCfg := initConfigVM(c)
+					// disable progress bars since openTSDB implementation
+					// does not use progress bar pool
+					vmCfg.DisableProgressBar = true
 					importer, err := vm.NewImporter(vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -92,8 +100,13 @@ func main() {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
-					processor := newInfluxProcessor(influxClient, importer,
-						c.Int(influxConcurrency), c.String(influxMeasurementFieldSeparator))
+					processor := newInfluxProcessor(
+						influxClient,
+						importer,
+						c.Int(influxConcurrency),
+						c.String(influxMeasurementFieldSeparator),
+						c.Bool(influxSkipDatabaseLabel),
+						c.Bool(influxPrometheusMode))
 					return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
@@ -161,7 +174,40 @@ func main() {
 							extraLabels: c.StringSlice(vmExtraLabel),
 						},
 					}
-					return p.run()
+					return p.run(ctx)
+				},
+			},
+			{
+				Name:  "verify-block",
+				Usage: "Verifies exported block with VictoriaMetrics Native format",
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "gunzip",
+						Usage: "Use GNU zip decompression for exported block",
+						Value: false,
+					},
+				},
+				Action: func(c *cli.Context) error {
+					common.StartUnmarshalWorkers()
+					blockPath := c.Args().First()
+					isBlockGzipped := c.Bool("gunzip")
+					if len(blockPath) == 0 {
+						return cli.Exit("you must provide path for exported data block", 1)
+					}
+					log.Printf("verifying block at path=%q", blockPath)
+					f, err := os.OpenFile(blockPath, os.O_RDONLY, 0600)
+					if err != nil {
+						return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1)
+					}
+					var blocksCount uint64
+					if err := parser.ParseStream(f, isBlockGzipped, func(block *parser.Block) error {
+						atomic.AddUint64(&blocksCount, 1)
+						return nil
+					}); err != nil {
+						return cli.Exit(fmt.Errorf("cannot parse block at path=%q, blocksCount=%d, err=%w", blockPath, blocksCount, err), 1)
+					}
+					log.Printf("successfully verified block at path=%q, blockCount=%d", blockPath, blocksCount)
+					return nil
 				},
 			},
 		},
@@ -175,11 +221,12 @@ func main() {
 		if importer != nil {
 			importer.Close()
 		}
+		cancelCtx()
 	}()
 
 	err = app.Run(os.Args)
 	if err != nil {
-		log.Println(err)
+		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
 }
@@ -197,5 +244,6 @@ func initConfigVM(c *cli.Context) vm.Config {
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
+		DisableProgressBar: c.Bool(vmDisableProgressBar),
 	}
 }

app/vmctl/opentsdb.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/cheggaaa/pb/v3"
+	"github.com/dmitryk-dk/pb/v3"
 )
 
 type otsdbProcessor struct {
@@ -67,7 +67,7 @@ func (op *otsdbProcessor) run(silent, verbose bool) error {
 		queryRanges += len(rt.QueryRanges)
 	}
 	for _, metric := range metrics {
-		log.Println(fmt.Sprintf("Starting work on %s", metric))
+		log.Printf("Starting work on %s", metric)
 		serieslist, err := op.oc.FindSeries(metric)
 		if err != nil {
 			return fmt.Errorf("couldn't retrieve series list for %s : %s", metric, err)
@@ -120,6 +120,7 @@ func (op *otsdbProcessor) run(silent, verbose bool) error {
 				}
 			}
 		}
+
 		// Drain channels per metric
 		close(seriesCh)
 		wg.Wait()
@@ -156,11 +157,14 @@ func (op *otsdbProcessor) do(s queryObj) error {
 	for k, v := range data.Tags {
 		labels = append(labels, vm.LabelPair{Name: k, Value: v})
 	}
-	op.im.Input() <- &vm.TimeSeries{
+	ts := vm.TimeSeries{
 		Name:       data.Metric,
 		LabelPairs: labels,
 		Timestamps: data.Timestamps,
 		Values:     data.Values,
 	}
+	if err := op.im.Input(&ts); err != nil {
+		return err
+	}
 	return nil
 }