docs/CHANGELOG.md: cut v1.80.0

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2941
Signed-off-by: hagen1778 <roman@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,10 +8,10 @@ assignees: ''
 
 **Describe the bug**
 A clear and concise description of what the bug is.
-It would be a great [upgrading](https://docs.victoriametrics.com/#how-to-upgrade) 
+It would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
 to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-and verifying whether the bug is reproducible there.
-It is also recommended reading [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
+and verify whether the bug is reproducible there.
+It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
 
 **To Reproduce**
 Steps to reproduce the behavior.
@@ -33,15 +33,14 @@ of Grafana dashboards if possible:
 
 See how to setup monitoring here:
 * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-* [montioring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+* [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
 
 **Version**
-The line returned when passing `--version` command line flag to binary. For example:
+The line returned when passing `--version` command line flag to the binary. For example:
 ```
 $ ./victoria-metrics-prod --version
 victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
 ```
 
 **Used command-line flags**
-Please provide applied command-line flags used for running VictoriaMetrics and its components. 
-
+Please provide the command-line flags used for running VictoriaMetrics and its components. 

.github/dependabot.yml

@@ -16,6 +16,7 @@ updates:
     directory: "/app/vmui/packages/vmui/web"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
@@ -24,3 +25,4 @@ updates:
     directory: "/app/vmui/packages/vmui"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 0

.github/workflows/check-licenses.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     paths:
       - 'vendor'
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build
@@ -14,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master, cluster ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master, cluster ]
+  schedule:
+    - cron: '30 18 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -8,6 +8,9 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**.md'
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build
@@ -16,7 +19,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       - name: Code checkout
         uses: actions/checkout@master
@@ -35,32 +38,10 @@ jobs:
           make test-full
           make test-pure
           make test-full-386
-          make victoria-metrics
-          make victoria-metrics-pure
-          make victoria-metrics-arm
-          make victoria-metrics-arm64
-          make vmutils
-          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=freebsd go build -mod=vendor ./app/vmagent
-          GOOS=freebsd go build -mod=vendor ./app/vmalert
-          GOOS=freebsd go build -mod=vendor ./app/vmbackup
-          GOOS=freebsd go build -mod=vendor ./app/vmrestore
-          GOOS=freebsd go build -mod=vendor ./app/vmctl
-          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=openbsd go build -mod=vendor ./app/vmagent
-          GOOS=openbsd go build -mod=vendor ./app/vmalert
-          GOOS=openbsd go build -mod=vendor ./app/vmbackup
-          GOOS=openbsd go build -mod=vendor ./app/vmrestore
-          GOOS=openbsd go build -mod=vendor ./app/vmctl
-          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
-          GOOS=darwin go build -mod=vendor ./app/vmagent
-          GOOS=darwin go build -mod=vendor ./app/vmalert
-          GOOS=darwin go build -mod=vendor ./app/vmbackup
-          GOOS=darwin go build -mod=vendor ./app/vmrestore
-          GOOS=darwin go build -mod=vendor ./app/vmctl
-          CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
+          make victoria-metrics-crossbuild
+          make vmuitils-crossbuild
       - name: Publish coverage
-        uses: codecov/codecov-action@v2.1.0
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
 

.github/workflows/wiki.yml

@@ -5,8 +5,13 @@ on:
       - 'docs/*'
     branches:
       - master
+permissions:
+  contents: read
+
 jobs:
   build:
+    permissions:
+      contents: write  # for Git to git push
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master

CODE_OF_CONDUCT.md

@@ -68,9 +68,9 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+available at <https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>
 
 [homepage]: https://www.contributor-covenant.org
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+<https://www.contributor-covenant.org/faq>

CODE_OF_CONDUCT_RU.md

@@ -79,7 +79,7 @@
 
 **Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
 На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие 
+включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
 в публичных пространствах, а так же во внешних каналах,
 таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
@@ -89,10 +89,10 @@
 **Общественное влияние**: Серьёзное нарушение стандартов сообщества,
 включая продолжительное неуместное поведение.
 
-**Последствия**: Временный запрет (бан) на любое взаимодействие 
+**Последствия**: Временный запрет (бан) на любое взаимодействие
 или публичное общение с сообществом на определенный период времени.
 На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие 
+вовлеченными в инцидент, включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса.
 Нарушение этих правил влечет за собой вечный бан.
 
@@ -108,7 +108,7 @@
 
 Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
 версии 2.0, доступной по адресу
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
 
 Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
 enforcement ladder](https://github.com/mozilla/diversity).
@@ -116,5 +116,5 @@ enforcement ladder](https://github.com/mozilla/diversity).
 [homepage]: https://www.contributor-covenant.org
 
 Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-https://www.contributor-covenant.org/faq. Переводы доступны по адресу
-https://www.contributor-covenant.org/translations.
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

Makefile

@@ -13,6 +13,10 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 
 .PHONY: $(MAKECMDGOALS)
 
+include app/*/Makefile
+include deployment/*/Makefile
+include snap/local/Makefile
+
 all: \
 	victoria-metrics-prod \
 	vmagent-prod \
@@ -22,15 +26,10 @@ all: \
 	vmrestore-prod \
 	vmctl-prod
 
-include app/*/Makefile
-include deployment/*/Makefile
-include snap/local/Makefile
-
-
 clean:
 	rm -rf bin/*
 
-publish: \
+publish: docker-scan \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
@@ -64,21 +63,77 @@ vmutils-pure: \
 	vmrestore-pure \
 	vmctl-pure
 
-vmutils-arm64: \
-	vmagent-arm64 \
-	vmalert-arm64 \
-	vmauth-arm64 \
-	vmbackup-arm64 \
-	vmrestore-arm64 \
-	vmctl-arm64
+vmutils-linux-amd64: \
+	vmagent-linux-amd64 \
+	vmalert-linux-amd64 \
+	vmauth-linux-amd64 \
+	vmbackup-linux-amd64 \
+	vmrestore-linux-amd64 \
+	vmctl-linux-amd64
 
-vmutils-arm: \
-	vmagent-arm \
-	vmalert-arm \
-	vmauth-arm \
-	vmbackup-arm \
-	vmrestore-arm \
-	vmctl-arm
+vmutils-linux-arm64: \
+	vmagent-linux-arm64 \
+	vmalert-linux-arm64 \
+	vmauth-linux-arm64 \
+	vmbackup-linux-arm64 \
+	vmrestore-linux-arm64 \
+	vmctl-linux-arm64
+
+vmutils-linux-arm: \
+	vmagent-linux-arm \
+	vmalert-linux-arm \
+	vmauth-linux-arm \
+	vmbackup-linux-arm \
+	vmrestore-linux-arm \
+	vmctl-linux-arm
+
+vmutils-linux-386: \
+	vmagent-linux-386 \
+	vmalert-linux-386 \
+	vmauth-linux-386 \
+	vmbackup-linux-386 \
+	vmrestore-linux-386 \
+	vmctl-linux-386
+
+vmutils-linux-ppc64le: \
+	vmagent-linux-ppc64le \
+	vmalert-linux-ppc64le \
+	vmauth-linux-ppc64le \
+	vmbackup-linux-ppc64le \
+	vmrestore-linux-ppc64le \
+	vmctl-linux-ppc64le
+
+vmutils-darwin-amd64: \
+	vmagent-darwin-amd64 \
+	vmalert-darwin-amd64 \
+	vmauth-darwin-amd64 \
+	vmbackup-darwin-amd64 \
+	vmrestore-darwin-amd64 \
+	vmctl-darwin-amd64
+
+vmutils-darwin-arm64: \
+	vmagent-darwin-arm64 \
+	vmalert-darwin-arm64 \
+	vmauth-darwin-arm64 \
+	vmbackup-darwin-arm64 \
+	vmrestore-darwin-arm64 \
+	vmctl-darwin-arm64
+
+vmutils-freebsd-amd64: \
+	vmagent-freebsd-amd64 \
+	vmalert-freebsd-amd64 \
+	vmauth-freebsd-amd64 \
+	vmbackup-freebsd-amd64 \
+	vmrestore-freebsd-amd64 \
+	vmctl-freebsd-amd64
+
+vmutils-openbsd-amd64: \
+	vmagent-openbsd-amd64 \
+	vmalert-openbsd-amd64 \
+	vmauth-openbsd-amd64 \
+	vmbackup-openbsd-amd64 \
+	vmrestore-openbsd-amd64 \
+	vmctl-openbsd-amd64
 
 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
@@ -86,6 +141,28 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
+victoria-metrics-crossbuild: \
+	victoria-metrics-linux-amd64 \
+	victoria-metrics-linux-arm64 \
+	victoria-metrics-linux-arm \
+	victoria-metrics-linux-386 \
+	victoria-metrics-linux-ppc64le \
+	victoria-metrics-darwin-amd64 \
+	victoria-metrics-darwin-arm64 \
+	victoria-metrics-freebsd-amd64 \
+	victoria-metrics-openbsd-amd64
+
+vmutils-crossbuild: \
+	vmutils-linux-amd64 \
+	vmutils-linux-arm64 \
+	vmutils-linux-arm \
+	vmutils-linux-386 \
+	vmutils-linux-ppc64le \
+	vmutils-darwin-amd64 \
+	vmutils-darwin-arm64 \
+	vmutils-freebsd-amd64 \
+	vmutils-openbsd-amd64 \
+	vmutils-windows-amd64
 
 publish-release:
 	git checkout $(TAG) && $(MAKE) release publish && \
@@ -98,86 +175,110 @@ release: \
 	release-vmutils
 
 release-victoria-metrics: \
-	release-victoria-metrics-amd64 \
-	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64 \
+	release-victoria-metrics-linux-amd64 \
+	release-victoria-metrics-linux-arm \
+	release-victoria-metrics-linux-arm64 \
 	release-victoria-metrics-darwin-amd64 \
-	release-victoria-metrics-darwin-arm64
+	release-victoria-metrics-darwin-arm64 \
+	release-victoria-metrics-freebsd-amd64 \
+	release-victoria-metrics-openbsd-amd64
 
-release-victoria-metrics-amd64:
-	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-arm:
-	OSARCH=arm $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-arm64:
-	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
+release-victoria-metrics-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
 release-vmutils: \
-	release-vmutils-amd64 \
-	release-vmutils-arm64 \
-	release-vmutils-arm \
-	release-vmutils-darwin-amd64  \
+	release-vmutils-linux-amd64 \
+	release-vmutils-linux-arm64 \
+	release-vmutils-linux-arm \
+	release-vmutils-darwin-amd64 \
 	release-vmutils-darwin-arm64 \
+	release-vmutils-freebsd-amd64 \
+	release-vmutils-openbsd-amd64 \
 	release-vmutils-windows-amd64
 
-release-vmutils-amd64:
-	OSARCH=amd64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
-release-vmutils-arm64:
-	OSARCH=arm64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch
 
-release-vmutils-arm:
-	OSARCH=arm $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic
+	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
 
-release-vmutils-generic: \
-	vmagent-$(OSARCH)-prod \
-	vmalert-$(OSARCH)-prod \
-	vmauth-$(OSARCH)-prod \
-	vmbackup-$(OSARCH)-prod \
-	vmrestore-$(OSARCH)-prod \
-	vmctl-$(OSARCH)-prod
+release-vmutils-goos-goarch: \
+	vmagent-$(GOOS)-$(GOARCH)-prod \
+	vmalert-$(GOOS)-$(GOARCH)-prod \
+	vmauth-$(GOOS)-$(GOARCH)-prod \
+	vmbackup-$(GOOS)-$(GOARCH)-prod \
+	vmrestore-$(GOOS)-$(GOARCH)-prod \
+	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-$(GOOS)-$(GOARCH)-prod \
+		vmalert-$(GOOS)-$(GOARCH)-prod \
+		vmauth-$(GOOS)-$(GOARCH)-prod \
+		vmbackup-$(GOOS)-$(GOARCH)-prod \
+		vmrestore-$(GOOS)-$(GOARCH)-prod \
+		vmctl-$(GOOS)-$(GOARCH)-prod
 
-release-vmutils-windows-generic: \
+release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
@@ -194,18 +295,23 @@ release-vmutils-windows-generic: \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-windows-$(GOARCH)-prod.exe \
+		vmalert-windows-$(GOARCH)-prod.exe \
+		vmauth-windows-$(GOARCH)-prod.exe \
+		vmctl-windows-$(GOARCH)-prod.exe
 
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	GO111MODULE=on gofmt -l -w -s ./lib
-	GO111MODULE=on gofmt -l -w -s ./app
+	gofmt -l -w -s ./lib
+	gofmt -l -w -s ./app
 
 vet:
-	GO111MODULE=on go vet -mod=vendor ./lib/...
-	GO111MODULE=on go vet -mod=vendor ./app/...
+	go vet ./lib/...
+	go vet ./app/...
 
 lint: install-golint
 	golint lib/...
@@ -232,45 +338,45 @@ install-errcheck:
 check-all: fmt vet lint errcheck golangci-lint
 
 test:
-	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+	go test -bench=. ./lib/...
+	go test -bench=. ./app/...
 
 benchmark-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	GO111MODULE=on go get -u -d ./lib/...
-	GO111MODULE=on go get -u -d ./app/...
-	GO111MODULE=on go mod tidy
-	GO111MODULE=on go mod vendor
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.17
+	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
-app-local-with-goarch:
-	GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-goos-goarch:
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
-app-local-windows-with-goarch:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-windows-goarch:
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
@@ -283,7 +389,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.44.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

app/victoria-metrics/Makefile

@@ -12,20 +12,20 @@ victoria-metrics-prod:
 victoria-metrics-pure-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure
 
-victoria-metrics-amd64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-amd64
+victoria-metrics-linux-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-amd64
 
-victoria-metrics-arm-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+victoria-metrics-linux-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm
 
-victoria-metrics-arm64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+victoria-metrics-linux-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm64
 
-victoria-metrics-ppc64le-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+victoria-metrics-linux-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-ppc64le
 
-victoria-metrics-386-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+victoria-metrics-linux-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-386
 
 victoria-metrics-darwin-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ victoria-metrics-darwin-amd64-prod:
 victoria-metrics-darwin-arm64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64
 
+victoria-metrics-freebsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-freebsd-amd64
+
+victoria-metrics-openbsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 
@@ -64,60 +70,68 @@ run-victoria-metrics:
 	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
 	$(MAKE) run-via-docker
 
-victoria-metrics-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-amd64 ./app/victoria-metrics
+victoria-metrics-linux-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-victoria-metrics-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+victoria-metrics-linux-arm:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-victoria-metrics-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
+victoria-metrics-linux-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-victoria-metrics-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
+victoria-metrics-linux-ppc64le:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-victoria-metrics-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+victoria-metrics-linux-386:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-freebsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-openbsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 
 ### Packaging as DEB - amd64
-victoria-metrics-package-deb: victoria-metrics-prod
+victoria-metrics-package-deb-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_deb.sh amd64
 
 ### Packaging as DEB - arm64
-victoria-metrics-package-deb-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-deb-arm: victoria-metrics-linux-arm-prod
+	./package/package_deb.sh arm
+
+### Packaging as DEB - arm64
+victoria-metrics-package-deb-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_deb.sh arm64
 
 ### Packaging as DEB - all
-victoria-metrics-package-deb-all: \
-        victoria-metrics-package-deb \
+victoria-metrics-package-deb: \
+        victoria-metrics-package-deb-amd64 \
+        victoria-metrics-package-deb-arm \
         victoria-metrics-package-deb-arm64
 
 ### Packaging as RPM - amd64
-victoria-metrics-package-rpm: victoria-metrics-prod
+victoria-metrics-package-rpm-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_rpm.sh amd64
 
 ### Packaging as RPM - arm64
-victoria-metrics-package-rpm-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-rpm-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_rpm.sh arm64
 
 ### Packaging as RPM - all
-victoria-metrics-package-rpm-all: \
-        victoria-metrics-package-rpm \
+victoria-metrics-package-rpm: \
+        victoria-metrics-package-rpm-amd64 \
         victoria-metrics-package-rpm-arm64
 
 ### Packaging as both DEB and RPM - all
-victoria-metrics-package-deb-rpm-all: \
+victoria-metrics-package-deb-rpm: \
         victoria-metrics-package-deb \
-        victoria-metrics-package-deb-arm64 \
-        victoria-metrics-package-rpm \
-        victoria-metrics-package-rpm-arm64
-
-### Packaging as snap
-victoria-metrics-package-snap:
-	which snapcraft || snap install snapcraft
-	which multipass || snap install multipass
-	snapcraft
-
+        victoria-metrics-package-rpm

app/victoria-metrics/main.go

@@ -19,12 +19,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
-	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -37,6 +38,7 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -86,12 +88,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"vmui", "Web UI"},
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},

app/victoria-metrics/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
 ARG TARGETARCH
-COPY victoria-metrics-${TARGETARCH}-prod ./victoria-metrics-prod
+COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod

app/victoria-metrics/self_scraper.go

@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -60,7 +60,7 @@ func selfScraper(scrapeInterval time.Duration) {
 			currentTimestamp = currentTime.UnixNano() / 1e6
 		}
 		bb.Reset()
-		httpserver.WritePrometheusMetrics(&bb)
+		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
 		rows.Unmarshal(s)

app/vmagent/Makefile

@@ -12,20 +12,20 @@ vmagent-prod:
 vmagent-pure-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-pure
 
-vmagent-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+vmagent-linux-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-amd64
 
-vmagent-arm-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+vmagent-linux-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm
 
-vmagent-arm64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+vmagent-linux-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm64
 
-vmagent-ppc64le-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+vmagent-linux-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-ppc64le
 
-vmagent-386-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-386
+vmagent-linux-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
 
 vmagent-darwin-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmagent-darwin-amd64-prod:
 vmagent-darwin-arm64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
 
+vmagent-freebsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-freebsd-amd64
+
+vmagent-openbsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-openbsd-amd64
+
 vmagent-windows-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
 
@@ -67,26 +73,35 @@ run-vmagent:
 	APP_NAME=vmagent \
 	$(MAKE) run-via-docker
 
-vmagent-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-amd64:
+	APP_NAME=vmagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-vmagent-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-vmagent-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-vmagent-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmagent-local-with-goarch
+vmagent-linux-ppc64le:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmagent-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-386:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
-vmagent-local-with-goarch:
-	APP_NAME=vmagent $(MAKE) app-local-with-goarch
+vmagent-darwin-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-darwin-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmagent-freebsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-openbsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-goarch
 
 vmagent-pure:
 	APP_NAME=vmagent $(MAKE) app-local-pure
-
-vmagent-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch

app/vmagent/csvimport/request_handler.go

@@ -67,7 +67,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -1,9 +1,7 @@
 package datadog
 
 import (
-	"fmt"
 	"net/http"
-	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
@@ -59,12 +57,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 			Value: ss.Host,
 		})
 		for _, tag := range ss.Tags {
-			n := strings.IndexByte(tag, ':')
-			if n < 0 {
-				return fmt.Errorf("cannot find ':' in tag %q", tag)
-			}
-			name := tag[:n]
-			value := tag[n+1:]
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -89,7 +82,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/graphite/request_handler.go

@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -134,7 +134,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"flag"
 	"fmt"
 	"io"
@@ -23,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -35,6 +37,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -43,7 +46,7 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
@@ -63,6 +66,12 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -71,6 +80,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -154,11 +164,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -169,6 +181,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -253,15 +270,24 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/intake/":
+	case "/datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
 	case "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
@@ -283,6 +309,22 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
+	case "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -304,11 +346,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			w.Write([]byte("OK"))
 		}
 		return true
+	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+		if remotewrite.MultitenancyEnabled() {
+			return processMultitenantRequest(w, r, path)
+		}
+		return false
 	}
-	if remotewrite.MultitenancyEnabled() {
-		return processMultitenantRequest(w, r, path)
-	}
-	return false
 }
 
 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
@@ -326,6 +373,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
+	if strings.HasPrefix(p.Suffix, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
+	}
 	switch p.Suffix {
 	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -409,11 +461,16 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/intake/":
+	case "datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -446,15 +503,18 @@ var (
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
 
-	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )

app/vmagent/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-COPY vmagent-${TARGETARCH}-prod ./vmagent-prod
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -30,8 +30,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(block *parser.Block) error {
+		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
 			return insertRows(at, block, extraLabels)
 		})
 	})
@@ -86,6 +87,6 @@ func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/opentsdb/request_handler.go

@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -65,7 +65,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -82,7 +82,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/promremotewrite/request_handler.go

@@ -81,7 +81,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -19,46 +20,50 @@ import (
 )
 
 var (
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to -remoteWrite.url. "+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to -remoteWrite.url")
-	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. "+
-		"Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsCAFile = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
-		"By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to -remoteWrite.url. "+
-		"By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	tlsCAFile  = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
 
-	basicAuthUsername = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for -remoteWrite.url. "+
-		"The file is re-read every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for -remoteWrite.url. "+
-		"The token is re-read from the file every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	headers = flagutil.NewArray("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
-	oauth2ClientID = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecret = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2TokenURL = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthUsername     = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthPassword     = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
+	bearerToken     = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+		"The token is re-read from the file every second")
+
+	oauth2ClientID         = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecret     = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	oauth2TokenURL         = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+
+	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
+	awsEC2Endpoint = flagutil.NewArray("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSTSEndpoint = flagutil.NewArray("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRegion      = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN     = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey   = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService     = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"Defaults to \"aps\"")
+	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
 )
 
 type client struct {
@@ -69,6 +74,7 @@ type client struct {
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
+	awsCfg    *awsapi.Config
 
 	rl rateLimiter
 
@@ -78,6 +84,7 @@ type client struct {
 	requestsOKCount *metrics.Counter
 	errorsCount     *metrics.Counter
 	packetsDropped  *metrics.Counter
+	rateLimit       *metrics.Gauge
 	retriesCount    *metrics.Counter
 	sendDuration    *metrics.FloatCounter
 
@@ -88,13 +95,17 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tlsCfg := authCfg.NewTLSConfig()
+	awsCfg, err := getAWSAPIConfig(argIdx)
+	if err != nil {
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
 	tr := &http.Transport{
-		Dial:                statDial,
+		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 5 * time.Second,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -115,6 +126,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
+		awsCfg:         awsCfg,
 		fq:             fq,
 		hc: &http.Client{
 			Transport: tr,
@@ -131,16 +143,22 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
-	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remote_write_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
+	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
+	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
 	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
+	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(*queues)
+	})
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -158,6 +176,11 @@ func (c *client) MustStop() {
 }
 
 func getAuthConfig(argIdx int) (*promauth.Config, error) {
+	headersValue := headers.GetOptionalArg(argIdx)
+	var hdrs []string
+	if headersValue != "" {
+		hdrs = strings.Split(headersValue, "^^")
+	}
 	username := basicAuthUsername.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
 	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
@@ -194,13 +217,39 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
 
-	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		OAuth2:          oauth2Cfg,
+		TLSConfig:       tlsCfg,
+		Headers:         hdrs,
+	}
+	authCfg, err := opts.NewConfig()
 	if err != nil {
 		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
 	return authCfg, nil
 }
 
+func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
+	if !awsUseSigv4.GetOptionalArg(argIdx) {
+		return nil, nil
+	}
+	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
+	region := awsRegion.GetOptionalArg(argIdx)
+	roleARN := awsRoleARN.GetOptionalArg(argIdx)
+	accessKey := awsAccessKey.GetOptionalArg(argIdx)
+	secretKey := awsSecretKey.GetOptionalArg(argIdx)
+	service := awsService.GetOptionalArg(argIdx)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
 func (c *client) runWorker() {
 	var ok bool
 	var block []byte
@@ -250,21 +299,28 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 	retriesCount := 0
 	c.bytesSent.Add(len(block))
 	c.blocksSent.Inc()
+	sigv4Hash := ""
+	if c.awsCfg != nil {
+		sigv4Hash = awsapi.HashHex(block)
+	}
 
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
+	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
 	h.Set("Content-Encoding", "snappy")
 	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	if ah := c.authCfg.GetAuthHeader(); ah != "" {
-		req.Header.Set("Authorization", ah)
+	if c.awsCfg != nil {
+		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
+		}
 	}
-
 	startTime := time.Now()
 	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)
@@ -297,13 +353,12 @@ again:
 	if statusCode == 409 || statusCode == 400 {
 		body, err := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 		if err != nil {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
 				"failed to read response body: %s",
 				len(block), c.sanitizedURL, statusCode, err)
 		} else {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
 				len(block), c.sanitizedURL, statusCode, string(body))
 		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
@@ -340,6 +395,8 @@ again:
 	goto again
 }
 
+var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
 type rateLimiter struct {
 	perSecondLimit int64
 

app/vmagent/remotewrite/pendingseries.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -210,7 +211,22 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		writeRequestBufPool.Put(bb)
 	}
 
-	// Too big block. Recursively split it into smaller parts.
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
+		samples := wr.Timeseries[0].Samples
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return
+		}
+		n := len(samples) / 2
+		wr.Timeseries[0].Samples = samples[:n]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples[n:]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples
+		return
+	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]

app/vmagent/remotewrite/remotewrite.go

@@ -22,7 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
-	xxhash "github.com/cespare/xxhash/v2"
+	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -40,7 +40,7 @@ var (
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
-		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
 	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+
 		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
@@ -234,15 +234,11 @@ func Stop() {
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// Note that wr may be modified by Push due to relabeling and rounding.
-func Push(wr *prompbmarshal.WriteRequest) {
-	PushWithAuthToken(nil, wr)
-}
-
-// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
+// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
+// If at isn't nil, the the data is pushed to the configured `-remoteWrite.multitenantURL`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
-func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
 		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
 		at = defaultAuthToken
@@ -252,7 +248,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rwctxs = rwctxsDefault
 	} else {
 		if len(*remoteWriteMultitenantURLs) == 0 {
-			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
 		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
@@ -274,6 +270,8 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
+	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
@@ -298,9 +296,10 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			tss = nil
 		}
 		if rctx != nil {
-			tssBlockLen := len(tssBlock)
+			rowsCountBeforeRelabel := getRowsCount(tssBlock)
 			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
-			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+			rowsCountAfterRelabel := getRowsCount(tssBlock)
+			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
@@ -414,7 +413,10 @@ func labelsToString(labels []prompbmarshal.Label) string {
 	return string(b)
 }
 
-var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+var (
+	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
+	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+)
 
 type remoteWriteCtx struct {
 	idx        int
@@ -423,7 +425,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
-	relabelMetricsDropped *metrics.Counter
+	rowsPushedAfterRelabel *metrics.Counter
+	rowsDroppedByRelabel   *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -467,7 +470,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 
@@ -483,7 +487,8 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
-	rwctx.relabelMetricsDropped = nil
+	rwctx.rowsPushedAfterRelabel = nil
+	rwctx.rowsDroppedByRelabel = nil
 }
 
 func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
@@ -499,12 +504,15 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
 		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
-		tssLen := len(tss)
+		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, nil, pcs)
-		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+		rowsCountAfterRelabel := getRowsCount(tss)
+		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 	pss[idx].Push(tss)
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
@@ -519,3 +527,11 @@ var tssRelabelPool = &sync.Pool{
 		return &a
 	},
 }
+
+func getRowsCount(tss []prompbmarshal.TimeSeries) int {
+	rowsCount := 0
+	for _, ts := range tss {
+		rowsCount += len(ts.Samples)
+	}
+	return rowsCount
+}

app/vmagent/remotewrite/statconn.go

@@ -1,7 +1,9 @@
 package remotewrite
 
 import (
+	"context"
 	"net"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -9,9 +11,26 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func statDial(networkUnused, addr string) (conn net.Conn, err error) {
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
-	conn, err = net.DialTimeout(network, addr, 5*time.Second)
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
 	dialsTotal.Inc()
 	if err != nil {
 		dialErrors.Inc()

app/vmagent/vmimport/request_handler.go

@@ -88,7 +88,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmalert/Makefile

@@ -12,20 +12,20 @@ vmalert-prod:
 vmalert-pure-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-pure
 
-vmalert-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+vmalert-linux-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-amd64
 
-vmalert-arm-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+vmalert-linux-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm
 
-vmalert-arm64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+vmalert-linux-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm64
 
-vmalert-ppc64le-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+vmalert-linux-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-ppc64le
 
-vmalert-386-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-386
+vmalert-linux-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
 
 vmalert-darwin-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmalert-darwin-amd64-prod:
 vmalert-darwin-arm64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
 
+vmalert-freebsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-freebsd-amd64
+
+vmalert-openbsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-openbsd-amd64
+
 vmalert-windows-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
 
@@ -62,13 +68,14 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-notifier.url=http://localhost:9093 \
 		-notifier.url=http://127.0.0.1:9093 \
@@ -77,17 +84,17 @@ run-vmalert: vmalert
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
 		-evaluationInterval=3s \
-		-rule.configCheckInterval=10s
+		-configCheckInterval=10s
 
 run-vmalert-sd: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
-		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-notifier.config=app/vmalert/notifier/testdata/mixed.good.yaml \
 		-configCheckInterval=10s
 
 replay-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
@@ -95,26 +102,35 @@ replay-vmalert: vmalert
 		-replay.timeFrom=2021-05-11T07:21:43Z \
 		-replay.timeTo=2021-05-29T18:40:43Z
 
-vmalert-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-amd64:
+	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-vmalert-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-vmalert-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-vmalert-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmalert-local-with-goarch
+vmalert-linux-ppc64le:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmalert-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-386:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
-vmalert-local-with-goarch:
-	APP_NAME=vmalert $(MAKE) app-local-with-goarch
+vmalert-darwin-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-darwin-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-freebsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-openbsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-goarch
 
 vmalert-pure:
 	APP_NAME=vmalert $(MAKE) app-local-pure
-
-vmalert-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch

app/vmalert/alerting.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -19,7 +20,7 @@ import (
 
 // AlertingRule is basic alert entity
 type AlertingRule struct {
-	Type         datasource.Type
+	Type         config.Type
 	RuleID       uint64
 	Name         string
 	Expr         string
@@ -38,6 +39,8 @@ type AlertingRule struct {
 	alerts map[uint64]*notifier.Alert
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -69,9 +72,10 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
 		metrics: &alertingRuleMetrics{},
@@ -139,6 +143,53 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+type labelSet struct {
+	// origin labels from series
+	// used for templating
+	origin map[string]string
+	// processed labels with additional data
+	// used as Alert labels
+	processed map[string]string
+}
+
+// toLabels converts labels from given Metric
+// to labelSet which contains original and processed labels.
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
+	ls := &labelSet{
+		origin:    make(map[string]string, len(m.Labels)),
+		processed: make(map[string]string),
+	}
+	for _, l := range m.Labels {
+		ls.origin[l.Name] = l.Value
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		ls.processed[l.Name] = l.Value
+	}
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+		Labels: ls.origin,
+		Value:  m.Values[0],
+		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
+	}
+	for k, v := range extraLabels {
+		ls.processed[k] = v
+	}
+
+	// set additional labels to identify group and rule name
+	if ar.Name != "" {
+		ls.processed[alertNameLabel] = ar.Name
+	}
+	if !*disableAlertGroupLabel && ar.GroupName != "" {
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+	}
+	return ls, nil
+}
+
 // ExecRange executes alerting rule on the given time range similarly to Exec.
 // It doesn't update internal states of the Rule and meant to be used just
 // to get time series for backfilling.
@@ -153,24 +204,7 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
-		// set additional labels to identify group and rule Name
-		if ar.Name != "" {
-			s.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			s.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(s, qFn, ar)
-		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
-		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			s.SetLabel(k, v)
-		}
-		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
 		}
@@ -189,9 +223,10 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
-				a.Start = at
-			} else if at.Sub(a.Start) >= ar.For {
+				a.ActiveAt = at
+			} else if at.Sub(a.ActiveAt) >= ar.For {
 				a.State = notifier.StateFiring
+				a.Start = at
 			}
 			prevT = at
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
@@ -200,15 +235,21 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 	return result, nil
 }
 
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequently repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := ar.q.Query(ctx, ar.Expr)
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
 	ar.lastExecError = err
-	ar.lastExecTime = time.Now()
 	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
@@ -216,62 +257,58 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
-		if a.State == notifier.StateInactive {
+		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
 			delete(ar.alerts, h)
 		}
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			m.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			m.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(m, qFn, ar)
+		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
 			return nil, fmt.Errorf("failed to expand labels: %s", err)
 		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			m.SetLabel(k, v)
-		}
-		h := hash(m)
+		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
 			// conflicting with the metric labels
-			return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
+			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, ar.lastExecError
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
+			if a.State == notifier.StateInactive {
+				// alert could be in inactive state for resolvedRetention
+				// so when we again receive metrics for it - we switch it
+				// back to notifier.StatePending
+				a.State = notifier.StatePending
+				a.ActiveAt = ts
+			}
 			if a.Value != m.Values[0] {
 				// update Value field with latest value
 				a.Value = m.Values[0]
 				// and re-exec template since Value can be used
 				// in annotations
-				a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 				if err != nil {
 					return nil, err
 				}
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ar.lastExecTime, qFn)
+		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
 		if err != nil {
 			ar.lastExecError = err
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = h
 		a.State = notifier.StatePending
+		a.ActiveAt = ts
 		ar.alerts[h] = a
 	}
-
+	var numActivePending int
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
@@ -282,28 +319,24 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 				delete(ar.alerts, h)
 				continue
 			}
-			a.State = notifier.StateInactive
+			if a.State == notifier.StateFiring {
+				a.State = notifier.StateInactive
+				a.ResolvedAt = ts
+			}
 			continue
 		}
-		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+		numActivePending++
+		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
+			a.Start = ts
 			alertsFired.Inc()
 		}
 	}
-	return ar.toTimeSeries(ar.lastExecTime.Unix()), nil
-}
-
-func expandLabels(m datasource.Metric, q notifier.QueryFn, ar *AlertingRule) (map[string]string, error) {
-	metricLabels := make(map[string]string)
-	for _, l := range m.Labels {
-		metricLabels[l.Name] = l.Value
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		return nil, fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
 	}
-	tpl := notifier.AlertTplData{
-		Labels: metricLabels,
-		Value:  m.Values[0],
-		Expr:   ar.Expr,
-	}
-	return notifier.ExecTemplate(q, ar.Labels, tpl)
+	return ar.toTimeSeries(ts.Unix()), nil
 }
 
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
@@ -336,42 +369,43 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 }
 
 // TODO: consider hashing algorithm in VM
-func hash(m datasource.Metric) uint64 {
+func hash(labels map[string]string) uint64 {
 	hash := fnv.New64a()
-	labels := m.Labels
-	sort.Slice(labels, func(i, j int) bool {
-		return labels[i].Name < labels[j].Name
-	})
-	for _, l := range labels {
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
 		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
+		if k == "__name__" {
 			continue
 		}
-		hash.Write([]byte(l.Name))
-		hash.Write([]byte(l.Value))
+		name, value := k, labels[k]
+		hash.Write([]byte(name))
+		hash.Write([]byte(value))
 		hash.Write([]byte("\xff"))
 	}
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
-	a := &notifier.Alert{
-		GroupID: ar.GroupID,
-		Name:    ar.Name,
-		Labels:  map[string]string{},
-		Value:   m.Values[0],
-		Start:   start,
-		Expr:    ar.Expr,
-	}
-	for _, l := range m.Labels {
-		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
-			continue
-		}
-		a.Labels[l.Name] = l.Value
-	}
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
 	var err error
-	a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+	}
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 	return a, err
 }
 
@@ -386,34 +420,54 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// RuleAPI returns Rule representation in form
-// of APIAlertingRule
-func (ar *AlertingRule) RuleAPI() APIAlertingRule {
-	var lastErr string
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
 	if ar.lastExecError != nil {
-		lastErr = ar.lastExecError.Error()
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
 	}
-	return APIAlertingRule{
-		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", ar.ID()),
-		GroupID:     fmt.Sprintf("%d", ar.GroupID),
-		Type:        ar.Type.String(),
-		Name:        ar.Name,
-		Expression:  ar.Expr,
-		For:         ar.For.String(),
-		LastError:   lastErr,
-		LastSamples: ar.lastExecSamples,
-		LastExec:    ar.lastExecTime,
-		Labels:      ar.Labels,
-		Annotations: ar.Annotations,
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
 	}
+	return r
 }
 
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
 	ar.mu.RLock()
 	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
 		alerts = append(alerts, ar.newAlertAPI(*a))
 	}
 	ar.mu.RUnlock()
@@ -432,7 +486,7 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Labels:      a.Labels,
 		Annotations: a.Annotations,
 		State:       a.State.String(),
-		ActiveAt:    a.Start,
+		ActiveAt:    a.ActiveAt,
 		Restored:    a.Restored,
 		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
@@ -458,7 +512,7 @@ const (
 	alertGroupNameLabel = "alertgroup"
 )
 
-// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+// alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	tss = append(tss, alertToTimeSeries(a, timestamp))
@@ -486,11 +540,11 @@ func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.Time
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
+	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
 // Restore restores the state of active alerts basing on previously written time series.
-// Restore restores only Start field. Field State will be always Pending and supposed
+// Restore restores only ActiveAt field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
 func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
@@ -498,7 +552,8 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		return fmt.Errorf("querier is nil")
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	ts := time.Now()
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 
 	// account for external labels in filter
 	var labelsFilter string
@@ -506,26 +561,63 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}
 
-	// Get the last data point in range via MetricsQL `last_over_time`.
-	// We don't use plain PromQL since Prometheus doesn't support
-	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr)
+	qMetrics, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return err
 	}
 
 	for _, m := range qMetrics {
-		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
+		ls := &labelSet{
+			origin:    make(map[string]string, len(m.Labels)),
+			processed: make(map[string]string, len(m.Labels)),
+		}
+		for _, l := range m.Labels {
+			if l.Name == "__name__" {
+				continue
+			}
+			ls.origin[l.Name] = l.Value
+			ls.processed[l.Name] = l.Value
+		}
+		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
-		a.ID = hash(m)
+		a.ID = hash(ls.processed)
 		a.State = notifier.StatePending
 		a.Restored = true
 		ar.alerts[a.ID] = a
-		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
 	return nil
 }
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	needsSending := func(a *notifier.Alert) bool {
+		if a.State == notifier.StatePending {
+			return false
+		}
+		if a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(ts)
+	}
+
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		if !needsSending(a) {
+			continue
+		}
+		a.End = ts.Add(resolveDuration)
+		if a.State == notifier.StateInactive {
+			a.End = a.ResolvedAt
+		}
+		a.LastSent = ts
+		alerts = append(alerts, *a)
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -60,7 +61,7 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 		},
 		{
 			newTestAlertingRule("for", time.Second),
-			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -75,7 +76,7 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 		},
 		{
 			newTestAlertingRule("for pending", 10*time.Second),
-			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StatePending, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -168,7 +169,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -176,7 +177,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			nil,
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -216,8 +219,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 			},
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
-			// 3: fire third alert, set second inactive, delete first one
+			// 3: fire third alert, set second inactive
 			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
@@ -300,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -311,10 +315,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 			}
 			expAlerts := make(map[uint64]*notifier.Alert)
 			for _, ta := range tc.expAlerts {
-				labels := ta.labels
-				labels = append(labels, alertNameLabel)
-				labels = append(labels, tc.rule.Name)
-				h := hash(metricWithLabels(t, labels...))
+				labels := make(map[string]string)
+				for i := 0; i < len(ta.labels); i += 2 {
+					k, v := ta.labels[i], ta.labels[i+1]
+					labels[k] = v
+				}
+				labels[alertNameLabel] = tc.rule.Name
+				h := hash(labels)
 				expAlerts[h] = ta.alert
 			}
 			for key, exp := range expAlerts {
@@ -379,9 +386,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(3, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
 			},
 		},
 		{
@@ -390,9 +397,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 		},
 		{
@@ -401,11 +408,11 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(5, 0)},
-				{State: notifier.StatePending, Start: time.Unix(20, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
 			},
 		},
 		{
@@ -417,15 +424,15 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 				//
-				{State: notifier.StatePending, Start: time.Unix(1, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
-				{State: notifier.StatePending, Start: time.Unix(5, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
@@ -478,7 +485,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 						a.Labels = make(map[string]string)
 					}
 					a.Labels[alertNameLabel] = tc.rule.Name
-					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
+					expTS = append(expTS, tc.rule.alertToTimeSeries(a, timestamp)...)
 					j++
 				}
 			}
@@ -509,8 +516,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(nil): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -525,13 +532,13 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					alertNameLabel, "metric labels",
-					alertGroupNameLabel, "groupID",
-					"foo", "bar",
-					"namespace", "baz",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					alertNameLabel:      "metric labels",
+					alertGroupNameLabel: "groupID",
+					"foo":               "bar",
+					"namespace":         "baz",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -546,12 +553,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					"foo", "bar",
-					"namespace", "baz",
-					"source", "vm",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					"foo":       "bar",
+					"namespace": "baz",
+					"source":    "vm",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -571,12 +578,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(2 * time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(3 * time.Hour)},
+				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(2 * time.Hour)},
+				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(3 * time.Hour)},
 			},
 		},
 	}
@@ -601,8 +608,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 				if got.State != exp.State {
 					t.Fatalf("expected state %d; got %d", exp.State, got.State)
 				}
-				if got.Start != exp.Start {
-					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				if got.ActiveAt != exp.ActiveAt {
+					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
 				}
 			}
 		})
@@ -617,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO())
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -633,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -642,6 +649,50 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	}
 }
 
+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
 func TestAlertingRule_Template(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
@@ -655,7 +706,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
 						alertNameLabel: "common",
@@ -663,7 +714,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
 						alertNameLabel: "common",
@@ -678,77 +729,70 @@ func TestAlertingRule_Template(t *testing.T) {
 				Name: "override label",
 				Labels: map[string]string{
 					"instance": "{{ $labels.instance }}",
-					"region":   "east",
 				},
 				Annotations: map[string]string{
-					"summary":     `Too high connection number for "{{ $labels.instance }}" for region {{ $labels.region }}`,
-					"description": `It is {{ $value }} connections for "{{ $labels.instance }}"`,
+					"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "instance", "foo"),
-				metricWithValueAndLabels(t, 10, "instance", "bar"),
+				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
 					Labels: map[string]string{
 						alertNameLabel: "override label",
 						"instance":     "foo",
-						"region":       "east",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "foo" for region east`,
-						"description": `It is 2 connections for "foo"`,
+						"summary":     `first: Too high connection number for "foo"`,
+						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
 					Labels: map[string]string{
 						alertNameLabel: "override label",
 						"instance":     "bar",
-						"region":       "east",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "bar" for region east`,
-						"description": `It is 10 connections for "bar"`,
+						"summary":     `second: Too high connection number for "bar"`,
+						"description": `override: It is 10 connections for "bar"`,
 					},
 				},
 			},
 		},
 		{
 			&AlertingRule{
-				Name:      "ExtraTemplating",
+				Name:      "OriginLabels",
 				GroupName: "Testing",
 				Labels: map[string]string{
-					"name":     "alert_{{ $labels.alertname }}",
-					"group":    "group_{{ $labels.alertgroup }}",
 					"instance": "{{ $labels.instance }}",
 				},
 				Annotations: map[string]string{
-					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
-					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 1, "instance", "foo"),
+				metricWithValueAndLabels(t, 1,
+					alertNameLabel, "originAlertname",
+					alertGroupNameLabel, "originGroupname",
+					"instance", "foo"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
-					"name", "alert_ExtraTemplating",
-					alertGroupNameLabel, "Testing",
-					"group", "group_Testing",
-					"instance", "foo")): {
+				hash(map[string]string{
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo"}): {
 					Labels: map[string]string{
-						alertNameLabel:      "ExtraTemplating",
-						"name":              "alert_ExtraTemplating",
+						alertNameLabel:      "OriginLabels",
 						alertGroupNameLabel: "Testing",
-						"group":             "group_Testing",
 						"instance":          "foo",
 					},
 					Annotations: map[string]string{
-						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
-						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
 					},
 				},
 			},
@@ -761,7 +805,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -781,6 +825,86 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 }
 
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+}
+
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)

app/vmalert/config/config.go

@@ -12,8 +12,6 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -23,17 +21,13 @@ import (
 // Group contains list of Rules grouped into
 // entity with one name and evaluation interval
 type Group struct {
-	Type        datasource.Type `yaml:"type,omitempty"`
+	Type        Type `yaml:"type,omitempty"`
 	File        string
-	Name        string             `yaml:"name"`
-	Interval    promutils.Duration `yaml:"interval"`
-	Rules       []Rule             `yaml:"rules"`
-	Concurrency int                `yaml:"concurrency"`
-	// ExtraFilterLabels is a list label filters applied to every rule
-	// request withing a group. Is compatible only with VM datasources.
-	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
-	// DEPRECATED: use Params field instead
-	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
+	Name        string              `yaml:"name"`
+	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Limit       int                 `yaml:"limit,omitempty"`
+	Rules       []Rule              `yaml:"rules"`
+	Concurrency int                 `yaml:"concurrency"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
 	Labels map[string]string `yaml:"labels"`
@@ -42,6 +36,8 @@ type Group struct {
 	Checksum string
 	// Optional HTTP URL parameters added to each rule request
 	Params url.Values `yaml:"params"`
+	// Headers contains optional HTTP headers added to each rule request
+	Headers []Header `yaml:"headers,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -59,23 +55,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	// change default value to prometheus datasource.
 	if g.Type.Get() == "" {
-		g.Type.Set(datasource.NewPrometheusType())
-	}
-
-	// backward compatibility with deprecated `ExtraFilterLabels` param
-	if len(g.ExtraFilterLabels) > 0 {
-		if g.Params == nil {
-			g.Params = url.Values{}
-		}
-		// Sort extraFilters for consistent order for query args across runs.
-		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
-		for k, v := range g.ExtraFilterLabels {
-			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
-		}
-		sort.Strings(extraFilters)
-		for _, extraFilter := range extraFilters {
-			g.Params.Add("extra_label", extraFilter)
-		}
+		g.Type.Set(NewPrometheusType())
 	}
 
 	h := md5.New()
@@ -85,7 +65,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // Validate check for internal Group or Rule configuration errors
-func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
+func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
@@ -111,11 +91,11 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
-		if validateAnnotations {
-			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+		if validateTplFn != nil {
+			if err := validateTplFn(r.Annotations); err != nil {
 				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
 			}
-			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+			if err := validateTplFn(r.Labels); err != nil {
 				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
@@ -127,12 +107,12 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Record      string             `yaml:"record,omitempty"`
-	Alert       string             `yaml:"alert,omitempty"`
-	Expr        string             `yaml:"expr"`
-	For         promutils.Duration `yaml:"for"`
-	Labels      map[string]string  `yaml:"labels,omitempty"`
-	Annotations map[string]string  `yaml:"annotations,omitempty"`
+	Record      string              `yaml:"record,omitempty"`
+	Alert       string              `yaml:"alert,omitempty"`
+	Expr        string              `yaml:"expr"`
+	For         *promutils.Duration `yaml:"for,omitempty"`
+	Labels      map[string]string   `yaml:"labels,omitempty"`
+	Annotations map[string]string   `yaml:"annotations,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -188,8 +168,11 @@ func (r *Rule) Validate() error {
 	return checkOverflow(r.XXX, "rule")
 }
 
+// ValidateTplFn must validate the given annotations
+type ValidateTplFn func(annotations map[string]string) error
+
 // Parse parses rule configs from given file patterns
-func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	var fp []string
 	for _, pattern := range pathPatterns {
 		matches, err := filepath.Glob(pattern)
@@ -199,7 +182,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
-	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -209,7 +191,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			continue
 		}
 		for _, g := range gr {
-			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
+			if err := g.Validate(validateTplFn, validateExpressions); err != nil {
 				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
 				continue
 			}
@@ -219,9 +201,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
-			if len(g.ExtraFilterLabels) > 0 {
-				isExtraFilterLabelsUsed = true
-			}
 			groups = append(groups, g)
 		}
 	}
@@ -231,9 +210,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
-	if isExtraFilterLabelsUsed {
-		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
-	}
 	return groups, nil
 }
 

app/vmalert/config/config_test.go

@@ -9,19 +9,20 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
 func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
 		t.Errorf("error parsing files %s", err)
 	}
 }
@@ -32,7 +33,7 @@ func TestParseBad(t *testing.T) {
 		expErr string
 	}{
 		{
-			[]string{"testdata/rules0-bad.rules"},
+			[]string{"testdata/rules/rules0-bad.rules"},
 			"unexpected token",
 		},
 		{
@@ -56,12 +57,16 @@ func TestParseBad(t *testing.T) {
 			"either `record` or `alert` must be set",
 		},
 		{
-			[]string{"testdata/rules1-bad.rules"},
+			[]string{"testdata/rules/rules1-bad.rules"},
 			"bad graphite expr",
 		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
 	}
 	for _, tc := range testCases {
-		_, err := Parse(tc.path, true, true)
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
 			t.Errorf("expected to get error")
 			return
@@ -219,7 +224,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test thanos",
-				Type: datasource.NewRawType("thanos"),
+				Type: NewRawType("thanos"),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -231,7 +236,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test graphite",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "some-description",
@@ -243,7 +248,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test prometheus",
-				Type: datasource.NewPrometheusType(),
+				Type: NewPrometheusType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -256,7 +261,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite inherit",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
@@ -271,7 +276,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite prometheus bad expr",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
@@ -285,8 +290,13 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "invalid rule",
 		},
 	}
+
 	for _, tc := range testCases {
-		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
 		if err == nil {
 			if tc.expErr != "" {
 				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
@@ -491,6 +501,40 @@ params:
 rules:
   - alert: foo
     expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+limit: 5
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+limit: 10
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
 `)
 	})
 }
@@ -528,30 +572,4 @@ rules:
     expr: sum by(job) (up == 1)
 `, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
 	})
-
-	t.Run("extra labels", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-  env: prod
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
-	})
-
-	t.Run("extra labels and params", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-params:
-  nocache: ["1"]
-  extra_label: ["env=prod"]
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
-	})
 }

app/vmalert/config/testdata/dir/rules6-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    headers:
+    - 'foobar'
+    rules:
+      - alert: rows
+        expr: vm_rows > 0

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -2,6 +2,7 @@ groups:
   - name: ReplayGroup
     interval: 1m
     concurrency: 1
+    limit: 1000
     rules:
       - record: type:vm_cache_entries:rate5m
         expr: sum(rate(vm_cache_entries[5m])) by (type)

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -2,11 +2,11 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
-    extra_filter_labels: # deprecated param, use `params` instead
-        job: victoriametrics
+    limit: 1000
+    headers:
+      - "MyHeader: foo"
     params:
       denyPartialResponse: ["true"]
-      extra_label: ["env=dev"]
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by(instance) > 1
@@ -49,4 +49,12 @@ groups:
         expr: |2
             sum(code:requests:rate5m{code="200"})
           /
-            sum(code:requests:rate5m)
+            sum(code:requests:rate5m)
+      - record: code:requests:slo
+        labels:
+           recording: true
+        expr: 0.95
+      - record: time:current
+        labels:
+           recording: true
+        expr: time()

app/vmalert/config/testdata/rules/rules_interval_good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupTest
+    interval: 1s
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/config/testdata/templates/templates0-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template0" }}
+Visit {{ externalURL }}
+{{ end }}

app/vmalert/config/testdata/templates/templates1-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template1" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates2-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template2" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates3-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ printf "%s to %s!" "welcome" "hell" | toUpper }}
+{{ end }}

app/vmalert/config/testdata/templates/templates4-good-tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ 1230912039102391023.0 | humanizeDuration }}
+{{ end }}

app/vmalert/config/types.go

@@ -1,7 +1,8 @@
-package datasource
+package config
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
 	"github.com/VictoriaMetrics/metricsql"
@@ -9,45 +10,45 @@ import (
 
 // Type represents data source type
 type Type struct {
-	name string
+	Name string
 }
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
 	return Type{
-		name: "prometheus",
+		Name: "prometheus",
 	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
 	return Type{
-		name: "graphite",
+		Name: "graphite",
 	}
 }
 
 // NewRawType returns datasource type from raw string
 // without validation.
 func NewRawType(d string) Type {
-	return Type{name: d}
+	return Type{Name: d}
 }
 
 // Get returns datasource type
 func (t *Type) Get() string {
-	return t.name
+	return t.Name
 }
 
 // Set changes datasource type
 func (t *Type) Set(d Type) {
-	t.name = d.name
+	t.Name = d.Name
 }
 
 // String implements String interface with default value.
 func (t Type) String() string {
-	if t.name == "" {
+	if t.Name == "" {
 		return "prometheus"
 	}
-	return t.name
+	return t.Name
 }
 
 // ValidateExpr validates query expression with datasource ql.
@@ -62,7 +63,7 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	default:
-		return fmt.Errorf("unknown datasource type=%q", t.name)
+		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}
 	return nil
 }
@@ -81,11 +82,35 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	default:
 		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
-	t.name = s
+	t.Name = s
 	return nil
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
 func (t Type) MarshalYAML() (interface{}, error) {
-	return t.name, nil
+	return t.Name, nil
+}
+
+// Header is a Key - Value struct for holding an HTTP header.
+type Header struct {
+	Key   string
+	Value string
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	if s == "" {
+		return nil
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, s)
+	}
+	h.Key = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
 }

app/vmalert/datasource/datasource.go

@@ -8,7 +8,7 @@ import (
 
 // Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string) ([]Metric, error)
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, error)
 	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
 }
 
@@ -19,9 +19,10 @@ type QuerierBuilder interface {
 
 // QuerierParams params for Querier.
 type QuerierParams struct {
-	DataSourceType     *Type
+	DataSourceType     string
 	EvaluationInterval time.Duration
 	QueryParams        url.Values
+	Headers            map[string]string
 }
 
 // Metric is the basic entity which should be return by datasource

app/vmalert/datasource/init.go

@@ -11,14 +11,20 @@ import (
 )
 
 var (
-	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
-		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix      = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
+		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend")
+	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+
+	headers = flag.String("datasource.headers", "", "Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. "+
+		"For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. "+
+		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
+
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
 	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
-	bearerToken           = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
-	bearerTokenFile       = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
+
+	bearerToken     = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
@@ -26,12 +32,22 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
+	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
+	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+
 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
 		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
 		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
-	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
@@ -52,6 +68,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
@@ -64,7 +81,11 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}
 
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
@@ -76,7 +97,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,
 		queryStep:        *queryStep,
-		dataSourceType:   NewPrometheusType(),
+		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,
 	}, nil
 }

app/vmalert/datasource/vm.go

@@ -12,6 +12,20 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
+type datasourceType string
+
+const (
+	datasourcePrometheus datasourceType = "prometheus"
+	datasourceGraphite   datasourceType = "graphite"
+)
+
+func toDatasourceType(s string) datasourceType {
+	if s == string(datasourceGraphite) {
+		return datasourceGraphite
+	}
+	return datasourcePrometheus
+}
+
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
@@ -21,33 +35,41 @@ type VMStorage struct {
 	lookBack         time.Duration
 	queryStep        time.Duration
 
-	dataSourceType     Type
+	dataSourceType     datasourceType
 	evaluationInterval time.Duration
 	extraParams        url.Values
-	disablePathAppend  bool
+	extraHeaders       []keyValue
+}
+
+type keyValue struct {
+	key   string
+	value string
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                 s.c,
-		authCfg:           s.authCfg,
-		datasourceURL:     s.datasourceURL,
-		lookBack:          s.lookBack,
-		queryStep:         s.queryStep,
-		appendTypePrefix:  s.appendTypePrefix,
-		dataSourceType:    s.dataSourceType,
-		disablePathAppend: s.disablePathAppend,
+		c:                s.c,
+		authCfg:          s.authCfg,
+		datasourceURL:    s.datasourceURL,
+		lookBack:         s.lookBack,
+		queryStep:        s.queryStep,
+		appendTypePrefix: s.appendTypePrefix,
+		dataSourceType:   s.dataSourceType,
 	}
 }
 
 // ApplyParams - changes given querier params.
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
-	if params.DataSourceType != nil {
-		s.dataSourceType = *params.DataSourceType
-	}
+	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
 	s.extraParams = params.QueryParams
+	if params.Headers != nil {
+		for key, value := range params.Headers {
+			kv := keyValue{key: key, value: value}
+			s.extraHeaders = append(s.extraHeaders, kv)
+		}
+	}
 	return s
 }
 
@@ -57,34 +79,32 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
 	return &VMStorage{
-		c:                 c,
-		authCfg:           authCfg,
-		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix:  appendTypePrefix,
-		lookBack:          lookBack,
-		queryStep:         queryStep,
-		dataSourceType:    NewPrometheusType(),
-		disablePathAppend: disablePathAppend,
+		c:                c,
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix: appendTypePrefix,
+		lookBack:         lookBack,
+		queryStep:        queryStep,
+		dataSourceType:   datasourcePrometheus,
 	}
 }
 
 // Query executes the given query and returns parsed response
-func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, error) {
 	req, err := s.newRequestPOST()
 	if err != nil {
 		return nil, err
 	}
 
-	ts := time.Now()
-	switch s.dataSourceType.String() {
-	case "prometheus":
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case "graphite":
+	case datasourceGraphite:
 		s.setGraphiteReqParams(req, query, ts)
 	default:
-		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
+		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
 	}
 
 	resp, err := s.do(ctx, req)
@@ -96,7 +116,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}()
 
 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != "prometheus" {
+	if s.dataSourceType != datasourcePrometheus {
 		parseFn = parseGraphiteResponse
 	}
 	return parseFn(req, resp)
@@ -106,8 +126,8 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != "prometheus" {
-		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
+	if s.dataSourceType != datasourcePrometheus {
+		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
 	}
 	req, err := s.newRequestPOST()
 	if err != nil {
@@ -150,9 +170,10 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
-		if auth := s.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		s.authCfg.SetHeaders(req, true)
+	}
+	for _, h := range s.extraHeaders {
+		req.Header.Set(h.key, h.value)
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_prom_api.go

@@ -2,12 +2,18 @@ package datasource
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"strconv"
 	"time"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+
+		"to the configured -datasource.url and -remoteRead.url")
+)
+
 type promResponse struct {
 	Status    string `json:"status"`
 	ErrorType string `json:"errorType"`
@@ -25,13 +31,6 @@ type promInstant struct {
 	} `json:"result"`
 }
 
-type promRange struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
-	} `json:"result"`
-}
-
 func (r promInstant) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -50,6 +49,13 @@ func (r promInstant) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promRange struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TVs    [][2]interface{}  `json:"values"`
+	} `json:"result"`
+}
+
 func (r promRange) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -74,9 +80,22 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promScalar [2]interface{}
+
+func (r promScalar) metrics() ([]Metric, error) {
+	var m Metric
+	f, err := strconv.ParseFloat(r[1].(string), 64)
+	if err != nil {
+		return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", r, r[1], err)
+	}
+	m.Values = append(m.Values, f)
+	m.Timestamps = append(m.Timestamps, int64(r[0].(float64)))
+	return []Metric{m}, nil
+}
+
 const (
-	statusSuccess, statusError = "success", "error"
-	rtVector, rtMatrix         = "vector", "matrix"
+	statusSuccess, statusError  = "success", "error"
+	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
 )
 
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
@@ -103,29 +122,29 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric,
 			return nil, err
 		}
 		return pr.metrics()
+	case rScalar:
+		var ps promScalar
+		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
+			return nil, err
+		}
+		return ps.metrics()
 	default:
 		return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
 	}
 }
 
-const (
-	prometheusInstantPath = "/api/v1/query"
-	prometheusRangePath   = "/api/v1/query_range"
-	prometheusPrefix      = "/prometheus"
-)
-
 func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusInstantPath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
 	}
-	if s.evaluationInterval > 0 {
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
@@ -136,10 +155,10 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 
 func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusRangePath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))

app/vmalert/datasource/vm_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -36,7 +37,7 @@ func TestVMInstantQuery(t *testing.T) {
 	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
 		c++
 		switch c {
-		case 7:
+		case 8:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -74,40 +75,38 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
 		case 6:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests"},"value":[1583786140,"2000"]}]}}`))
+		case 7:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	p := datasourcePrometheus
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
+	ts := time.Now()
 
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected connection error got nil")
+	expErr := func(err string) {
+		if _, err := pq.Query(ctx, query, ts); err == nil {
+			t.Fatalf("expected %q got nil", err)
+		}
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected invalid response status error got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected response body error got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected error status got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected unknown status got nil")
-	}
-	if _, err := pq.Query(ctx, query); err == nil {
-		t.Fatalf("expected non-vector resultType error  got nil")
-	}
-	m, err := pq.Query(ctx, query)
+
+	expErr("connection error")              // 0
+	expErr("invalid response status error") // 1
+	expErr("response body error")           // 2
+	expErr("error status")                  // 3
+	expErr("unknown status")                // 4
+	expErr("non-vector resultType error")   // 5
+
+	m, err := pq.Query(ctx, query, ts) // 6 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -130,10 +129,26 @@ func TestVMInstantQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m, expected)
 	}
 
-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	m, err = pq.Query(ctx, query, ts) // 7 - scalar
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metrics got %d in %+v", len(m), m)
+	}
+	expected = []Metric{
+		{
+			Timestamps: []int64{1583786142},
+			Values:     []float64{1},
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m, expected)
+	}
 
-	m, err = gq.Query(ctx, queryRender)
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
+
+	m, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -190,14 +205,13 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})
 
 	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
@@ -223,15 +237,14 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
 
-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
 	_, err = gq.QueryRange(ctx, queryRender, start, end)
 	expectError(t, err, "is not supported")
 }
 
 func TestRequestParams(t *testing.T) {
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
@@ -247,95 +260,49 @@ func TestRequestParams(t *testing.T) {
 			"prometheus path",
 			false,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus path with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
 			},
 		},
 		{
 			"prometheus prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus prefix with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
 			},
 		},
 		{
 			"prometheus range path",
 			true,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range path with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
 			"prometheus range prefix",
 			true,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range prefix with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
 			"graphite path",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 			},
 			func(t *testing.T, r *http.Request) {
 				checkEqualString(t, graphitePath, r.URL.Path)
@@ -345,7 +312,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewGraphiteType(),
+				dataSourceType:   datasourceGraphite,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
@@ -483,7 +450,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite extra params",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 				extraParams: url.Values{
 					"nocache":      {"1"},
 					"max_lookback": {"1h"},
@@ -502,14 +469,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.String() {
-			case "prometheus":
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case "graphite":
+			case datasourceGraphite:
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -517,6 +484,93 @@ func TestRequestParams(t *testing.T) {
 	}
 }
 
+func TestHeaders(t *testing.T) {
+	var testCases = []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
+}
+
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {

app/vmalert/group.go

@@ -5,31 +5,39 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu          sync.RWMutex
-	Name        string
-	File        string
-	Rules       []Rule
-	Type        datasource.Type
-	Interval    time.Duration
-	Concurrency int
-	Checksum    string
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           config.Type
+	Interval       time.Duration
+	Limit          int
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
 
-	Labels map[string]string
-	Params url.Values
+	Labels  map[string]string
+	Params  url.Values
+	Headers map[string]string
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -43,13 +51,22 @@ type Group struct {
 type groupMetrics struct {
 	iterationTotal    *utils.Counter
 	iterationDuration *utils.Summary
+	iterationMissed   *utils.Counter
+	iterationInterval *utils.Gauge
 }
 
-func newGroupMetrics(name, file string) *groupMetrics {
+func newGroupMetrics(g *Group) *groupMetrics {
 	m := &groupMetrics{}
-	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
+	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
 	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		i := g.Interval.Seconds()
+		g.mu.RUnlock()
+		return i
+	})
 	return m
 }
 
@@ -76,22 +93,27 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:        cfg.Name,
 		File:        cfg.File,
 		Interval:    cfg.Interval.Duration(),
+		Limit:       cfg.Limit,
 		Concurrency: cfg.Concurrency,
 		Checksum:    cfg.Checksum,
 		Params:      cfg.Params,
+		Headers:     make(map[string]string),
 		Labels:      cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
 		updateCh:   make(chan *Group),
 	}
-	g.metrics = newGroupMetrics(g.Name, g.File)
 	if g.Interval == 0 {
 		g.Interval = defaultInterval
 	}
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
+	for _, h := range cfg.Headers {
+		g.Headers[h.Key] = h.Value
+	}
+	g.metrics = newGroupMetrics(g)
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
 		var extraLabels map[string]string
@@ -200,7 +222,9 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
 	g.Params = newGroup.Params
+	g.Headers = newGroup.Headers
 	g.Labels = newGroup.Labels
+	g.Limit = newGroup.Limit
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
@@ -215,6 +239,8 @@ func (g *Group) close() {
 
 	g.metrics.iterationDuration.Unregister()
 	g.metrics.iterationTotal.Unregister()
+	g.metrics.iterationMissed.Unregister()
+	g.metrics.iterationInterval.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -225,6 +251,11 @@ var skipRandSleepOnGroupStart bool
 func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
 
+	e := &executor{
+		rw:                       rw,
+		notifiers:                nts,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
+
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
 		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
@@ -245,8 +276,34 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 	}
 
+	evalTS := time.Now()
+
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{rw: rw, notifiers: nts}
+
+	eval := func(ts time.Time) {
+		g.metrics.iterationTotal.Inc()
+
+		start := time.Now()
+
+		if len(g.Rules) < 1 {
+			g.metrics.iterationDuration.UpdateDuration(start)
+			g.LastEvaluation = start
+			return
+		}
+
+		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
+		for err := range errs {
+			if err != nil {
+				logger.Errorf("group %q: %s", g.Name, err)
+			}
+		}
+		g.metrics.iterationDuration.UpdateDuration(start)
+		g.LastEvaluation = start
+	}
+
+	eval(evalTS)
+
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -265,6 +322,10 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				g.mu.Unlock()
 				continue
 			}
+
+			// ensure that staleness is tracked or existing rules only
+			e.purgeStaleSeries(g.Rules)
+
 			if g.Interval != ng.Interval {
 				g.Interval = ng.Interval
 				t.Stop()
@@ -273,44 +334,48 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			g.mu.Unlock()
 			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
-			g.metrics.iterationTotal.Inc()
-			iterationStart := time.Now()
-			if len(g.Rules) > 0 {
-				resolveDuration := getResolveDuration(g.Interval)
-				errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
-				for err := range errs {
-					if err != nil {
-						logger.Errorf("group %q: %s", g.Name, err)
-					}
-				}
+			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed > 0 {
+				g.metrics.iterationMissed.Inc()
 			}
-			g.metrics.iterationDuration.UpdateDuration(iterationStart)
+			evalTS = evalTS.Add((missed + 1) * g.Interval)
+
+			eval(evalTS)
 		}
 	}
 }
 
-// resolveDuration for alerts is equal to 3 interval evaluations
-// so in case if vmalert stops sending updates for some reason,
-// notifier could automatically resolve the alert.
-func getResolveDuration(groupInterval time.Duration) time.Duration {
-	resolveInterval := groupInterval * 3
-	if *maxResolveDuration > 0 && (resolveInterval > *maxResolveDuration) {
-		return *maxResolveDuration
+// getResolveDuration returns the duration after which firing alert
+// can be considered as resolved.
+func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Duration {
+	if groupInterval > delta {
+		delta = groupInterval
 	}
-	return resolveInterval
+	resolveDuration := delta * 4
+	if maxDuration > 0 && resolveDuration > maxDuration {
+		resolveDuration = maxDuration
+	}
+	return resolveDuration
 }
 
 type executor struct {
 	notifiers func() []notifier.Notifier
 	rw        *remotewrite.Client
+
+	previouslySentSeriesToRWMu sync.Mutex
+	// previouslySentSeriesToRW stores series sent to RW on previous iteration
+	// map[ruleID]map[ruleLabels][]prompb.Label
+	// where `ruleID` is ID of the Rule within a Group
+	// and `ruleLabels` is []prompb.Label marshalled to a string
+	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, resolveDuration)
+			res <- e.exec(ctx, rule, ts, resolveDuration, limit)
 		}
 		close(res)
 		return res
@@ -323,7 +388,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, resolveDuration)
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -344,51 +409,122 @@ var (
 	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration, limit int) error {
 	execTotal.Inc()
 
-	tss, err := rule.Exec(ctx)
+	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
 	}
 
-	if len(tss) > 0 && e.rw != nil {
-		for _, ts := range tss {
-			remoteWriteTotal.Inc()
-			if err := e.rw.Push(ts); err != nil {
-				remoteWriteErrors.Inc()
-				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
+	errGr := new(utils.ErrGroup)
+	if e.rw != nil {
+		pushToRW := func(tss []prompbmarshal.TimeSeries) {
+			for _, ts := range tss {
+				remoteWriteTotal.Inc()
+				if err := e.rw.Push(ts); err != nil {
+					remoteWriteErrors.Inc()
+					errGr.Add(fmt.Errorf("rule %q: remote write failure: %w", rule, err))
+				}
 			}
 		}
+		pushToRW(tss)
+		staleSeries := e.getStaleSeries(rule, tss, ts)
+		pushToRW(staleSeries)
 	}
 
 	ar, ok := rule.(*AlertingRule)
 	if !ok {
 		return nil
 	}
-	var alerts []notifier.Alert
-	for _, a := range ar.alerts {
-		switch a.State {
-		case notifier.StateFiring:
-			a.End = time.Now().Add(resolveDuration)
-			alerts = append(alerts, *a)
-		case notifier.StateInactive:
-			// set End to execStart to notify
-			// that it was just resolved
-			a.End = time.Now()
-			alerts = append(alerts, *a)
-		}
-	}
+
+	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
 
-	errGr := new(utils.ErrGroup)
+	wg := sync.WaitGroup{}
 	for _, nt := range e.notifiers() {
-		if err := nt.Send(ctx, alerts); err != nil {
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
-		}
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
 	}
+	wg.Wait()
 	return errGr.Err()
 }
+
+// getStaledSeries checks whether there are stale series from previously sent ones.
+func (e *executor) getStaleSeries(rule Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
+	for _, ts := range tss {
+		// convert labels to strings so we can compare with previously sent series
+		key := labelsToString(ts.Labels)
+		ruleLabels[key] = ts.Labels
+	}
+
+	rID := rule.ID()
+	var staleS []prompbmarshal.TimeSeries
+	// check whether there are series which disappeared and need to be marked as stale
+	e.previouslySentSeriesToRWMu.Lock()
+	for key, labels := range e.previouslySentSeriesToRW[rID] {
+		if _, ok := ruleLabels[key]; ok {
+			continue
+		}
+		// previously sent series are missing in current series, so we mark them as stale
+		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
+		staleS = append(staleS, ss)
+	}
+	// set previous series to current
+	e.previouslySentSeriesToRW[rID] = ruleLabels
+	e.previouslySentSeriesToRWMu.Unlock()
+
+	return staleS
+}
+
+// purgeStaleSeries deletes references in tracked
+// previouslySentSeriesToRW list to Rules which aren't present
+// in the given activeRules list. The method is used when the list
+// of loaded rules has changed and executor has to remove
+// references to non-existing rules.
+func (e *executor) purgeStaleSeries(activeRules []Rule) {
+	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
+
+	e.previouslySentSeriesToRWMu.Lock()
+
+	for _, rule := range activeRules {
+		id := rule.ID()
+		prev, ok := e.previouslySentSeriesToRW[id]
+		if ok {
+			// keep previous series for staleness detection
+			newPreviouslySentSeriesToRW[id] = prev
+		}
+	}
+	e.previouslySentSeriesToRW = nil
+	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
+
+	e.previouslySentSeriesToRWMu.Unlock()
+}
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b strings.Builder
+	b.WriteRune('{')
+	for i, label := range labels {
+		if len(label.Name) == 0 {
+			b.WriteString("__name__")
+		} else {
+			b.WriteString(label.Name)
+		}
+		b.WriteRune('=')
+		b.WriteString(strconv.Quote(label.Value))
+		if i < len(labels)-1 {
+			b.WriteRune(',')
+		}
+	}
+	b.WriteRune('}')
+	return b.String()
+}

app/vmalert/group_test.go

@@ -3,6 +3,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -145,7 +148,7 @@ func TestUpdateWith(t *testing.T) {
 					t.Fatalf("expected to have rule %q; got %q", want, got)
 				}
 				if err := compareRules(t, got, want); err != nil {
-					t.Fatalf("comparsion error: %s", err)
+					t.Fatalf("comparison error: %s", err)
 				}
 			}
 		})
@@ -154,14 +157,15 @@ func TestUpdateWith(t *testing.T) {
 
 func TestGroupStart(t *testing.T) {
 	// TODO: make parsing from string instead of file
-	groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
-	const evalInterval = time.Millisecond
+
 	fs := &fakeQuerier{}
 	fn := &fakeNotifier{}
 
+	const evalInterval = time.Millisecond
 	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
 	g.Concurrency = 2
 
@@ -170,7 +174,7 @@ func TestGroupStart(t *testing.T) {
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
 	r := g.Rules[0].(*AlertingRule)
-	alert1, err := r.newAlert(m1, time.Now(), nil)
+	alert1, err := r.newAlert(m1, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -183,13 +187,9 @@ func TestGroupStart(t *testing.T) {
 	// add service labels
 	alert1.Labels[alertNameLabel] = alert1.Name
 	alert1.Labels[alertGroupNameLabel] = g.Name
-	var labels1 []string
-	for k, v := range alert1.Labels {
-		labels1 = append(labels1, k, v)
-	}
-	alert1.ID = hash(metricWithLabels(t, labels1...))
+	alert1.ID = hash(alert1.Labels)
 
-	alert2, err := r.newAlert(m2, time.Now(), nil)
+	alert2, err := r.newAlert(m2, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -202,11 +202,7 @@ func TestGroupStart(t *testing.T) {
 	// add service labels
 	alert2.Labels[alertNameLabel] = alert2.Name
 	alert2.Labels[alertGroupNameLabel] = g.Name
-	var labels2 []string
-	for k, v := range alert2.Labels {
-		labels2 = append(labels2, k, v)
-	}
-	alert2.ID = hash(metricWithLabels(t, labels2...))
+	alert2.ID = hash(alert2.Labels)
 
 	finished := make(chan struct{})
 	fs.add(m1)
@@ -223,6 +219,12 @@ func TestGroupStart(t *testing.T) {
 	expectedAlerts := []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
+	gotAlertsNum := fn.getCounter()
+	if gotAlertsNum < len(expectedAlerts)*2 {
+		t.Fatalf("expected to receive at least %d alerts; got %d instead",
+			len(expectedAlerts)*2, gotAlertsNum)
+	}
+
 	// reset previous data
 	fs.reset()
 	// and set only one datapoint for response
@@ -232,7 +234,8 @@ func TestGroupStart(t *testing.T) {
 	time.Sleep(20 * evalInterval)
 
 	gotAlerts = fn.getAlerts()
-	expectedAlerts = []notifier.Alert{*alert1}
+	alert2.State = notifier.StateInactive
+	expectedAlerts = []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
 	g.close()
@@ -243,22 +246,209 @@ func TestResolveDuration(t *testing.T) {
 	testCases := []struct {
 		groupInterval time.Duration
 		maxDuration   time.Duration
+		resendDelay   time.Duration
 		expected      time.Duration
 	}{
-		{time.Minute, 0, 3 * time.Minute},
-		{3 * time.Minute, 0, 9 * time.Minute},
-		{time.Minute, 2 * time.Minute, 2 * time.Minute},
-		{0, 0, 0},
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
 	}
-	defaultResolveDuration := *maxResolveDuration
-	defer func() { *maxResolveDuration = defaultResolveDuration }()
+
 	for _, tc := range testCases {
 		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
-			*maxResolveDuration = tc.maxDuration
-			got := getResolveDuration(tc.groupInterval)
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
 			if got != tc.expected {
 				t.Errorf("expected to have %v; got %v", tc.expected, got)
 			}
 		})
 	}
 }
+
+func TestGetStaleSeries(t *testing.T) {
+	ts := time.Now()
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	f := func(rule Rule, labels, expLabels [][]prompbmarshal.Label) {
+		t.Helper()
+		var tss []prompbmarshal.TimeSeries
+		for _, l := range labels {
+			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
+		}
+		staleS := e.getStaleSeries(rule, tss, ts)
+		if staleS == nil && expLabels == nil {
+			return
+		}
+		if len(staleS) != len(expLabels) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expLabels), len(staleS))
+		}
+		for i, exp := range expLabels {
+			got := staleS[i]
+			if !reflect.DeepEqual(exp, got.Labels) {
+				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
+					exp, got.Labels)
+			}
+			if len(got.Samples) != 1 {
+				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
+			}
+			if !decimal.IsStaleNaN(got.Samples[0].Value) {
+				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
+			}
+		}
+	}
+
+	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
+
+	// single series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		nil)
+
+	// multiple series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
+
+	// multiple rules and series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 2},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+}
+
+func TestPurgeStaleSeries(t *testing.T) {
+	ts := time.Now()
+	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
+	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
+
+	f := func(curRules, newRules, expStaleRules []Rule) {
+		t.Helper()
+		e := &executor{
+			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		}
+		// seed executor with series for
+		// current rules
+		for _, rule := range curRules {
+			e.getStaleSeries(rule, tss, ts)
+		}
+
+		e.purgeStaleSeries(newRules)
+
+		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expStaleRules), len(e.previouslySentSeriesToRW))
+		}
+
+		for _, exp := range expStaleRules {
+			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
+				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
+			}
+		}
+	}
+
+	f(nil, nil, nil)
+	f(
+		nil,
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+	)
+}
+
+func TestFaultyNotifier(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := newTestAlertingRule("instant", 0)
+	r.q = fq
+
+	fn := &fakeNotifier{}
+	e := &executor{
+		notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&faultyNotifier{},
+				fn,
+			}
+		},
+	}
+	delay := 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), delay)
+	defer cancel()
+
+	go func() {
+		_ = e.exec(ctx, r, time.Now(), 0, 10)
+	}()
+
+	tn := time.Now()
+	deadline := tn.Add(delay / 2)
+	for {
+		if fn.getCounter() > 0 {
+			return
+		}
+		if tn.After(deadline) {
+			break
+		}
+		tn = time.Now()
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Fatalf("alive notifier didn't receive notification by %v", deadline)
+}

app/vmalert/helpers_test.go

@@ -44,10 +44,10 @@ func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Qu
 }
 
 func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
-	return fq.Query(ctx, q)
+	return fq.Query(ctx, q, time.Now())
 }
 
-func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
@@ -61,6 +61,8 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric,
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert
+	// records number of received alerts in total
+	counter int
 }
 
 func (*fakeNotifier) Close()       {}
@@ -68,16 +70,35 @@ func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
+	fn.counter += len(alerts)
 	fn.alerts = alerts
 	return nil
 }
 
+func (fn *fakeNotifier) getCounter() int {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.counter
+}
+
 func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	fn.Lock()
 	defer fn.Unlock()
 	return fn.alerts
 }
 
+type faultyNotifier struct {
+	fakeNotifier
+}
+
+func (fn *faultyNotifier) Send(ctx context.Context, _ []notifier.Alert) error {
+	d, ok := ctx.Deadline()
+	if ok {
+		time.Sleep(time.Until(d))
+	}
+	return fmt.Errorf("send failed")
+}
+
 func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
 	return metricWithValuesAndLabels(t, []float64{value}, labels...)
 }
@@ -107,6 +128,21 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
+func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	var ls []prompbmarshal.Label
+	for i := 0; i < len(labels); i += 2 {
+		ls = append(ls, prompbmarshal.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return ls
+}
+
 func compareGroups(t *testing.T, a, b *Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -128,7 +164,7 @@ func compareGroups(t *testing.T, a, b *Group) {
 			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
 		}
 		if err := compareRules(t, want, got); err != nil {
-			t.Fatalf("comparsion error: %s", err)
+			t.Fatalf("comparison error: %s", err)
 		}
 	}
 }

app/vmalert/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -22,6 +23,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -34,6 +36,13 @@ Examples:
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
+	ruleTemplatesPath = flagutil.NewArray("rule.templates", `Path or glob pattern to location with go template definitions
+	for rules annotations templating. Flag can be specified multiple times.
+Examples:
+ -rule.templates="/path/to/file". Path to a single file with go templates
+ -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+absolute path to all .tpl files in root.`)
+
 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
 		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
 
@@ -47,9 +56,11 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
 	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
 		"which is by default equal to 3 evaluation intervals of the parent group.")
+	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
+eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/vmalert/api/v1/alert?group_id=&alert_id=' is used`)
 	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
 
@@ -71,11 +82,15 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
+
+	err := templates.Load(*ruleTemplatesPath, true)
+	if err != nil {
+		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
+	}
 
 	if *dryRun {
-		u, _ := url.Parse("https://victoriametrics.com/")
-		notifier.InitTemplateFunc(u)
-		groups, err := config.Parse(*rulePath, true, true)
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -89,12 +104,17 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init `external.url`: %s", err)
 	}
-	notifier.InitTemplateFunc(eu)
+
 	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
 	if err != nil {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
 	}
 
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
@@ -103,8 +123,7 @@ func main() {
 		if rw == nil {
 			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
-		notifier.InitTemplateFunc(eu)
-		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
 		}
@@ -125,9 +144,8 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
-
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
-	groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 	if err != nil {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
@@ -168,7 +186,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	labels := make(map[string]string, 0)
+	labels := make(map[string]string)
 	for _, s := range *externalLabels {
 		if len(s) == 0 {
 			continue
@@ -226,8 +244,9 @@ func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL
 
 func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
 	if externalAlertSource == "" {
-		return func(alert notifier.Alert) string {
-			return fmt.Sprintf("%s/api/v1/%s/%s/status", externalURL, strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10))
+		return func(a notifier.Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/api/v1/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
 		}, nil
 	}
 	if validateTemplate {
@@ -241,7 +260,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, m)
+		templated, err := alert.ExecTemplate(nil, nil, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
@@ -271,6 +290,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		defer ticker.Stop()
 	}
 
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	// init reload metrics with positive values to improve alerting conditions
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
@@ -279,7 +303,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		case <-ctx.Done():
 			return
 		case <-sighupCh:
-			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
+			tmplMsg := ""
+			if len(*ruleTemplatesPath) > 0 {
+				tmplMsg = fmt.Sprintf("and templates %q ", *ruleTemplatesPath)
+			}
+			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
@@ -289,7 +317,14 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		err := templates.Load(*ruleTemplatesPath, false)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to load new templates: %s", err)
+			continue
+		}
+		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
@@ -297,6 +332,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
 			// set success to 1 since previous reload
 			// could have been unsuccessful
 			configSuccess.Set(1)
@@ -309,6 +345,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		templates.Reload()
 		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())

app/vmalert/main_test.go

@@ -41,12 +41,13 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/api/v1/42/2/status"; exp != fn(testAlert) {
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/api/v1/alert?%s=42&%s=2", paramGroupID, paramAlertID)
+	if exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Errorf("expected tempalte validation error got nil")
+		t.Errorf("expected template validation error got nil")
 	}
 	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true)
 	if err != nil {

app/vmalert/manager.go

@@ -37,7 +37,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 
 	g, ok := m.groups[gID]
 	if !ok {
-		return nil, fmt.Errorf("can't find group with id %q", gID)
+		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
 	for _, rule := range g.Rules {
 		ar, ok := rule.(*AlertingRule)
@@ -48,7 +48,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 			return apiAlert, nil
 		}
 	}
-	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
+	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)
 }
 
 func (m *manager) start(ctx context.Context, groupsCfg []config.Group) error {
@@ -163,21 +163,18 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:        g.Name,
-		Type:        g.Type.String(),
-		File:        g.File,
-		Interval:    g.Interval.String(),
-		Concurrency: g.Concurrency,
-		Params:      urlValuesToStrings(g.Params),
-		Labels:      g.Labels,
+		Name:           g.Name,
+		Type:           g.Type.String(),
+		File:           g.File,
+		Interval:       g.Interval.Seconds(),
+		LastEvaluation: g.LastEvaluation,
+		Concurrency:    g.Concurrency,
+		Params:         urlValuesToStrings(g.Params),
+		Headers:        headersToStrings(g.Headers),
+		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
-		switch v := r.(type) {
-		case *AlertingRule:
-			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
-		case *RecordingRule:
-			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
-		}
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return ag
 }
@@ -202,3 +199,23 @@ func urlValuesToStrings(values url.Values) []string {
 	}
 	return res
 }
+
+func headersToStrings(headers map[string]string) []string {
+	if len(headers) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(headers))
+	for k := range headers {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		v := headers[k]
+		res = append(res, fmt.Sprintf("%s: %s", k, v))
+	}
+
+	return res
+}

app/vmalert/manager_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"math/rand"
-	"net/url"
 	"os"
 	"strings"
 	"sync"
@@ -11,14 +10,15 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
@@ -29,7 +29,7 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
 	cfg := loadCfg(t, []string{"foo/bar"}, true, true)
 	if err := m.update(context.Background(), cfg, false); err != nil {
-		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
+		t.Fatalf("expected to load successfully with empty rules dir; got err instead: %v", err)
 	}
 }
 
@@ -47,9 +47,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 		"config/testdata/dir/rules0-bad.rules",
 		"config/testdata/dir/rules1-good.rules",
 		"config/testdata/dir/rules1-bad.rules",
-		"config/testdata/rules0-good.rules",
-		"config/testdata/rules1-good.rules",
-		"config/testdata/rules2-good.rules",
+		"config/testdata/rules/rules0-good.rules",
+		"config/testdata/rules/rules1-good.rules",
+		"config/testdata/rules/rules2-good.rules",
 	}
 	evalInterval := *evaluationInterval
 	defer func() { *evaluationInterval = evalInterval }()
@@ -68,7 +68,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			defer wg.Done()
 			for i := 0; i < iterations; i++ {
 				rnd := rand.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, true, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -125,13 +125,13 @@ func TestManagerUpdate(t *testing.T) {
 	}{
 		{
 			name:       "update good rules",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules1-good.rules",
 			want: []*Group{
 				{
 					File:     "config/testdata/dir/rules1-good.rules",
 					Name:     "duplicatedGroupDiffFiles",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules: []Rule{
 						&AlertingRule{
@@ -150,20 +150,20 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update good rules from 1 to 2 groups",
-			initPath:   "config/testdata/dir/rules1-good.rules",
-			updatePath: "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/dir/rules/rules1-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules:    []Rule{VMRows},
 					Interval: defaultEvalInterval,
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -172,21 +172,21 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update with one bad rule file",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules2-bad.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Name:     "TestGroup",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -196,19 +196,19 @@ func TestManagerUpdate(t *testing.T) {
 		{
 			name:       "update empty dir rules from 0 to 2 groups",
 			initPath:   "config/testdata/empty/*",
-			updatePath: "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -231,7 +231,7 @@ func TestManagerUpdate(t *testing.T) {
 				t.Fatalf("failed to complete initial rules update: %s", err)
 			}
 
-			cfgUpdate, err := config.Parse([]string{tc.updatePath}, true, true)
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
 			if err == nil { // update can fail and that's expected
 				_ = m.update(ctx, cfgUpdate, false)
 			}
@@ -329,7 +329,11 @@ func TestManagerUpdateNegative(t *testing.T) {
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
-	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)
+	var validateTplFn config.ValidateTplFn
+	if validateAnnotations {
+		validateTplFn = notifier.ValidateTemplates
+	}
+	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {
 		t.Fatal(err)
 	}

app/vmalert/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
 ARG TARGETARCH
-COPY vmalert-${TARGETARCH}-prod ./vmalert-prod
+COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod

app/vmalert/notifier/alert.go

@@ -5,10 +5,13 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"text/template"
+	textTpl "text/template"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // Alert the triggered alert
@@ -26,10 +29,16 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
-	// Start defines the moment of time when Alert has triggered
+	// ActiveAt defines the moment of time when Alert has become active
+	ActiveAt time.Time
+	// Start defines the moment of time when Alert has become firing
 	Start time.Time
 	// End defines the moment of time when Alert supposed to expire
 	End time.Time
+	// ResolvedAt defines the moment when Alert was switched from Firing to Inactive
+	ResolvedAt time.Time
+	// LastSent defines the moment when Alert was sent last time
+	LastSent time.Time
 	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
 	// ID is the unique identifer for the Alert
@@ -82,26 +91,38 @@ var tplHeaders = []string{
 // map of annotations.
 // Every alert could have a different datasource, so function
 // requires a queryFunction as an argument.
-func (a *Alert) ExecTemplate(q QueryFn, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
-	return templateAnnotations(annotations, tplData, funcsWithQuery(q))
+func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error getting a template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ExecTemplate executes the given template for given annotations map.
-func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) {
-	return templateAnnotations(annotations, tpl, funcsWithQuery(q))
+func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error cloning template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ValidateTemplates validate annotations for possible template error, uses empty data for template population
 func ValidateTemplates(annotations map[string]string) error {
-	_, err := templateAnnotations(annotations, AlertTplData{
+	tmpl, err := templates.Get()
+	if err != nil {
+		return err
+	}
+	_, err = templateAnnotations(annotations, AlertTplData{
 		Labels: map[string]string{},
 		Value:  0,
-	}, tmplFunc)
+	}, tmpl, false)
 	return err
 }
 
-func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap) (map[string]string, error) {
+func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
@@ -114,7 +135,7 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 		builder.Grow(len(header) + len(text))
 		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, tmpl, execute); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -130,14 +151,35 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
-	t := template.New("").Funcs(funcs).Option("missingkey=zero")
-	tpl, err := t.Parse(text)
+func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
+	tpl, err := tmpl.Clone()
 	if err != nil {
-		return fmt.Errorf("error parsing annotation: %w", err)
+		return fmt.Errorf("error cloning template before parse annotation: %w", err)
+	}
+	tpl, err = tpl.Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation template: %w", err)
+	}
+	if !execute {
+		return nil
 	}
 	if err = tpl.Execute(dst, data); err != nil {
 		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}
 	return nil
 }
+
+func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
+	for k, v := range a.Labels {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	promrelabel.SortLabels(labels)
+	if relabelCfg != nil {
+		return relabelCfg.Apply(labels, 0, false)
+	}
+	return labels
+}

app/vmalert/notifier/alert_test.go

@@ -2,13 +2,16 @@ package notifier
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
-	extLabels := make(map[string]string, 0)
+	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
 		extDC      = "east"
@@ -130,7 +133,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			tpl, err := tc.alert.ExecTemplate(qFn, tc.annotations)
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -146,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestAlert_toPromLabels(t *testing.T) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
+		t.Helper()
+		a := Alert{Labels: labels}
+		got := a.toPromLabels(relabel)
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
+				exp, got)
+		}
+	}
+
+	fn(nil, nil, nil)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		nil,
+	)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	fn(
+		map[string]string{"a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		pcs,
+	)
+}

app/vmalert/notifier/alertmanager.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
@@ -21,6 +23,8 @@ type AlertManager struct {
 	timeout time.Duration
 
 	authCfg *promauth.Config
+	// stores already parsed RelabelConfigs object
+	relabelConfigs *promrelabel.ParsedConfigs
 
 	metrics *metrics
 }
@@ -58,7 +62,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 
 func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
-	writeamRequest(b, alerts, am.argFunc)
+	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
 	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
@@ -75,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	req = req.WithContext(ctx)
 
 	if am.authCfg != nil {
-		if auth := am.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		am.authCfg.SetHeaders(req, true)
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -102,7 +104,8 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig, timeout time.Duration) (*AlertManager, error) {
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
+	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
@@ -112,21 +115,30 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
-	ba := &promauth.BasicAuthConfig{}
+	ba := new(promauth.BasicAuthConfig)
+	oauth := new(promauth.OAuth2Config)
 	if authCfg.BasicAuth != nil {
 		ba = authCfg.BasicAuth
 	}
-	aCfg, err := utils.AuthConfig(ba.Username, ba.Password.String(), ba.PasswordFile, authCfg.BearerToken.String(), authCfg.BearerTokenFile)
+	if authCfg.OAuth2 != nil {
+		oauth = authCfg.OAuth2
+	}
+
+	aCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
 	return &AlertManager{
-		addr:    alertManagerURL,
-		argFunc: fn,
-		authCfg: aCfg,
-		client:  &http.Client{Transport: tr},
-		timeout: timeout,
-		metrics: newMetrics(alertManagerURL),
+		addr:           alertManagerURL,
+		argFunc:        fn,
+		authCfg:        aCfg,
+		relabelConfigs: relabelCfg,
+		client:         &http.Client{Transport: tr},
+		timeout:        timeout,
+		metrics:        newMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,9 +1,11 @@
 {% import (
     "time"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
 {
@@ -14,8 +16,9 @@
     {% endif %}
     "labels": {
         "alertname":{%q= alert.Name %}
-        {% for k,v := range alert.Labels %}
-            ,{%q= k %}:{%q= v %}
+        {% code lbls := alert.toPromLabels(relabelCfg) %}
+        {% for _, l := range lbls %}
+            ,{%q= l.Name %}:{%q= l.Value %}
         {% endfor %}
     },
     "annotations": {

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -7,124 +7,129 @@ package notifier
 //line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		qw422016.N().S(`"labels": {"alertname":`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-		for k, v := range alert.Labels {
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().Q(alert.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:19
-		qw422016.N().S(`},"annotations": {`)
+		lbls := alert.toPromLabels(relabelCfg)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		for _, l := range lbls {
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Value)
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 		c := len(alert.Annotations)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		for k, v := range alert.Annotations {
-//line app/vmalert/notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
 			c = c - 1
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	streamamRequest(qw422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	writeamRequest(qb422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

app/vmalert/notifier/alertmanager_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
-	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, 0)
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -89,7 +89,7 @@ func TestAlertManager_Send(t *testing.T) {
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, aCfg, 0)
+	}, aCfg, nil, 0)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}

app/vmalert/notifier/config.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -29,16 +30,21 @@ type Config struct {
 	// ConsulSDConfigs contains list of settings for service discovery via Consul
 	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// DNSSDConfigs ontains list of settings for service discovery via DNS.
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
+
 	// StaticConfigs contains list of static targets
 	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
 
 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
-	// RelabelConfigs contains list of relabeling rules
+	// RelabelConfigs contains list of relabeling rules for entities discovered via SD
 	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
-
+	// AlertRelabelConfigs contains list of relabeling rules alert labels
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
 	// The timeout used when sending alerts.
-	Timeout promutils.Duration `yaml:"timeout,omitempty"`
+	Timeout *promutils.Duration `yaml:"timeout,omitempty"`
 
 	// Checksum stores the hash of yaml definition for the config.
 	// May be used to detect any changes to the config file.
@@ -52,11 +58,14 @@ type Config struct {
 
 	// stores already parsed RelabelConfigs object
 	parsedRelabelConfigs *promrelabel.ParsedConfigs
+	// stores already parsed AlertRelabelConfigs object
+	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }
 
 // StaticConfig contains list of static targets in the following form:
-// 	targets:
-//  	[ - '<host>' ]
+//
+//	targets:
+//	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
 }
@@ -78,6 +87,11 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return fmt.Errorf("failed to parse relabeling config: %w", err)
 	}
 	cfg.parsedRelabelConfigs = rCfg
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
+	}
+	cfg.parsedAlertRelabelConfigs = arCfg
 
 	b, err := yaml.Marshal(cfg)
 	if err != nil {

app/vmalert/notifier/config_test.go

@@ -12,6 +12,7 @@ func TestConfigParseGood(t *testing.T) {
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
+	f("testdata/dns.good.yaml")
 	f("testdata/static.good.yaml")
 }
 

app/vmalert/notifier/config_watcher.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
 
 // configWatcher supports dynamic reload of Notifier objects
@@ -141,7 +142,7 @@ func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator)
 		}
 		duplicates[u] = struct{}{}
 
-		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.Timeout.Duration())
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			errors = append(errors, err)
 			continue
@@ -165,7 +166,7 @@ func (cw *configWatcher) start() error {
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
@@ -195,6 +196,24 @@ func (cw *configWatcher) start() error {
 			return fmt.Errorf("failed to start consulSD discovery: %s", err)
 		}
 	}
+
+	if len(cw.cfg.DNSSDConfigs) > 0 {
+		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.DNSSDConfigs {
+				sdc := &cw.cfg.DNSSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start DNSSD discovery: %s", err)
+		}
+	}
 	return nil
 }
 

app/vmalert/notifier/init.go

@@ -3,9 +3,11 @@ package notifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -15,11 +17,15 @@ var (
 	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
 	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
 
-	addrs                 = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+
 	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
+	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
+
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
 	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
 	tlsKeyFile            = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
@@ -27,6 +33,17 @@ var (
 		"By default system CA is used")
 	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")
+
+	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )
 
 // cw holds a configWatcher for configPath configuration file
@@ -57,9 +74,10 @@ var (
 
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
-//   * configuration via flags (for backward compatibility). Is always static
+//   - configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
-//   * configuration via file. Supports live reloads and service discovery.
+//   - configuration via file. Supports live reloads and service discovery.
+//
 // Init returns an error if both mods are used.
 func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
 	if externalLabels != nil || externalURL != "" {
@@ -68,6 +86,12 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 
 	externalURL = extURL
 	externalLabels = extLabels
+	eu, err := url.Parse(externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse external URL: %s", err)
+	}
+
+	templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
 
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
@@ -87,7 +111,6 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 		return staticNotifiersFn, nil
 	}
 
-	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %s", err)
@@ -111,9 +134,19 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
 				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
 			},
+			BearerToken:     promauth.NewSecret(bearerToken.GetOptionalArg(i)),
+			BearerTokenFile: bearerTokenFile.GetOptionalArg(i),
+			OAuth2: &promauth.OAuth2Config{
+				ClientID:         oauth2ClientID.GetOptionalArg(i),
+				ClientSecret:     promauth.NewSecret(oauth2ClientSecret.GetOptionalArg(i)),
+				ClientSecretFile: oauth2ClientSecretFile.GetOptionalArg(i),
+				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
+				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
+			},
 		}
+
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
 		if err != nil {
 			return nil, err
 		}
@@ -137,6 +170,8 @@ const (
 	TargetStatic TargetType = "static"
 	// TargetConsul is for targets discovered via Consul
 	TargetConsul TargetType = "consulSD"
+	// TargetDNS is for targets discovered via DNS
+	TargetDNS TargetType = "DNSSD"
 )
 
 // GetTargets returns list of static or discovered targets

app/vmalert/notifier/package_test.go

@@ -1,13 +1,14 @@
 package notifier
 
 import (
-	"net/url"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"os"
 	"testing"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }

app/vmalert/notifier/testdata/consul.good.yaml

@@ -10,4 +10,7 @@ relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/dns.good.yaml

@@ -0,0 +1,12 @@
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+relabel_configs:
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -11,8 +11,18 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - consul
+
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+
 relabel_configs:
   - source_labels: [__meta_consul_tags]
     regex: .*,__scheme__=([^,]+),.*
     replacement: '${1}'
-    target_label: __scheme__
+    target_label: __scheme__
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name

app/vmalert/notifier/testdata/static.good.yaml

@@ -2,3 +2,6 @@ static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/recording.go

@@ -18,7 +18,7 @@ import (
 // to evaluate configured Expression and
 // return TimeSeries as result.
 type RecordingRule struct {
-	Type    datasource.Type
+	Type    config.Type
 	RuleID  uint64
 	Name    string
 	Expr    string
@@ -31,6 +31,8 @@ type RecordingRule struct {
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -68,9 +70,10 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
 		}),
 	}
 
@@ -122,18 +125,24 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }
 
 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := rr.q.Query(ctx, rr.Expr)
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
 
-	rr.lastExecTime = time.Now()
+	rr.lastExecTime = ts
+	rr.lastExecDuration = time.Since(ts)
 	rr.lastExecError = err
 	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
 	}
 
+	numSeries := len(qMetrics)
+	if limit > 0 && numSeries > limit {
+		return nil, fmt.Errorf("exec exceeded limit of %d with %d series", limit, numSeries)
+	}
+
 	duplicates := make(map[string]struct{}, len(qMetrics))
 	var tss []prompbmarshal.TimeSeries
 	for _, r := range qMetrics {
@@ -193,23 +202,27 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	return nil
 }
 
-// RuleAPI returns Rule representation in form
-// of APIRecordingRule
-func (rr *RecordingRule) RuleAPI() APIRecordingRule {
-	var lastErr string
-	if rr.lastExecError != nil {
-		lastErr = rr.lastExecError.Error()
-	}
-	return APIRecordingRule{
+// ToAPI returns Rule's representation in form
+// of APIRule
+func (rr *RecordingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "recording",
+		DatasourceType: rr.Type.String(),
+		Name:           rr.Name,
+		Query:          rr.Expr,
+		Labels:         rr.Labels,
+		LastEvaluation: rr.lastExecTime,
+		EvaluationTime: rr.lastExecDuration.Seconds(),
+		Health:         "ok",
+		LastSamples:    rr.lastExecSamples,
 		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", rr.ID()),
-		GroupID:     fmt.Sprintf("%d", rr.GroupID),
-		Name:        rr.Name,
-		Type:        rr.Type.String(),
-		Expression:  rr.Expr,
-		LastError:   lastErr,
-		LastSamples: rr.lastExecSamples,
-		LastExec:    rr.lastExecTime,
-		Labels:      rr.Labels,
+		ID:      fmt.Sprintf("%d", rr.ID()),
+		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
+
+	if rr.lastExecError != nil {
+		r.LastError = rr.lastExecError.Error()
+		r.Health = "err"
+	}
+	return r
 }

app/vmalert/recording_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestRecoridngRule_Exec(t *testing.T) {
+func TestRecordingRule_Exec(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -77,7 +77,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now(), 0)
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -88,7 +88,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecRange(t *testing.T) {
+func TestRecordingRule_ExecRange(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -169,7 +169,48 @@ func TestRecoridngRule_ExecRange(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecNegative(t *testing.T) {
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
+		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+	}
+	rule := &RecordingRule{Name: "job:foo", Labels: map[string]string{
+		"source": "test_limit",
+	}}
+	var err error
+	for _, testCase := range testCases {
+		fq := &fakeQuerier{}
+		fq.add(testMetrics...)
+		rule.q = fq
+		_, err = rule.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRecordingRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
 		"job": "test",
 	}}
@@ -178,7 +219,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO())
+	_, err := rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +234,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO())
+	_, err = rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remoteread/init.go

@@ -10,14 +10,21 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
-		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+		"Remote read is used to restore alerts state."+
+		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
 		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+
+	headers = flag.String("remoteRead.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteRead.url. "+
+		"For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. "+
+		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")
+
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
 	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
-	bearerToken           = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
-	bearerTokenFile       = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
+	bearerToken     = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
@@ -26,7 +33,12 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
-	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
+
+	oauth2ClientID         = flag.String("remoteRead.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteRead.url.")
+	oauth2ClientSecret     = flag.String("remoteRead.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteRead.url.")
+	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
+	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
+	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
 // Init creates a Querier from provided flag values.
@@ -39,10 +51,15 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c), nil
 }

app/vmalert/remotewrite/init.go

@@ -13,11 +13,17 @@ var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+
+	headers = flag.String("remoteWrite.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
+
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
 	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
-	bearerToken           = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
-	bearerTokenFile       = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
+
+	bearerToken     = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -31,7 +37,12 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
-	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
+
+	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url.")
+	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url.")
+	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
+	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
+	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
 // Init creates Client object from given flags.
@@ -46,7 +57,11 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
-	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -19,17 +20,20 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
+)
+
 // Client is an asynchronous HTTP client for writing
 // timeseries via remote write protocol.
 type Client struct {
-	addr              string
-	c                 *http.Client
-	authCfg           *promauth.Config
-	input             chan prompbmarshal.TimeSeries
-	flushInterval     time.Duration
-	maxBatchSize      int
-	maxQueueSize      int
-	disablePathAppend bool
+	addr          string
+	c             *http.Client
+	authCfg       *promauth.Config
+	input         chan prompbmarshal.TimeSeries
+	flushInterval time.Duration
+	maxBatchSize  int
+	maxQueueSize  int
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
@@ -70,8 +74,6 @@ const (
 	defaultWriteTimeout  = 30 * time.Second
 )
 
-const writePath = "/api/v1/write"
-
 // NewClient returns asynchronous client for
 // writing timeseries via remotewrite protocol.
 func NewClient(ctx context.Context, cfg Config) (*Client, error) {
@@ -102,14 +104,13 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
-		addr:              strings.TrimSuffix(cfg.Addr, "/"),
-		authCfg:           cfg.AuthCfg,
-		flushInterval:     cfg.FlushInterval,
-		maxBatchSize:      cfg.MaxBatchSize,
-		maxQueueSize:      cfg.MaxQueueSize,
-		doneCh:            make(chan struct{}),
-		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
-		disablePathAppend: cfg.DisablePathAppend,
+		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		authCfg:       cfg.AuthCfg,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		maxQueueSize:  cfg.MaxQueueSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 	}
 
 	for i := 0; i < cc; i++ {
@@ -225,7 +226,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 	droppedRows.Add(len(wr.Timeseries))
 	droppedBytes.Add(len(b))
-	logger.Errorf("all %d attempts to send request failed - dropping %d timeseries",
+	logger.Errorf("all %d attempts to send request failed - dropping %d time series",
 		attempts, len(wr.Timeseries))
 }
 
@@ -235,13 +236,19 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
+
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+
 	if c.authCfg != nil {
-		if auth := c.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		c.authCfg.SetHeaders(req, true)
 	}
-	if !c.disablePathAppend {
-		req.URL.Path = path.Join(req.URL.Path, writePath)
+	if !*disablePathAppend {
+		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {

app/vmalert/remotewrite/remotewrite_test.go

@@ -80,6 +80,22 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("bad method %q", r.Method))
 		return
 	}
+
+	h := r.Header.Get("Content-Encoding")
+	if h != "snappy" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	}
+
+	h = r.Header.Get("Content-Type")
+	if h != "application/x-protobuf" {
+		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
+	}
+
+	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
+	if h != "0.1.0" {
+		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	}
+
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		rw.err(w, fmt.Errorf("body read err: %w", err))

app/vmalert/replay.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cheggaaa/pb/v3"
+	"github.com/dmitryk-dk/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -29,6 +29,8 @@ var (
 		"Max number of data points expected in one request. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
+		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw *remotewrite.Client) error {
@@ -86,9 +88,16 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 		"\nrequests to make: \t%d"+
 		"\nmax range per request: \t%v\n",
 		g.Name, g.Interval, iterations, step)
+	if g.Limit > 0 {
+		fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
+			g.Limit)
+	}
 	for _, rule := range g.Rules {
 		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
-		bar := pb.StartNew(iterations)
+		var bar *pb.ProgressBar
+		if !*disableProgressBar {
+			bar = pb.StartNew(iterations)
+		}
 		ri.reset()
 		for ri.next() {
 			n, err := replayRule(rule, ri.s, ri.e, rw)
@@ -96,9 +105,13 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 				logger.Fatalf("rule %q: %s", rule, err)
 			}
 			total += n
-			bar.Increment()
+			if bar != nil {
+				bar.Increment()
+			}
+		}
+		if bar != nil {
+			bar.Finish()
 		}
-		bar.Finish()
 		// sleep to let remote storage to flush data on-disk
 		// so chained rules could be calculated correctly
 		time.Sleep(*replayRulesDelay)

app/vmalert/rule.go

@@ -3,8 +3,9 @@ package main
 import (
 	"context"
 	"errors"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Rule represents alerting or recording rule
@@ -14,13 +15,16 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context
-	Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error)
-	// ExecRange executes the rule on the given time range
+	// Exec executes the rule with given context at the given timestamp and limit.
+	// returns an err if number of resulting time series exceeds the limit.
+	Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error)
+	// ExecRange executes the rule on the given time range.
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error
+	// ToAPI converts Rule into APIRule
+	ToAPI() APIRule
 	// Close performs the shutdown procedures for rule
 	// such as metrics unregister
 	Close()

app/vmalert/templates/template.go

@@ -11,25 +11,118 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package notifier
+package templates
 
 import (
 	"errors"
 	"fmt"
+	htmlTpl "html/template"
+	"io/ioutil"
 	"math"
 	"net"
 	"net/url"
+	"path/filepath"
 	"regexp"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	htmlTpl "html/template"
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
+// go template execution fails when it's tree is empty
+const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
+
+var tplMu sync.RWMutex
+
+type textTemplate struct {
+	current     *textTpl.Template
+	replacement *textTpl.Template
+}
+
+var masterTmpl textTemplate
+
+func newTemplate() *textTpl.Template {
+	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(defaultTemplate))
+}
+
+// Load func loads templates from multiple globs specified in pathPatterns and either
+// sets them directly to current template if it's undefined or with overwrite=true
+// or sets replacement templates and adds templates with new names to a current
+func Load(pathPatterns []string, overwrite bool) error {
+	var err error
+	tmpl := newTemplate()
+	for _, tp := range pathPatterns {
+		p, err := filepath.Glob(tp)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
+		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseGlob(tp)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	if len(tmpl.Templates()) > 0 {
+		err := tmpl.Execute(ioutil.Discard, nil)
+		if err != nil {
+			return fmt.Errorf("failed to execute template: %w", err)
+		}
+	}
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.current == nil || overwrite {
+		masterTmpl.replacement = nil
+		masterTmpl.current = newTemplate()
+	} else {
+		masterTmpl.replacement = newTemplate()
+		if err = copyTemplates(tmpl, masterTmpl.replacement, overwrite); err != nil {
+			return err
+		}
+	}
+	return copyTemplates(tmpl, masterTmpl.current, overwrite)
+}
+
+func copyTemplates(from *textTpl.Template, to *textTpl.Template, overwrite bool) error {
+	if from == nil {
+		return nil
+	}
+	if to == nil {
+		to = newTemplate()
+	}
+	tmpl, err := from.Clone()
+	if err != nil {
+		return err
+	}
+	for _, t := range tmpl.Templates() {
+		if to.Lookup(t.Name()) == nil || overwrite {
+			to, err = to.AddParseTree(t.Name(), t.Tree)
+			if err != nil {
+				return fmt.Errorf("failed to add template %q: %w", t.Name(), err)
+			}
+		}
+	}
+	return nil
+}
+
+// Reload func replaces current template with a replacement template
+// which was set by Load with override=false
+func Reload() {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.replacement != nil {
+		masterTmpl.current = masterTmpl.replacement
+		masterTmpl.replacement = nil
+	}
+}
+
 // metric is private copy of datasource.Metric,
 // it is used for templating annotations,
 // Labels as map simplifies templates evaluation.
@@ -59,12 +152,62 @@ func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
 // for templating functions.
 type QueryFn func(query string) ([]datasource.Metric, error)
 
-var tmplFunc textTpl.FuncMap
+// UpdateWithFuncs updates existing or sets a new function map for a template
+func UpdateWithFuncs(funcs textTpl.FuncMap) {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	masterTmpl.current = masterTmpl.current.Funcs(funcs)
+}
 
-// InitTemplateFunc initiates template helper functions
-func InitTemplateFunc(externalURL *url.URL) {
+// GetWithFuncs returns a copy of current template with additional FuncMap
+// provided with funcs argument
+func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	tmpl, err := masterTmpl.current.Clone()
+	if err != nil {
+		return nil, err
+	}
+	return tmpl.Funcs(funcs), nil
+}
+
+// Get returns a copy of a template
+func Get() (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	return masterTmpl.current.Clone()
+}
+
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
+}
+
+// FuncsWithExternalURL returns a function map that depends on externalURL value
+func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
+}
+
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
 	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	tmplFunc = textTpl.FuncMap{
+	return textTpl.FuncMap{
 		/* Strings */
 
 		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
@@ -116,9 +259,13 @@ func InitTemplateFunc(externalURL *url.URL) {
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(v float64) string {
+		"humanize": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				prefix := ""
@@ -129,7 +276,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 					prefix = p
 					v /= 1000
 				}
-				return fmt.Sprintf("%.4g%s", v, prefix)
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -139,13 +286,17 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(v float64) string {
+		"humanize1024": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
@@ -155,16 +306,20 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v /= 1024
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanizeDuration converts given seconds to a human readable duration
-		"humanizeDuration": func(v float64) string {
+		"humanizeDuration": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if v == 0 {
-				return fmt.Sprintf("%.4gs", v)
+				return fmt.Sprintf("%.4gs", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				sign := ""
@@ -178,16 +333,16 @@ func InitTemplateFunc(externalURL *url.URL) {
 				days := int64(v) / 60 / 60 / 24
 				// For days to minutes, we display seconds as an integer.
 				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
 				}
 				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
 				}
 				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
 				}
 				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v)
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -197,33 +352,51 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%ss", v, prefix)
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(v float64) string {
-			return fmt.Sprintf("%.4g%%", v*100)
+		"humanizePercentage": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(v float64) string {
+		"humanizeTimestamp": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
-			return fmt.Sprint(t)
+			return fmt.Sprint(t), nil
 		},
 
 		/* URLs */
 
 		// externalURL returns value of `external.url` flag
 		"externalURL": func() string {
-			return externalURL.String()
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathPrefix returns a Path segment from the URL value in `external.url` flag
 		"pathPrefix": func() string {
-			return externalURL.Path
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathEscape escapes the string so it can be safely placed inside a URL path segment,
@@ -258,7 +431,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// execute "/api/v1/query?query=foo" request and will return
 		// the first value in response.
 		"query": func(q string) ([]metric, error) {
-			// query function supposed to be substituted at funcsWithQuery().
+			// query function supposed to be substituted at FuncsWithQuery().
 			// it is present here only for validation purposes, when there is no
 			// provided datasource.
 			//
@@ -282,6 +455,14 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return m.Labels[label]
 		},
 
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
 		// value returns the value of the given metric.
 		// usually used alongside with `query` template function.
 		"value": func(m metric) float64 {
@@ -307,21 +488,6 @@ func InitTemplateFunc(externalURL *url.URL) {
 	}
 }
 
-func funcsWithQuery(query QueryFn) textTpl.FuncMap {
-	fm := make(textTpl.FuncMap)
-	for k, fn := range tmplFunc {
-		fm[k] = fn
-	}
-	fm["query"] = func(q string) ([]metric, error) {
-		result, err := query(q)
-		if err != nil {
-			return nil, err
-		}
-		return datasourceMetricsToTemplateMetrics(result), nil
-	}
-	return fm
-}
-
 // Time is the number of milliseconds since the epoch
 // (1970-01-01 00:00 UTC) excluding leap seconds.
 type Time int64
@@ -346,3 +512,28 @@ const second = int64(time.Second / minimumTick)
 func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
+
+func toFloat64(v interface{}) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
+}

app/vmalert/templates/template_test.go

@@ -0,0 +1,275 @@
+package templates
+
+import (
+	"strings"
+	"testing"
+	textTpl "text/template"
+)
+
+func mkTemplate(current, replacement interface{}) textTemplate {
+	tmpl := textTemplate{}
+	if current != nil {
+		switch val := current.(type) {
+		case string:
+			tmpl.current = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	if replacement != nil {
+		switch val := replacement.(type) {
+		case string:
+			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	return tmpl
+}
+
+func equalTemplates(tmpls ...*textTpl.Template) bool {
+	var cmp *textTpl.Template
+	for i, tmpl := range tmpls {
+		if i == 0 {
+			cmp = tmpl
+		} else {
+			if cmp == nil || tmpl == nil {
+				if cmp != tmpl {
+					return false
+				}
+				continue
+			}
+			if len(tmpl.Templates()) != len(cmp.Templates()) {
+				return false
+			}
+			for _, t := range tmpl.Templates() {
+				tp := cmp.Lookup(t.Name())
+				if tp == nil {
+					return false
+				}
+				if tp.Root.String() != t.Root.String() {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+func TestTemplates_Load(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		pathPatterns     []string
+		overwrite        bool
+		expectedTemplate textTemplate
+		expErr           string
+	}{
+		{
+			"non existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"non existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`, nil),
+			"",
+		},
+		{
+			"existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				`, `
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`),
+			"",
+		},
+		{
+			"load template with syntax error",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/bad0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			"failed to parse template glob",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			err := Load(tc.pathPatterns, tc.overwrite)
+			if tc.expErr == "" && err != nil {
+				t.Error("happened error that wasn't expected: %w", err)
+			}
+			if tc.expErr != "" && err == nil {
+				t.Error("%+w", err)
+				t.Error("expected error that didn't happened")
+			}
+			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
+				t.Error("%+w", err)
+				t.Error("expected string doesn't exist in error message")
+			}
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}
+
+func TestTemplates_Reload(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		expectedTemplate textTemplate
+	}{
+		{
+			"empty current and replacement templates",
+			mkTemplate(nil, nil),
+			mkTemplate(nil, nil),
+		},
+		{
+			"empty current template only",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"empty replacement template only",
+			mkTemplate(nil, `
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"defined both templates",
+			mkTemplate(`
+				{{- define "test.0" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{- printf "before" -}}
+				{{- end -}}
+			`, `
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`, nil),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			Reload()
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}

app/vmalert/templates/templates/other/nested/bad0-test.tpl

@@ -0,0 +1,3 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL" }}
+{{- end -}}

app/vmalert/templates/templates/other/nested/good0-test.tpl

@@ -0,0 +1,9 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.0" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.3" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}