app/vmselect/vmui: make vmui-update after a54987f671

Also put a link to the function type in docs for every function.
This should simplify understanding of MetricsQL functions for novice users.

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.17
+          go-version: 1.19.1
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.17
+          go-version: 1.19.1
         id: go
       - name: Code checkout
         uses: actions/checkout@master
@@ -29,8 +29,6 @@ jobs:
           make install-errcheck
           make install-golangci-lint
       - name: Build
-        env:
-          GO111MODULE: on
         run: |
           export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
           make check-all
@@ -38,30 +36,8 @@ jobs:
           make test-full
           make test-pure
           make test-full-386
-          make victoria-metrics
-          make victoria-metrics-pure
-          make victoria-metrics-arm
-          make victoria-metrics-arm64
-          make vmutils
-          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=freebsd go build -mod=vendor ./app/vmagent
-          GOOS=freebsd go build -mod=vendor ./app/vmalert
-          GOOS=freebsd go build -mod=vendor ./app/vmbackup
-          GOOS=freebsd go build -mod=vendor ./app/vmrestore
-          GOOS=freebsd go build -mod=vendor ./app/vmctl
-          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=openbsd go build -mod=vendor ./app/vmagent
-          GOOS=openbsd go build -mod=vendor ./app/vmalert
-          GOOS=openbsd go build -mod=vendor ./app/vmbackup
-          GOOS=openbsd go build -mod=vendor ./app/vmrestore
-          GOOS=openbsd go build -mod=vendor ./app/vmctl
-          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
-          GOOS=darwin go build -mod=vendor ./app/vmagent
-          GOOS=darwin go build -mod=vendor ./app/vmalert
-          GOOS=darwin go build -mod=vendor ./app/vmbackup
-          GOOS=darwin go build -mod=vendor ./app/vmrestore
-          GOOS=darwin go build -mod=vendor ./app/vmctl
-          CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
+          make victoria-metrics-crossbuild
+          make vmuitils-crossbuild
       - name: Publish coverage
         uses: codecov/codecov-action@v3
         with:

.gitignore

@@ -19,4 +19,5 @@
 .DS_store
 Gemfile.lock
 /_site
-_site
+_site
+*.tmp

Makefile

@@ -13,6 +13,11 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 
 .PHONY: $(MAKECMDGOALS)
 
+include app/*/Makefile
+include deployment/*/Makefile
+include snap/local/Makefile
+include package/release/Makefile
+
 all: \
 	victoria-metrics-prod \
 	vmagent-prod \
@@ -22,15 +27,10 @@ all: \
 	vmrestore-prod \
 	vmctl-prod
 
-include app/*/Makefile
-include deployment/*/Makefile
-include snap/local/Makefile
-
-
 clean:
 	rm -rf bin/*
 
-publish: \
+publish: docker-scan \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
@@ -64,21 +64,77 @@ vmutils-pure: \
 	vmrestore-pure \
 	vmctl-pure
 
-vmutils-arm64: \
-	vmagent-arm64 \
-	vmalert-arm64 \
-	vmauth-arm64 \
-	vmbackup-arm64 \
-	vmrestore-arm64 \
-	vmctl-arm64
+vmutils-linux-amd64: \
+	vmagent-linux-amd64 \
+	vmalert-linux-amd64 \
+	vmauth-linux-amd64 \
+	vmbackup-linux-amd64 \
+	vmrestore-linux-amd64 \
+	vmctl-linux-amd64
 
-vmutils-arm: \
-	vmagent-arm \
-	vmalert-arm \
-	vmauth-arm \
-	vmbackup-arm \
-	vmrestore-arm \
-	vmctl-arm
+vmutils-linux-arm64: \
+	vmagent-linux-arm64 \
+	vmalert-linux-arm64 \
+	vmauth-linux-arm64 \
+	vmbackup-linux-arm64 \
+	vmrestore-linux-arm64 \
+	vmctl-linux-arm64
+
+vmutils-linux-arm: \
+	vmagent-linux-arm \
+	vmalert-linux-arm \
+	vmauth-linux-arm \
+	vmbackup-linux-arm \
+	vmrestore-linux-arm \
+	vmctl-linux-arm
+
+vmutils-linux-386: \
+	vmagent-linux-386 \
+	vmalert-linux-386 \
+	vmauth-linux-386 \
+	vmbackup-linux-386 \
+	vmrestore-linux-386 \
+	vmctl-linux-386
+
+vmutils-linux-ppc64le: \
+	vmagent-linux-ppc64le \
+	vmalert-linux-ppc64le \
+	vmauth-linux-ppc64le \
+	vmbackup-linux-ppc64le \
+	vmrestore-linux-ppc64le \
+	vmctl-linux-ppc64le
+
+vmutils-darwin-amd64: \
+	vmagent-darwin-amd64 \
+	vmalert-darwin-amd64 \
+	vmauth-darwin-amd64 \
+	vmbackup-darwin-amd64 \
+	vmrestore-darwin-amd64 \
+	vmctl-darwin-amd64
+
+vmutils-darwin-arm64: \
+	vmagent-darwin-arm64 \
+	vmalert-darwin-arm64 \
+	vmauth-darwin-arm64 \
+	vmbackup-darwin-arm64 \
+	vmrestore-darwin-arm64 \
+	vmctl-darwin-arm64
+
+vmutils-freebsd-amd64: \
+	vmagent-freebsd-amd64 \
+	vmalert-freebsd-amd64 \
+	vmauth-freebsd-amd64 \
+	vmbackup-freebsd-amd64 \
+	vmrestore-freebsd-amd64 \
+	vmctl-freebsd-amd64
+
+vmutils-openbsd-amd64: \
+	vmagent-openbsd-amd64 \
+	vmalert-openbsd-amd64 \
+	vmauth-openbsd-amd64 \
+	vmbackup-openbsd-amd64 \
+	vmrestore-openbsd-amd64 \
+	vmctl-openbsd-amd64
 
 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
@@ -86,6 +142,28 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
+victoria-metrics-crossbuild: \
+	victoria-metrics-linux-amd64 \
+	victoria-metrics-linux-arm64 \
+	victoria-metrics-linux-arm \
+	victoria-metrics-linux-386 \
+	victoria-metrics-linux-ppc64le \
+	victoria-metrics-darwin-amd64 \
+	victoria-metrics-darwin-arm64 \
+	victoria-metrics-freebsd-amd64 \
+	victoria-metrics-openbsd-amd64
+
+vmutils-crossbuild: \
+	vmutils-linux-amd64 \
+	vmutils-linux-arm64 \
+	vmutils-linux-arm \
+	vmutils-linux-386 \
+	vmutils-linux-ppc64le \
+	vmutils-darwin-amd64 \
+	vmutils-darwin-arm64 \
+	vmutils-freebsd-amd64 \
+	vmutils-openbsd-amd64 \
+	vmutils-windows-amd64
 
 publish-release:
 	git checkout $(TAG) && $(MAKE) release publish && \
@@ -98,86 +176,110 @@ release: \
 	release-vmutils
 
 release-victoria-metrics: \
-	release-victoria-metrics-amd64 \
-	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64 \
+	release-victoria-metrics-linux-amd64 \
+	release-victoria-metrics-linux-arm \
+	release-victoria-metrics-linux-arm64 \
 	release-victoria-metrics-darwin-amd64 \
-	release-victoria-metrics-darwin-arm64
+	release-victoria-metrics-darwin-arm64 \
+	release-victoria-metrics-freebsd-amd64 \
+	release-victoria-metrics-openbsd-amd64
 
-release-victoria-metrics-amd64:
-	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-arm:
-	OSARCH=arm $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-arm64:
-	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
+release-victoria-metrics-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
 release-vmutils: \
-	release-vmutils-amd64 \
-	release-vmutils-arm64 \
-	release-vmutils-arm \
-	release-vmutils-darwin-amd64  \
+	release-vmutils-linux-amd64 \
+	release-vmutils-linux-arm64 \
+	release-vmutils-linux-arm \
+	release-vmutils-darwin-amd64 \
 	release-vmutils-darwin-arm64 \
+	release-vmutils-freebsd-amd64 \
+	release-vmutils-openbsd-amd64 \
 	release-vmutils-windows-amd64
 
-release-vmutils-amd64:
-	OSARCH=amd64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
-release-vmutils-arm64:
-	OSARCH=arm64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch
 
-release-vmutils-arm:
-	OSARCH=arm $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic
+	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
 
-release-vmutils-generic: \
-	vmagent-$(OSARCH)-prod \
-	vmalert-$(OSARCH)-prod \
-	vmauth-$(OSARCH)-prod \
-	vmbackup-$(OSARCH)-prod \
-	vmrestore-$(OSARCH)-prod \
-	vmctl-$(OSARCH)-prod
+release-vmutils-goos-goarch: \
+	vmagent-$(GOOS)-$(GOARCH)-prod \
+	vmalert-$(GOOS)-$(GOARCH)-prod \
+	vmauth-$(GOOS)-$(GOARCH)-prod \
+	vmbackup-$(GOOS)-$(GOARCH)-prod \
+	vmrestore-$(GOOS)-$(GOARCH)-prod \
+	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-$(GOOS)-$(GOARCH)-prod \
+		vmalert-$(GOOS)-$(GOARCH)-prod \
+		vmauth-$(GOOS)-$(GOARCH)-prod \
+		vmbackup-$(GOOS)-$(GOARCH)-prod \
+		vmrestore-$(GOOS)-$(GOARCH)-prod \
+		vmctl-$(GOOS)-$(GOARCH)-prod
 
-release-vmutils-windows-generic: \
+release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
@@ -194,112 +296,122 @@ release-vmutils-windows-generic: \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-windows-$(GOARCH)-prod.exe \
+		vmalert-windows-$(GOARCH)-prod.exe \
+		vmauth-windows-$(GOARCH)-prod.exe \
+		vmctl-windows-$(GOARCH)-prod.exe
 
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	GO111MODULE=on gofmt -l -w -s ./lib
-	GO111MODULE=on gofmt -l -w -s ./app
+	gofmt -l -w -s ./lib
+	gofmt -l -w -s ./app
 
 vet:
-	GO111MODULE=on go vet -mod=vendor ./lib/...
-	GO111MODULE=on go vet -mod=vendor ./app/...
+	go vet ./lib/...
+	go vet ./app/...
 
 lint: install-golint
 	golint lib/...
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get golang.org/x/lint/golint
+	which golint || go install golang.org/x/lint/golint@latest
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
+	errcheck -exclude=errcheck_excludes.txt ./app/...
 
 install-errcheck:
-	which errcheck || GO111MODULE=off go get github.com/kisielk/errcheck
+	which errcheck || go install github.com/kisielk/errcheck@latest
 
-check-all: fmt vet lint errcheck golangci-lint
+check-all: fmt vet lint errcheck golangci-lint govulncheck
 
 test:
-	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+	go test -bench=. ./lib/...
+	go test -bench=. ./app/...
 
 benchmark-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	GO111MODULE=on go get -u -d ./lib/...
-	GO111MODULE=on go get -u -d ./app/...
-	GO111MODULE=on go mod tidy
-	GO111MODULE=on go mod vendor
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.19
+	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
-app-local-with-goarch:
-	GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-goos-goarch:
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
-app-local-windows-with-goarch:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-windows-goarch:
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || GO111MODULE=off go get github.com/valyala/quicktemplate/qtc
+	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
 
 
 golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.45.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
+
+govulncheck: install-govulncheck
+	govulncheck ./...
+
+install-govulncheck:
+	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
 install-wwhrd:
-	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
+	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
 copy-docs:
-	echo "---\nsort: ${ORDER}\n---\n" > ${DST}
+	echo '' > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "---\nsort: ${ORDER}\n---\n" > ${DST}; \
+	fi
 	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
 
 # Copies docs for all components and adds the order tag.
+# For ORDER=0 it adds no order tag.
+# Images starting with <img src="docs/ are replaced with <img src="
 # Cluster docs are supposed to be ordered as 9th.
-# For The rest of docs is ordered manually.t
+# The rest of docs is ordered manually.
 docs-sync:
-	cp README.md docs/README.md
+	SRC=README.md DST=docs/README.md ORDER=0 $(MAKE) copy-docs
 	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md ORDER=1 $(MAKE) copy-docs
 	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
 	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+|---------|--------------------|
+| 1.81.x  | :white_check_mark: |
+| 1.80.x  | :x:                |
+| 1.79.x  | :white_check_mark: |
+| < 1.78  | :x:                |
+
+## Reporting a Vulnerability
+
+Please report any security issues to security@victoriametrics.com

app/victoria-metrics/Makefile

@@ -12,20 +12,20 @@ victoria-metrics-prod:
 victoria-metrics-pure-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure
 
-victoria-metrics-amd64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-amd64
+victoria-metrics-linux-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-amd64
 
-victoria-metrics-arm-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+victoria-metrics-linux-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm
 
-victoria-metrics-arm64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+victoria-metrics-linux-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm64
 
-victoria-metrics-ppc64le-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+victoria-metrics-linux-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-ppc64le
 
-victoria-metrics-386-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+victoria-metrics-linux-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-386
 
 victoria-metrics-darwin-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ victoria-metrics-darwin-amd64-prod:
 victoria-metrics-darwin-arm64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64
 
+victoria-metrics-freebsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-freebsd-amd64
+
+victoria-metrics-openbsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 
@@ -64,60 +70,68 @@ run-victoria-metrics:
 	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
 	$(MAKE) run-via-docker
 
-victoria-metrics-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-amd64 ./app/victoria-metrics
+victoria-metrics-linux-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-victoria-metrics-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+victoria-metrics-linux-arm:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-victoria-metrics-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
+victoria-metrics-linux-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-victoria-metrics-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
+victoria-metrics-linux-ppc64le:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-victoria-metrics-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+victoria-metrics-linux-386:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-freebsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-openbsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 
 ### Packaging as DEB - amd64
-victoria-metrics-package-deb: victoria-metrics-prod
+victoria-metrics-package-deb-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_deb.sh amd64
 
 ### Packaging as DEB - arm64
-victoria-metrics-package-deb-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-deb-arm: victoria-metrics-linux-arm-prod
+	./package/package_deb.sh arm
+
+### Packaging as DEB - arm64
+victoria-metrics-package-deb-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_deb.sh arm64
 
 ### Packaging as DEB - all
-victoria-metrics-package-deb-all: \
-        victoria-metrics-package-deb \
+victoria-metrics-package-deb: \
+        victoria-metrics-package-deb-amd64 \
+        victoria-metrics-package-deb-arm \
         victoria-metrics-package-deb-arm64
 
 ### Packaging as RPM - amd64
-victoria-metrics-package-rpm: victoria-metrics-prod
+victoria-metrics-package-rpm-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_rpm.sh amd64
 
 ### Packaging as RPM - arm64
-victoria-metrics-package-rpm-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-rpm-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_rpm.sh arm64
 
 ### Packaging as RPM - all
-victoria-metrics-package-rpm-all: \
-        victoria-metrics-package-rpm \
+victoria-metrics-package-rpm: \
+        victoria-metrics-package-rpm-amd64 \
         victoria-metrics-package-rpm-arm64
 
 ### Packaging as both DEB and RPM - all
-victoria-metrics-package-deb-rpm-all: \
+victoria-metrics-package-deb-rpm: \
         victoria-metrics-package-deb \
-        victoria-metrics-package-deb-arm64 \
-        victoria-metrics-package-rpm \
-        victoria-metrics-package-rpm-arm64
-
-### Packaging as snap
-victoria-metrics-package-snap:
-	which snapcraft || snap install snapcraft
-	which multipass || snap install multipass
-	snapcraft
-
+        victoria-metrics-package-rpm

app/victoria-metrics/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -37,6 +38,7 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -92,7 +94,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"vmui", "Web UI"},
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},

app/victoria-metrics/main_test.go

@@ -6,7 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -139,7 +138,7 @@ func setUp() {
 		if err != nil {
 			return false
 		}
-		resp.Body.Close()
+		_ = resp.Body.Close()
 		return resp.StatusCode == 200
 	}
 	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
@@ -308,7 +307,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 		if filepath.Ext(path) != ".json" {
 			return nil
 		}
-		b, err := ioutil.ReadFile(path)
+		b, err := os.ReadFile(path)
 		s.noError(err)
 		item := test{}
 		s.noError(json.Unmarshal(b, &item))
@@ -338,7 +337,9 @@ func tcpWrite(t *testing.T, address string, data string) {
 	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
 	s.noError(err)
-	defer conn.Close()
+	defer func() {
+		_ = conn.Close()
+	}()
 	n, err := conn.Write([]byte(data))
 	s.noError(err)
 	s.equalInt(n, len(data))
@@ -349,7 +350,9 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
 	for dec := json.NewDecoder(resp.Body); dec.More(); {
@@ -364,7 +367,9 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }

app/victoria-metrics/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
 ARG TARGETARCH
-COPY victoria-metrics-${TARGETARCH}-prod ./victoria-metrics-prod
+COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod

app/victoria-metrics/self_scraper.go

@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -60,7 +60,7 @@ func selfScraper(scrapeInterval time.Duration) {
 			currentTimestamp = currentTime.UnixNano() / 1e6
 		}
 		bb.Reset()
-		httpserver.WritePrometheusMetrics(&bb)
+		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
 		rows.Unmarshal(s)
@@ -85,7 +85,9 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr.Timestamp = currentTimestamp
 			mr.Value = r.Value
 		}
-		vmstorage.AddRows(mrs)
+		if err := vmstorage.AddRows(mrs); err != nil {
+			logger.Errorf("cannot store self-scraped metrics: %s", err)
+		}
 	}
 }
 

app/vmagent/Makefile

@@ -12,20 +12,20 @@ vmagent-prod:
 vmagent-pure-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-pure
 
-vmagent-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+vmagent-linux-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-amd64
 
-vmagent-arm-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+vmagent-linux-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm
 
-vmagent-arm64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+vmagent-linux-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm64
 
-vmagent-ppc64le-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+vmagent-linux-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-ppc64le
 
-vmagent-386-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-386
+vmagent-linux-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
 
 vmagent-darwin-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmagent-darwin-amd64-prod:
 vmagent-darwin-arm64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
 
+vmagent-freebsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-freebsd-amd64
+
+vmagent-openbsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-openbsd-amd64
+
 vmagent-windows-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
 
@@ -67,26 +73,35 @@ run-vmagent:
 	APP_NAME=vmagent \
 	$(MAKE) run-via-docker
 
-vmagent-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-amd64:
+	APP_NAME=vmagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-vmagent-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-vmagent-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-vmagent-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmagent-local-with-goarch
+vmagent-linux-ppc64le:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmagent-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-386:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
-vmagent-local-with-goarch:
-	APP_NAME=vmagent $(MAKE) app-local-with-goarch
+vmagent-darwin-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-darwin-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmagent-freebsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-openbsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-goarch
 
 vmagent-pure:
 	APP_NAME=vmagent $(MAKE) app-local-pure
-
-vmagent-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch

app/vmagent/csvimport/request_handler.go

@@ -67,7 +67,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -1,9 +1,7 @@
 package datadog
 
 import (
-	"fmt"
 	"net/http"
-	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
@@ -59,12 +57,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 			Value: ss.Host,
 		})
 		for _, tag := range ss.Tags {
-			n := strings.IndexByte(tag, ':')
-			if n < 0 {
-				return fmt.Errorf("cannot find ':' in tag %q", tag)
-			}
-			name := tag[:n]
-			value := tag[n+1:]
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -89,7 +82,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/graphite/request_handler.go

@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -134,7 +134,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"flag"
 	"fmt"
 	"io"
@@ -23,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -35,6 +37,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -43,7 +46,7 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
@@ -63,6 +66,12 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -71,6 +80,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -104,10 +114,12 @@ func main() {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, httpInsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, httpInsertHandler)
 	}
 
 	promscrape.Init(remotewrite.Push)
@@ -149,6 +161,40 @@ func main() {
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
+func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
+	if !remotewrite.MultitenancyEnabled() {
+		return func(req *http.Request) error {
+			path := strings.Replace(req.URL.Path, "//", "/", -1)
+			if path != "/api/put" {
+				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
+			}
+			return opentsdbhttp.InsertHandler(nil, req)
+		}
+	}
+	return func(req *http.Request) error {
+		path := strings.Replace(req.URL.Path, "//", "/", -1)
+		at, err := getAuthTokenFromPath(path)
+		if err != nil {
+			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
+		}
+		return opentsdbhttp.InsertHandler(at, req)
+	}
+}
+
+func getAuthTokenFromPath(path string) (*auth.Token, error) {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
+	}
+	if p.Prefix != "insert" {
+		return nil, fmt.Errorf(`unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+	}
+	if p.Suffix != "opentsdb/api/put" {
+		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
+	}
+	return auth.NewToken(p.AuthToken)
+}
+
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
 		if r.Method != "GET" {
@@ -159,7 +205,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -170,8 +217,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
 	switch path {
-	case "/api/v1/write":
+	case "/prometheus/api/v1/write", "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -180,7 +232,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import":
+	case "/prometheus/api/v1/import", "/api/v1/import":
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
@@ -189,7 +241,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/csv":
+	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
@@ -198,7 +250,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/prometheus":
+	case "/prometheus/api/v1/import/prometheus", "/api/v1/import/prometheus":
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(nil, r); err != nil {
 			prometheusimportErrors.Inc()
@@ -207,7 +259,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/native":
+	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
@@ -216,7 +268,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/write", "/api/v2/write":
+	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
@@ -225,7 +277,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/query":
+	case "/influx/query", "/query":
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
@@ -254,16 +306,31 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/intake/":
+	case "/datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
-	case "/targets":
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "/prometheus/targets", "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
-	case "/target_response":
+	case "/prometheus/service-discovery", "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
+	case "/prometheus/api/v1/targets", "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
+		return true
+	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
 			promscrapeTargetResponseErrors.Inc()
@@ -271,7 +338,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
-	case "/config":
+	case "/prometheus/config", "/config":
 		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
 			err := &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
@@ -284,13 +351,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
-	case "/api/v1/targets":
-		promscrapeAPIV1TargetsRequests.Inc()
+	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		state := r.FormValue("state")
-		promscrape.WriteAPIV1Targets(w, state)
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
-	case "/-/reload":
+	case "/prometheus/-/reload", "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -305,11 +382,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			w.Write([]byte("OK"))
 		}
 		return true
+	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+		if remotewrite.MultitenancyEnabled() {
+			return processMultitenantRequest(w, r, path)
+		}
+		return false
 	}
-	if remotewrite.MultitenancyEnabled() {
-		return processMultitenantRequest(w, r, path)
-	}
-	return false
 }
 
 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
@@ -327,6 +409,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
+	if strings.HasPrefix(p.Suffix, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
+	}
 	switch p.Suffix {
 	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -410,11 +497,16 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/intake/":
+	case "datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -447,15 +539,18 @@ var (
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
 
-	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )

app/vmagent/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-COPY vmagent-${TARGETARCH}-prod ./vmagent-prod
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -87,6 +87,6 @@ func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/opentsdb/request_handler.go

@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
@@ -19,19 +20,19 @@ var (
 
 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -65,7 +66,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -82,7 +82,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/promremotewrite/request_handler.go

@@ -81,7 +81,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -3,7 +3,7 @@ package remotewrite
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -20,58 +20,50 @@ import (
 )
 
 var (
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to -remoteWrite.url. "+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to -remoteWrite.url")
-	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. "+
-		"Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsCAFile = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
-		"By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to -remoteWrite.url. "+
-		"By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
 
-	basicAuthUsername = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for -remoteWrite.url. "+
-		"The file is re-read every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for -remoteWrite.url. "+
-		"The token is re-read from the file every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
-	oauth2ClientID = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecret = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2TokenURL = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
+	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+		"The token is re-read from the file every second")
 
-	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for -remoteWrite.url. "+
-		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+
+	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
+	awsEC2Endpoint = flagutil.NewArrayString("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSTSEndpoint = flagutil.NewArrayString("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRegion      = flagutil.NewArrayString("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN     = flagutil.NewArrayString("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey   = flagutil.NewArrayString("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService     = flagutil.NewArrayString("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"Defaults to \"aps\"")
+	awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
 )
 
 type client struct {
@@ -151,7 +143,7 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
-	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remote_write_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
@@ -164,6 +156,9 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
+	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(*queues)
+	})
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -181,6 +176,11 @@ func (c *client) MustStop() {
 }
 
 func getAuthConfig(argIdx int) (*promauth.Config, error) {
+	headersValue := headers.GetOptionalArg(argIdx)
+	var hdrs []string
+	if headersValue != "" {
+		hdrs = strings.Split(headersValue, "^^")
+	}
 	username := basicAuthUsername.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
 	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
@@ -217,7 +217,15 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
 
-	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		OAuth2:          oauth2Cfg,
+		TLSConfig:       tlsCfg,
+		Headers:         hdrs,
+	}
+	authCfg, err := opts.NewConfig()
 	if err != nil {
 		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
@@ -228,11 +236,14 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	if !awsUseSigv4.GetOptionalArg(argIdx) {
 		return nil, nil
 	}
+	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
 	region := awsRegion.GetOptionalArg(argIdx)
 	roleARN := awsRoleARN.GetOptionalArg(argIdx)
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey)
+	service := awsService.GetOptionalArg(argIdx)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
 	if err != nil {
 		return nil, err
 	}
@@ -298,16 +309,14 @@ again:
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
+	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
 	h.Set("Content-Encoding", "snappy")
 	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	if ah := c.authCfg.GetAuthHeader(); ah != "" {
-		req.Header.Set("Authorization", ah)
-	}
 	if c.awsCfg != nil {
-		if err := c.awsCfg.SignRequest(req, "aps", sigv4Hash); err != nil {
+		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
 			// there is no need in retry, request will be rejected by client.Do and retried by code below
 			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
@@ -342,15 +351,14 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 		if err != nil {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
 				"failed to read response body: %s",
 				len(block), c.sanitizedURL, statusCode, err)
 		} else {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
 				len(block), c.sanitizedURL, statusCode, string(body))
 		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
@@ -367,7 +375,7 @@ again:
 	if retryDuration > time.Minute {
 		retryDuration = time.Minute
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
 		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
@@ -387,6 +395,8 @@ again:
 	goto again
 }
 
+var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
 type rateLimiter struct {
 	perSecondLimit int64
 

app/vmagent/remotewrite/pendingseries_test.go

@@ -0,0 +1,62 @@
+package remotewrite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+func TestPushWriteRequest(t *testing.T) {
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
+			testPushWriteRequest(t, rowsCount)
+		})
+	}
+}
+
+func testPushWriteRequest(t *testing.T, rowsCount int) {
+	wr := newTestWriteRequest(rowsCount, 10)
+	pushBlockLen := 0
+	pushBlock := func(block []byte) {
+		if pushBlockLen > 0 {
+			panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+		}
+		pushBlockLen = len(block)
+	}
+	pushWriteRequest(wr, pushBlock)
+	b := prompbmarshal.MarshalWriteRequest(nil, wr)
+	zb := snappy.Encode(nil, b)
+	maxPushBlockLen := len(zb)
+	minPushBlockLen := maxPushBlockLen / 2
+	if pushBlockLen < minPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be at least %d bytes", pushBlockLen, minPushBlockLen)
+	}
+	if pushBlockLen > maxPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be smaller or equal to %d bytes", pushBlockLen, maxPushBlockLen)
+	}
+}
+
+func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
+	var wr prompbmarshal.WriteRequest
+	for i := 0; i < seriesCount; i++ {
+		var labels []prompbmarshal.Label
+		for j := 0; j < labelsCount; j++ {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  fmt.Sprintf("label_%d_%d", i, j),
+				Value: fmt.Sprintf("value_%d_%d", i, j),
+			})
+		}
+		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
+			Labels: labels,
+			Samples: []prompbmarshal.Sample{
+				{
+					Value:     float64(i),
+					Timestamp: 1000 * int64(i),
+				},
+			},
+		})
+	}
+	return &wr
+}

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -0,0 +1,36 @@
+package remotewrite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+	"github.com/klauspost/compress/s2"
+)
+
+func BenchmarkCompressWriteRequestSnappy(b *testing.B) {
+	b.Run("snappy", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, snappy.Encode)
+	})
+	b.Run("s2", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, s2.EncodeSnappy)
+	})
+}
+
+func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []byte) []byte) {
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
+			wr := newTestWriteRequest(rowsCount, 10)
+			data := prompbmarshal.MarshalWriteRequest(nil, wr)
+			b.ReportAllocs()
+			b.SetBytes(int64(rowsCount))
+			b.RunParallel(func(pb *testing.PB) {
+				var zb []byte
+				for pb.Next() {
+					zb = compressFunc(zb[:cap(zb)], data)
+				}
+			})
+		})
+	}
+}

app/vmagent/remotewrite/relabel.go

@@ -13,18 +13,22 @@ import (
 )
 
 var (
-	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
 		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
 		"The path can point either to local file or to http url")
 	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
+
+	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
+		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
+		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
 )
 
 var labelsGlobal []prompbmarshal.Label
@@ -107,6 +111,18 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 				labels = append(labels, *extraLabel)
 			}
 		}
+		if *usePromCompatibleNaming {
+			// Replace unsupported Prometheus chars in label names and metric names with underscores.
+			tmpLabels := labels[labelsLen:]
+			for j := range tmpLabels {
+				label := &tmpLabels[j]
+				if label.Name == "__name__" {
+					label.Value = promrelabel.SanitizeName(label.Value)
+				} else {
+					label.Name = promrelabel.SanitizeName(label.Name)
+				}
+			}
+		}
 		labels = pcs.Apply(labels, labelsLen, true)
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.

app/vmagent/remotewrite/remotewrite.go

@@ -22,14 +22,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
-	xxhash "github.com/cespare/xxhash/v2"
+	"github.com/cespare/xxhash/v2"
 )
 
 var (
-	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
-	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
@@ -38,7 +38,7 @@ var (
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
-	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
 		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
@@ -234,15 +234,11 @@ func Stop() {
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// Note that wr may be modified by Push due to relabeling and rounding.
-func Push(wr *prompbmarshal.WriteRequest) {
-	PushWithAuthToken(nil, wr)
-}
-
-// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
+// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
+// If at isn't nil, the the data is pushed to the configured `-remoteWrite.multitenantURL`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
-func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
 		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
 		at = defaultAuthToken
@@ -252,7 +248,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rwctxs = rwctxsDefault
 	} else {
 		if len(*remoteWriteMultitenantURLs) == 0 {
-			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
 		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
@@ -274,6 +270,8 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
+	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
@@ -298,9 +296,10 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			tss = nil
 		}
 		if rctx != nil {
-			tssBlockLen := len(tssBlock)
+			rowsCountBeforeRelabel := getRowsCount(tssBlock)
 			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
-			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+			rowsCountAfterRelabel := getRowsCount(tssBlock)
+			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
@@ -414,7 +413,10 @@ func labelsToString(labels []prompbmarshal.Label) string {
 	return string(b)
 }
 
-var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+var (
+	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
+	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+)
 
 type remoteWriteCtx struct {
 	idx        int
@@ -423,7 +425,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
-	relabelMetricsDropped *metrics.Counter
+	rowsPushedAfterRelabel *metrics.Counter
+	rowsDroppedByRelabel   *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -433,7 +436,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
 	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
@@ -467,7 +471,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 
@@ -483,7 +488,8 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
-	rwctx.relabelMetricsDropped = nil
+	rwctx.rowsPushedAfterRelabel = nil
+	rwctx.rowsDroppedByRelabel = nil
 }
 
 func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
@@ -499,12 +505,15 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
 		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
-		tssLen := len(tss)
+		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, nil, pcs)
-		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+		rowsCountAfterRelabel := getRowsCount(tss)
+		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 	pss[idx].Push(tss)
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
@@ -519,3 +528,11 @@ var tssRelabelPool = &sync.Pool{
 		return &a
 	},
 }
+
+func getRowsCount(tss []prompbmarshal.TimeSeries) int {
+	rowsCount := 0
+	for _, ts := range tss {
+		rowsCount += len(ts.Samples)
+	}
+	return rowsCount
+}

app/vmagent/vmimport/request_handler.go

@@ -88,7 +88,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmalert/Makefile

@@ -12,20 +12,20 @@ vmalert-prod:
 vmalert-pure-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-pure
 
-vmalert-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+vmalert-linux-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-amd64
 
-vmalert-arm-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+vmalert-linux-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm
 
-vmalert-arm64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+vmalert-linux-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm64
 
-vmalert-ppc64le-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+vmalert-linux-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-ppc64le
 
-vmalert-386-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-386
+vmalert-linux-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
 
 vmalert-darwin-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmalert-darwin-amd64-prod:
 vmalert-darwin-arm64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
 
+vmalert-freebsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-freebsd-amd64
+
+vmalert-openbsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-openbsd-amd64
+
 vmalert-windows-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
 
@@ -62,13 +68,14 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-notifier.url=http://localhost:9093 \
 		-notifier.url=http://127.0.0.1:9093 \
@@ -77,7 +84,7 @@ run-vmalert: vmalert
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
 		-evaluationInterval=3s \
-		-rule.configCheckInterval=10s
+		-configCheckInterval=10s
 
 run-vmalert-sd: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
@@ -87,7 +94,7 @@ run-vmalert-sd: vmalert
 		-configCheckInterval=10s
 
 replay-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
@@ -95,26 +102,35 @@ replay-vmalert: vmalert
 		-replay.timeFrom=2021-05-11T07:21:43Z \
 		-replay.timeTo=2021-05-29T18:40:43Z
 
-vmalert-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-amd64:
+	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-vmalert-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
-vmalert-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
-vmalert-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmalert-local-with-goarch
+vmalert-linux-ppc64le:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmalert-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-386:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
-vmalert-local-with-goarch:
-	APP_NAME=vmalert $(MAKE) app-local-with-goarch
+vmalert-darwin-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-darwin-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-freebsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-openbsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-goarch
 
 vmalert-pure:
 	APP_NAME=vmalert $(MAKE) app-local-pure
-
-vmalert-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch

app/vmalert/README.md

@@ -2,7 +2,7 @@
 
 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured `-datasource.url`. For sending alerting notifications
+rules against configured `-datasource.url` compatible with Prometheus HTTP API. For sending alerting notifications
 vmalert relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag.
 Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
 protocol and require `-remoteWrite.url` to be configured.
@@ -13,29 +13,30 @@ implementation and aims to be compatible with its syntax.
 
 * Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
 * VictoriaMetrics [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html)
- support and expressions validation;
+  support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
- support;
+  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-aplha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
 * Keeps the alerts [state on restarts](#alerts-state-on-restarts);
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
-* Lightweight without extra dependencies.
+* Lightweight and without extra dependencies.
+* Supports [reusable templates](#reusable-templates) for annotations.
 
 ## Limitations
 
 * `vmalert` execute queries against remote datasource which has reliability risks because of the network.
-It is recommended to configure alerts thresholds and rules expressions with the understanding that network
-requests may fail;
+  It is recommended to configure alerts thresholds and rules expressions with the understanding that network
+  requests may fail;
 * by default, rules execution is sequential within one group, but persistence of execution results to remote
-storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
-recording rule is reused in the next one;
+  storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
+  recording rule is reused in the next one;
 
 ## QuickStart
 
 To build `vmalert` from sources:
 
-```bash
+```console
 git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
@@ -46,28 +47,29 @@ The build binary will be placed in `VictoriaMetrics/bin` folder.
 To start using `vmalert` you will need the following things:
 
 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable MetricsQL endpoint to run queries against;
+* datasource address - reachable endpoint with [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) support for running queries against;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
-[config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
+  aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
+  [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-  compatible storage to persist rules and alerts state info;
+  compatible storage to persist rules and alerts state info. To persist results to multiple destinations use vmagent
+  configured with multiple remote writes as a proxy;
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
 
-```bash
+```console
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
-    -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
+    -datasource.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
     -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
-    -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
+    -remoteRead.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource to restore alerts state from
     -external.label=cluster=east-1 \         # External label to be applied for each rule
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
-Note there's a separate `remoteRead.url` to allow writing results of
+Note there's a separate `remoteWrite.url` to allow writing results of
 alerting/recording rules into a different storage than the initial data that's
 queried. This allows using `vmalert` to aggregate data from a short-term,
 high-frequency, high-cardinality storage into a long-term storage with
@@ -100,6 +102,10 @@ name: <string>
 # How often rules in the group are evaluated.
 [ interval: <duration> | default = -evaluationInterval flag ]
 
+# Limit the number of alerts an alerting rule and series a recording
+# rule can produce. 0 is no limit.
+[ limit: <int> | default = 0 ]
+
 # How many rules execute at once within a group. Increasing concurrency may speed
 # up round execution speed.
 [ concurrency: <integer> | default = 1 ]
@@ -108,24 +114,27 @@ name: <string>
 # By default "prometheus" type is used.
 [ type: <string> ]
 
-# Warning: DEPRECATED
-# Please use `params` instead:
-#   params:
-#     extra_label: ["job=nodeexporter", "env=prod"]
-extra_filter_labels:
-  [ <labelname>: <labelvalue> ... ]
-
 # Optional list of HTTP URL parameters
 # applied for all rules requests within a group
 # For example:
 #  params:
 #    nocache: ["1"]                # disable caching for vmselect
 #    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
-#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests
 # see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
 params:
   [ <string>: [<string>, ...]]
 
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all rules requests within a group
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+# Headers set via this param have priority over headers set via `-datasource.headers` flag.
+headers:
+  [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
@@ -140,18 +149,18 @@ rules:
 ### Rules
 
 Every rule contains `expr` field for [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
-or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. Vmalert will execute the configured
+or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. `vmalert` will execute the configured
 expression and then act according to the Rule type.
 
 There are two types of Rules:
 
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allow defining alert conditions via `expr` field and to send notifications to
-[Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
+  Alerting rules allow defining alert conditions via `expr` field and to send notifications to
+  [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allow defining `expr` which result will be then backfilled to configured
-`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
-expensive expressions and save their result as a new set of time series.
+  Recording rules allow defining `expr` which result will be then backfilled to configured
+  `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
+  expensive expressions and save their result as a new set of time series.
 
 `vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.
@@ -165,7 +174,7 @@ The syntax for alerting rule is the following:
 alert: <string>
 
 # The expression to evaluate. The expression language depends on the type value.
-# By default PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
+# By default, PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
@@ -175,6 +184,13 @@ expr: <string>
 # as firing once they return.
 [ for: <duration> | default = 0s ]
 
+# Whether to print debug information into logs.
+# Information includes alerts state changes and requests sent to the datasource.
+# Please note, that if rule's query params contain sensitive
+# information - it will be printed to logs.
+# Is applicable to alerting rules only.
+[ debug: <bool> | default = false ]
+
 # Labels to add or overwrite for each alert.
 labels:
   [ <labelname>: <tmpl_string> ]
@@ -184,10 +200,69 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ```
 
-It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations
-to format data, iterate over it or execute expressions.
+#### Templating
+
+It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations to format data, iterate over
+or execute expressions.
+The following variables are available in templating:
+
+| Variable                           | Description                                                                                               | Example                                                                                                                                                                |
+|------------------------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| $value or .Value                   | The current alert's value. Avoid using value in labels, it may cause unexpected issues.                   | {% raw %}Number of connections is {{ $value }}{% endraw %}                                                                                                             |
+| $activeAt or .ActiveAt             | The moment of [time](https://pkg.go.dev/time) when alert became active (`pending` or `firing`).           | {% raw %}http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}{% endraw %} |
+| $labels or .Labels                 | The list of labels of the current alert. Use as ".Labels.<label_name>".                                   | {% raw %}Too high number of connections for {{ .Labels.instance }}{% endraw %}                                                                                         |
+| $alertID or .AlertID               | The current alert's ID generated by vmalert.                                                              | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %}                                                                                   |
+| $groupID or .GroupID               | The current alert's group ID generated by vmalert.                                                        | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %}                                                                                   |
+| $expr or .Expr                     | Alert's expression. Can be used for generating links to Grafana or other systems.                         | {% raw %}/api/v1/query?query={{ $expr&#124;quotesEscape&#124;queryEscape }}{% endraw %}                                                                                |
+| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag.                                        | {% raw %}Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }}){% endraw %}                                                                          |
+| $externalURL or .ExternalURL       | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}Visit {{ $externalURL }} for more details{% endraw %}                                                                                                         |
+
 Additionally, `vmalert` provides some extra templating functions
-listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go).
+listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go)
+and [reusable templates](#reusable-templates).
+
+#### Reusable templates
+
+Like in Alertmanager you can define [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates)
+to share same templates across annotations. Just define the templates in a file and
+set the path via `-rule.templates` flag.
+
+For example, template `grafana.filter` can be defined as following:
+
+{% raw  %}
+```
+{{ define "grafana.filter" -}}
+  {{- $labels := .arg0 -}}
+  {{- range $name, $label := . -}}
+    {{- if (ne $name "arg0") -}}
+      {{- ( or (index $labels $label) "All" ) | printf "&var-%s=%s" $label -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+```
+{% endraw %}
+
+And then used in annotations:
+
+{% raw  %}
+```yaml
+groups:
+  - name: AlertGroupName
+    rules:
+      - alert: AlertName
+        expr: any_metric > 100
+        for: 30s
+        labels:
+          alertname: 'Any metric is too high'
+          severity: 'warning'
+        annotations:
+          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1{{ template "grafana.filter" (args .CommonLabels "account_id" "any_label") }}'
+```
+{% endraw %}
+
+The `-rule.templates` flag supports wildcards so multiple files with templates can be loaded.
+The content of `-rule.templates` can be also [hot reloaded](#hot-config-reload).
+
 
 #### Recording rules
 
@@ -198,7 +273,7 @@ The syntax for recording rules is following:
 record: <string>
 
 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If group.type="graphite", then the expression
+# By default, MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
@@ -215,11 +290,11 @@ For recording rules to work `-remoteWrite.url` must be specified.
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
 
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
-into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and maybe queried from VM just as any other time series.
-The state is stored to the configured address on every rule evaluation.
+  into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
+  These are regular time series and maybe queried from VM just as any other time series.
+  The state is stored to the configured address on every rule evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
-from configured address by querying time series with name `ALERTS_FOR_STATE`.
+  from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
 Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
@@ -256,6 +331,10 @@ groups:
     # Rules for accountID=456, projectID=789
 ```
 
+The results of alerting and recording rules contain `vm_account_id` and `vm_project_id` labels
+if `-clusterMode` is enabled. These labels can be used during [templating](https://docs.victoriametrics.com/vmalert.html#templating),
+and help to identify to which account or project the triggered alert or produced recording belongs.
+
 If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
 contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
 `vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
@@ -275,7 +354,7 @@ for different scenarios.
 Please note, not all flags in examples are required:
 
 * `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
-you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+  you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
 * `-notifier.url` is optional and is needed only if you have alerting rules.
 
 #### Single-node VictoriaMetrics
@@ -341,6 +420,11 @@ Alertmanagers.
 
 To avoid recording rules results and alerts state duplication in VictoriaMetrics server
 don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert's `evaluation_interval`.
+If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
+command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
+But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
+which would affect deduplication logic and result into "jumping" datapoints.
 
 Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
 all `vmalert`s are having the same config.
@@ -353,6 +437,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics)
 
 #### Downsampling and aggregation via vmalert
 
+`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries
+via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`.
+This ability allows to aggregate data. For example, the following rule will calculate the average value for
+metric `http_requests` on the `5m` interval:
+
+```yaml
+  - record: http_requests:avg5m
+    expr: avg_over_time(http_requests[5m])
+```
+
+Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m`
+to the configured `-remoteWrite.url`.
+
+`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval`
+or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series.
+This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`:
+
+```yaml
+groups:
+  - name: my_group
+    interval: 5m
+    rules:
+    - record: http_requests:avg5m
+      expr: avg_over_time(http_requests[5m])
+```
+
+Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows
+reading data from one data source and backfilling results to another. This helps to build a system
+for aggregating and downsampling the data.
+
 The following example shows how to build a topology where `vmalert` will process data from one cluster
 and write results into another. Such clusters may be called as "hot" (low retention,
 high-speed disks, used for operative monitoring) and "cold" (long term retention,
@@ -364,7 +478,7 @@ or reducing resolution) and push results to "cold" cluster.
 
 ```
 ./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
-    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recording rules expressions
     -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus    # vminsert addr to persist recording rules results
 ```
 
@@ -376,6 +490,21 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec
 
 See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
 
+#### Multiple remote writes
+
+For persisting recording or alerting rule results `vmalert` requires `-remoteWrite.url` to be set.
+But this flag supports only one destination. To persist rule results to multiple destinations
+we recommend using [vmagent](https://docs.victoriametrics.com/vmagent.html) as fan-out proxy:
+
+<img alt="vmalert multiple remote write destinations" src="vmalert_multiple_rw.png">
+
+In this topology, `vmalert` is configured to persist rule results to `vmagent`. And `vmagent`
+is configured to fan-out received data to two or more destinations.
+Using `vmagent` as a proxy provides additional benefits such as
+[data persisting when storage is unreachable](https://docs.victoriametrics.com/vmagent.html#replication-and-high-availability),
+or time series modification via [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling).
+
+
 ### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -383,11 +512,21 @@ See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
 * `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
-* `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status"` - get alert status by ID.
-Used as alert source in AlertManager.
+* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in JSON format.
+  Used as alert source in AlertManager.
+* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in web UI.
+* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in web UI.
 * `http://<vmalert-addr>/metrics` - application metrics.
 * `http://<vmalert-addr>/-/reload` - hot configuration reload.
 
+`vmalert` web UI can be accessed from [single-node version of VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html)
+and from [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+This may be used for better integraion with Grafana unified alerting system. See the following docs for details:
+
+* [How to query vmalert from single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmalert)
+* [How to query vmalert from VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmalert)
+
+
 ## Graphite
 
 vmalert sends requests to `<-datasource.url>/render?format=json` during evaluation of alerting and recording rules
@@ -410,7 +549,7 @@ To run vmalert in `replay` mode:
 
 ```
 ./bin/vmalert -rule=path/to/your.rules \        # path to files with rules you usually use with vmalert
-    -datasource.url=http://localhost:8428 \     # PromQL/MetricsQL compatible datasource
+    -datasource.url=http://localhost:8428 \     # Prometheus HTTP API compatible datasource
     -remoteWrite.url=http://localhost:8428 \    # remote write compatible storage to persist results
     -replay.timeFrom=2021-05-11T07:21:43Z \     # time from begin replay
     -replay.timeTo=2021-05-29T18:40:43Z         # time to finish replay
@@ -446,7 +585,7 @@ max range per request:  8h20m0s
 
 In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
 executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
-to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
+to [/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) endpoint
 of the configured `-datasource.url`. Returned data is then processed according to the rule type and
 backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
@@ -473,24 +612,25 @@ Execute the query against storage which was used for `-remoteWrite.url` during t
 There are following non-required `replay` flags:
 
 * `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
-In two words, it affects the max time range for every `/query_range` request. The higher the value,
-the fewer requests will be issued during `replay`.
+  In two words, it affects the max time range for every `/query_range` request. The higher the value,
+  the fewer requests will be issued during `replay`.
 * `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
-per rule before giving up.
+  per rule before giving up.
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
-(rules which depend on each other) rules. It is expected, that remote storage will be able to persist
-previously accepted data during the delay, so data will be available for the subsequent queries.
-Keep it equal or bigger than `-remoteWrite.flushInterval`.
-* `replay.disableProgressBar` - whether to disable progress bar which shows progress work.
-Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
-It could break logs parsing by external system and generate additional load on it.
+  (rules which depend on each other) rules. It is expected, that remote storage will be able to persist
+  previously accepted data during the delay, so data will be available for the subsequent queries.
+  Keep it equal or bigger than `-remoteWrite.flushInterval`.
+* `-replay.disableProgressBar` - whether to disable progress bar which shows progress work.
+  Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
+  It could break logs parsing by external system and generate additional load on it.
 
-See full description for these flags in `./vmalert --help`.
+See full description for these flags in `./vmalert -help`.
 
 ### Limitations
 
 * Graphite engine isn't supported yet;
 * `query` template function is disabled for performance reasons (might be changed in future);
+* `limit` group's param has no effect during replay (might be changed in future);
 
 ## Monitoring
 
@@ -502,6 +642,95 @@ Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/1495
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.
 
+## Troubleshooting
+
+vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed,
+the data is already present in configured `-datasource.url`:
+
+<img alt="vmalert expected evaluation" src="vmalert_ts_normal.gif">
+
+Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations
+may get empty response from datasource and produce empty recording rules or reset alerts state:
+
+<img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
+
+_By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
+This behavior is controlled by `-search.latencyOffset` command-line flag on vmselect. Usually, this results into
+a 30s shift for recording rules results.
+Note that too small value passed to `-search.latencyOffset` may lead to incomplete query results._
+
+Try the following recommendations in such cases:
+
+* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics
+are delivered to the datasource;
+* If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback`
+command-line flag to add a time shift for evaluations;
+* If time intervals between datapoints in datasource are irregular - try changing vmalert's `-datasource.queryStep`
+command-line flag to specify how far search query can lookback for the recent datapoint. By default, this value
+is equal to group's evaluation interval.
+
+Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that
+alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only
+after multiple consecutive evaluations, and at each evaluation their expression must be true. If at least one evaluation
+becomes false, then alert's state resets to the initial state.
+
+If `-remoteWrite.url` command-line flag is configured, vmalert will persist alert's state in form of time series
+`ALERTS` and `ALERTS_FOR_STATE` to the specified destination. Such time series can be then queried via
+[vmui](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) or Grafana to track how alerts state
+changed in time.
+
+vmalert also stores last N state updates for each rule. To check updates, click on `Details` link next to rule's name
+on `/vmalert/groups` page and check the `Last updates` section:
+
+<img alt="vmalert state" src="vmalert_state.png">
+
+Rows in the section represent ordered rule evaluations and their results. The column `curl` contains an example of
+HTTP request sent by vmalert to the `-datasource.url` during evaluation. If specific state shows that there were
+no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
+moment when rule was evaluated.
+
+vmalert also alows configuring more detailed logging for specific rule. Just set `debug: true` in rule's configuration
+and vmalert will start printing additional log messages:
+```terminal
+2022-09-15T13:35:41.155Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 samples (elapsed: 5.896041ms)
+2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
+2022-09-15T13:35:56.178Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: query returned 1 samples (elapsed: 28.368208ms)
+2022-09-15T13:35:56.178Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29&step=15s&time=1663248945"
+2022-09-15T13:35:56.179Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} created in state PENDING
+...
+2022-09-15T13:36:56.153Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:36:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} PENDING => FIRING: 1m0s since becoming active at 2022-09-15 15:35:56.126006 +0200 CEST m=+39.384575417
+```
+
+
+## Profiling
+
+`vmalert` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
+
+```console
+curl http://0.0.0.0:8880/debug/pprof/heap > mem.pprof
+```
+
+</div>
+
+* CPU profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
+
+```console
+curl http://0.0.0.0:8880/debug/pprof/profile > cpu.pprof
+```
+
+</div>
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+It is safe sharing the collected profiles from security point of view, since they do not contain sensitive information.
+
 ## Configuration
 
 ### Flags
@@ -510,10 +739,10 @@ Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.
 
 The shortlist of configuration flags is the following:
-
+{% raw  %}
 ```
   -clusterMode
-     If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy . This flag is available only in enterprise version of VictoriaMetrics
   -configCheckInterval duration
      Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
   -datasource.appendTypePrefix
@@ -530,26 +759,30 @@ The shortlist of configuration flags is the following:
      Optional path to bearer token file to use for -datasource.url.
   -datasource.disableKeepAlive
      Whether to disable long-lived connections to the datasource. If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.
+  -datasource.headers string
+     Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'
   -datasource.lookback duration
      Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
   -datasource.maxIdleConnections int
      Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
   -datasource.oauth2.clientID string
-     Optional OAuth2 clientID to use for -datasource.url. 
+     Optional OAuth2 clientID to use for -datasource.url.
   -datasource.oauth2.clientSecret string
      Optional OAuth2 clientSecret to use for -datasource.url.
   -datasource.oauth2.clientSecretFile string
-     Optional OAuth2 clientSecretFile to use for -datasource.url. 
+     Optional OAuth2 clientSecretFile to use for -datasource.url.
   -datasource.oauth2.scopes string
      Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
   -datasource.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -datasource.url.
   -datasource.queryStep duration
-     queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+     How far a value can fallback to when evaluating queries. For example, if -datasource.queryStep=15s then param "step" with value "15s" will be added to every query. If set to 0, rule's evaluation interval will be used instead. (default 5m0s)
   -datasource.queryTimeAlignment
      Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
      Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+  -datasource.showURL
+     Whether to show -datasource.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -datasource.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
@@ -561,14 +794,14 @@ The shortlist of configuration flags is the following:
   -datasource.tlsServerName string
      Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-     VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+     Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.
   -defaultTenant.graphite string
-     Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy .This flag is available only in enterprise version of VictoriaMetrics
   -defaultTenant.prometheus string
-     Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy . This flag is available only in enterprise version of VictoriaMetrics
   -disableAlertgroupLabel
      Whether to disable adding group's Name as label to generated alerts and time series.
-  -dryRun -rule
+  -dryRun
      Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
@@ -577,17 +810,19 @@ The shortlist of configuration flags is the following:
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in enterprise version of VictoriaMetrics
   -evaluationInterval duration
      How often to evaluate the rules (default 1m0s)
   -external.alert.source string
-     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-     eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
+     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used
+     If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
   -external.label array
      Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
      Supports an array of values separated by comma or specified via multiple flags.
   -external.url string
      External URL is used as alert's source for sent alerts to the notifier
+  -flagsAuthKey string
+     Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
@@ -628,7 +863,7 @@ The shortlist of configuration flags is the following:
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
      Optional basic auth password for -notifier.url
      Supports an array of values separated by comma or specified via multiple flags.
@@ -682,17 +917,25 @@ The shortlist of configuration flags is the following:
      Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
      Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.consul.waitTime duration
      Wait time used by Consul service discovery. Default value is used if not set
   -promscrape.consulSDCheckInterval duration
-     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs for details (default 30s)
   -promscrape.discovery.concurrency int
      The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
   -promscrape.discovery.concurrentWaitTime duration
      The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
   -promscrape.dnsSDCheckInterval duration
-     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
+     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#dns_sd_configs for details (default 30s)
+  -pushmetrics.extraLabel array
+     Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -pushmetrics.interval duration
+     Interval for pushing metrics to -pushmetrics.url (default 10s)
+  -pushmetrics.url array
+     Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default metrics exposed at /metrics page aren't pushed to any remote storage
+     Supports an array of values separated by comma or specified via multiple flags.
   -remoteRead.basicAuth.password string
      Optional basic auth password for -remoteRead.url
   -remoteRead.basicAuth.passwordFile string
@@ -704,7 +947,9 @@ The shortlist of configuration flags is the following:
   -remoteRead.bearerTokenFile string
      Optional path to bearer token file to use for -remoteRead.url.
   -remoteRead.disablePathAppend
-     Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.
+     Whether to disable automatic appending of '/api/v1/query' path to the configured -datasource.url and -remoteRead.url
+  -remoteRead.headers string
+     Optional HTTP headers to send with each request to the corresponding -remoteRead.url. For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'
   -remoteRead.ignoreRestoreErrors
      Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
   -remoteRead.lookback duration
@@ -718,7 +963,9 @@ The shortlist of configuration flags is the following:
   -remoteRead.oauth2.scopes string
      Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
   -remoteRead.oauth2.tokenUrl string
-     Optional OAuth2 tokenURL to use for -remoteRead.url. 
+     Optional OAuth2 tokenURL to use for -remoteRead.url.
+  -remoteRead.showURL
+     Whether to show -remoteRead.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteRead.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
@@ -730,7 +977,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
      Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-     Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
+     Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
   -remoteWrite.basicAuth.password string
      Optional basic auth password for -remoteWrite.url
   -remoteWrite.basicAuth.passwordFile string
@@ -747,6 +994,8 @@ The shortlist of configuration flags is the following:
      Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
   -remoteWrite.flushInterval duration
      Defines interval of flushes to remote write endpoint (default 5s)
+  -remoteWrite.headers string
+     Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'
   -remoteWrite.maxBatchSize int
      Defines defines max number of timeseries to be flushed at once (default 1000)
   -remoteWrite.maxQueueSize int
@@ -761,6 +1010,8 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
   -remoteWrite.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -notifier.url.
+  -remoteWrite.showURL
+     Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteWrite.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
@@ -772,7 +1023,9 @@ The shortlist of configuration flags is the following:
   -remoteWrite.tlsServerName string
      Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
+  -replay.disableProgressBar
+     Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int
      Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
@@ -798,6 +1051,14 @@ The shortlist of configuration flags is the following:
      Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
   -rule.resendDelay duration
      Minimum amount of time to wait before resending an alert to notifier
+  -rule.templates array
+     Path or glob pattern to location with go template definitions
+      for rules annotations templating. Flag can be specified multiple times.
+     Examples:
+      -rule.templates="/path/to/file". Path to a single file with go templates
+      -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+     absolute path to all .tpl files in root.
+     Supports an array of values separated by comma or specified via multiple flags.
   -rule.validateExpressions
      Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
@@ -811,9 +1072,12 @@ The shortlist of configuration flags is the following:
      Supports an array of values separated by comma or specified via multiple flags.
   -tlsKeyFile string
      Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+  -tlsMinVersion string
+     Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
   -version
      Show VictoriaMetrics version
 ```
+{% endraw  %}
 
 ### Hot config reload
 
@@ -822,7 +1086,7 @@ The shortlist of configuration flags is the following:
 * send SIGHUP signal to `vmalert` process;
 * send GET request to `/-/reload` endpoint;
 * configure `-configCheckInterval` flag for periodic reload
-on config change.
+  on config change.
 
 ### URL params
 
@@ -860,7 +1124,7 @@ and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configurati
 For example:
 
 ```
-static_configs: 
+static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
@@ -869,7 +1133,7 @@ consul_sd_configs:
   - server: localhost:8500
     services:
       - alertmanager
-      
+
 dns_sd_configs:
   - names:
       - my.domain.com
@@ -897,7 +1161,7 @@ is the following:
 # password and password_file are mutually exclusive.
 basic_auth:
   [ username: <string> ]
-  [ password: <secret> ]
+  [ password: <string> ]
   [ password_file: <string> ]
 
 # Optional `Authorization` header configuration.
@@ -916,10 +1180,41 @@ authorization:
 tls_config:
   [ <tls_config> ]
 
+# Configures Bearer authentication token via string
+bearer_token: <string>
+# or by passing path to the file with token.
+bearer_token_file: <string>
+
+# Configures OAuth 2.0 authentication
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2
+oauth2:
+  [ <oauth2_config> ]
+  
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all requests to notifiers
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+headers:
+  [ <string>, ...]
+
 # List of labeled statically configured Notifiers.
+#
+# Each list of targets may be additionally instructed with
+# authorization params. Target's authorization params will
+# inherit params from global authorization params if there
+# are no conflicts.
 static_configs:
-  targets:
-    [ - '<host>' ]
+  [ - targets: ]
+      [ - '<host>' ]
+      [ oauth2 ]
+      [ basic_auth ]
+      [ authorization ]
+      [ tls_config ]
+      [ bearer_token ]
+      [ bearer_token_file ]
+      [ headers ]
 
 # List of Consul service discovery configurations.
 # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
@@ -959,9 +1254,28 @@ It is recommended using
 
 * `vmalert` is located in `vmutils-*` archives there.
 
+### Docker image
+
+You can build `vmalert` docker image from source and push it to your own docker repository.
+Run the following commands from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics):
+```console
+make package-vmalert
+docker tag victoria-metrics/vmalert:version my-repo:my-version-name
+docker push my-repo:my-version-name
+```
+
+To run the built image in `victoria-metrics-k8s-stack` or `VMAlert` CR object apply the following config change:
+```yaml
+kind: VMAlert
+spec:
+  image:
+    repository: my-repo
+    tag: my-version-name
+```
+
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.1.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
@@ -977,12 +1291,12 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
-2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.1.
+2. Run `make vmalert-linux-arm` or `make vmalert-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmalert-linux-arm` or `vmalert-linux-arm64` binary respectively and puts it into the `bin` folder.
 
 ### Production ARM build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmalert-arm-prod` or `make vmalert-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmalert-arm-prod` or `vmalert-arm64-prod` binary respectively and puts it into the `bin` folder.
+2. Run `make vmalert-linux-arm-prod` or `make vmalert-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmalert-linux-arm-prod` or `vmalert-linux-arm64-prod` binary respectively and puts it into the `bin` folder.

app/vmalert/alerting.go

@@ -6,12 +6,14 @@ import (
 	"hash/fnv"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -19,7 +21,7 @@ import (
 
 // AlertingRule is basic alert entity
 type AlertingRule struct {
-	Type         datasource.Type
+	Type         config.Type
 	RuleID       uint64
 	Name         string
 	Expr         string
@@ -29,24 +31,17 @@ type AlertingRule struct {
 	GroupID      uint64
 	GroupName    string
 	EvalInterval time.Duration
+	Debug        bool
 
 	q datasource.Querier
 
-	// guard status fields
-	mu sync.RWMutex
+	alertsMu sync.RWMutex
 	// stores list of active alerts
 	alerts map[uint64]*notifier.Alert
-	// stores last moment of time Exec was called
-	lastExecTime time.Time
-	// stores the duration of the last Exec call
-	lastExecDuration time.Duration
-	// stores last error that happened in Exec func
-	// resets on every successful Exec
-	// may be used as Health state
-	lastExecError error
-	// stores the number of samples returned during
-	// the last evaluation
-	lastExecSamples int
+
+	// state stores recent state changes
+	// during evaluations
+	state *ruleState
 
 	metrics *alertingRuleMetrics
 }
@@ -70,20 +65,24 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupID:      group.ID(),
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
+		Debug:        cfg.Debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
+			Debug:              cfg.Debug,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
+		state:   newRuleState(),
 		metrics: &alertingRuleMetrics{},
 	}
 
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StatePending {
@@ -94,8 +93,8 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		})
 	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StateFiring {
@@ -106,18 +105,16 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		})
 	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
-			if ar.lastExecError == nil {
+			e := ar.state.getLast()
+			if e.err == nil {
 				return 0
 			}
 			return 1
 		})
 	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
-			return float64(ar.lastExecSamples)
+			e := ar.state.getLast()
+			return float64(e.samples)
 		})
 	return ar
 }
@@ -141,28 +138,58 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
+	if !ar.Debug {
+		return
+	}
+	prefix := fmt.Sprintf("DEBUG rule %q:%q (%d) at %v: ",
+		ar.GroupName, ar.Name, ar.RuleID, at.Format(time.RFC3339))
+
+	if a != nil {
+		labelKeys := make([]string, len(a.Labels))
+		var i int
+		for k := range a.Labels {
+			labelKeys[i] = k
+			i++
+		}
+		sort.Strings(labelKeys)
+		labels := make([]string, len(labelKeys))
+		for i, l := range labelKeys {
+			labels[i] = fmt.Sprintf("%s=%q", l, a.Labels[l])
+		}
+		labelsStr := strings.Join(labels, ",")
+		prefix += fmt.Sprintf("alert %d {%s} ", a.ID, labelsStr)
+	}
+	msg := fmt.Sprintf(format, args...)
+	logger.Infof("%s", prefix+msg)
+}
+
 type labelSet struct {
-	// origin labels from series
-	// used for templating
+	// origin labels extracted from received time series
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, origin labels from time series preferred.
+	// used for templating annotations
 	origin map[string]string
-	// processed labels with additional data
-	// used as Alert labels
+	// processed labels includes origin labels
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, extra labels are preferred.
+	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
 	processed map[string]string
 }
 
 // toLabels converts labels from given Metric
 // to labelSet which contains original and processed labels.
-func (ar *AlertingRule) toLabels(m datasource.Metric, qFn notifier.QueryFn) (*labelSet, error) {
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
 	ls := &labelSet{
-		origin:    make(map[string]string, len(m.Labels)),
+		origin:    make(map[string]string),
 		processed: make(map[string]string),
 	}
 	for _, l := range m.Labels {
+		ls.origin[l.Name] = l.Value
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
 			continue
 		}
-		ls.origin[l.Name] = l.Value
 		ls.processed[l.Name] = l.Value
 	}
 
@@ -176,14 +203,23 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn notifier.QueryFn) (*la
 	}
 	for k, v := range extraLabels {
 		ls.processed[k] = v
+		if _, ok := ls.origin[k]; !ok {
+			ls.origin[k] = v
+		}
 	}
 
 	// set additional labels to identify group and rule name
 	if ar.Name != "" {
 		ls.processed[alertNameLabel] = ar.Name
+		if _, ok := ls.origin[alertNameLabel]; !ok {
+			ls.origin[alertNameLabel] = ar.Name
+		}
 	}
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
 		ls.processed[alertGroupNameLabel] = ar.GroupName
+		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
+			ls.origin[alertGroupNameLabel] = ar.GroupName
+		}
 	}
 	return ls, nil
 }
@@ -239,41 +275,57 @@ const resolvedRetention = 15 * time.Minute
 
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	start := time.Now()
-	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
-	ar.mu.Lock()
-	defer ar.mu.Unlock()
+	qMetrics, req, err := ar.q.Query(ctx, ar.Expr, ts)
+	curState := ruleStateEntry{
+		time:     start,
+		at:       ts,
+		duration: time.Since(start),
+		samples:  len(qMetrics),
+		err:      err,
+		req:      req,
+	}
+
+	defer func() {
+		ar.state.add(curState)
+	}()
+
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
 
-	ar.lastExecTime = start
-	ar.lastExecDuration = time.Since(start)
-	ar.lastExecError = err
-	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
+	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.samples, curState.duration)
+
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
 		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
+			ar.logDebugf(ts, a, "deleted as inactive")
 			delete(ar.alerts, h)
 		}
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res, err
+	}
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
 		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
+			curState.err = fmt.Errorf("failed to expand labels: %s", err)
+			return nil, curState.err
 		}
 		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
 			// conflicting with the metric labels
-			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
-			return nil, ar.lastExecError
+			curState.err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, curState.err
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
@@ -283,30 +335,28 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 				// back to notifier.StatePending
 				a.State = notifier.StatePending
 				a.ActiveAt = ts
+				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
-			if a.Value != m.Values[0] {
-				// update Value field with latest value
-				a.Value = m.Values[0]
-				// and re-exec template since Value can be used
-				// in annotations
-				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
-				if err != nil {
-					return nil, err
-				}
+			a.Value = m.Values[0]
+			// re-exec template since Value or query can be used in annotations
+			a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+			if err != nil {
+				return nil, err
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
+		a, err := ar.newAlert(m, ls, start, qFn)
 		if err != nil {
-			ar.lastExecError = err
-			return nil, fmt.Errorf("failed to create alert: %w", err)
+			curState.err = fmt.Errorf("failed to create alert: %w", err)
+			return nil, curState.err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
 		a.ActiveAt = ts
 		ar.alerts[h] = a
+		ar.logDebugf(ts, a, "created in state PENDING")
 	}
-
+	var numActivePending int
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
@@ -315,20 +365,29 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 				// alert was in Pending state - it is not
 				// active anymore
 				delete(ar.alerts, h)
+				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
 			}
 			if a.State == notifier.StateFiring {
 				a.State = notifier.StateInactive
 				a.ResolvedAt = ts
+				ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
 			}
 			continue
 		}
-		if a.State == notifier.StatePending && time.Since(a.ActiveAt) >= ar.For {
+		numActivePending++
+		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
+			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
 		}
 	}
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+		return nil, curState.err
+	}
 	return ar.toTimeSeries(ts.Unix()), nil
 }
 
@@ -382,7 +441,7 @@ func hash(labels map[string]string) uint64 {
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
 	var err error
 	if ls == nil {
 		ls, err = ar.toLabels(m, qFn)
@@ -404,8 +463,8 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 
 // AlertAPI generates APIAlert object from alert by its id(hash)
 func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
-	ar.mu.RLock()
-	defer ar.mu.RUnlock()
+	ar.alertsMu.RLock()
+	defer ar.alertsMu.RUnlock()
 	a, ok := ar.alerts[id]
 	if !ok {
 		return nil
@@ -413,9 +472,10 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// ToAPI returns Rule representation in form
-// of APIRule
+// ToAPI returns Rule representation in form of APIRule
+// Isn't thread-safe. Call must be protected by AlertingRule mutex.
 func (ar *AlertingRule) ToAPI() APIRule {
+	lastState := ar.state.getLast()
 	r := APIRule{
 		Type:           "alerting",
 		DatasourceType: ar.Type.String(),
@@ -424,19 +484,20 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		Duration:       ar.For.Seconds(),
 		Labels:         ar.Labels,
 		Annotations:    ar.Annotations,
-		LastEvaluation: ar.lastExecTime,
-		EvaluationTime: ar.lastExecDuration.Seconds(),
+		LastEvaluation: lastState.time,
+		EvaluationTime: lastState.duration.Seconds(),
 		Health:         "ok",
 		State:          "inactive",
 		Alerts:         ar.AlertsToAPI(),
-		LastSamples:    ar.lastExecSamples,
+		LastSamples:    lastState.samples,
+		Updates:        ar.state.getAll(),
 
 		// encode as strings to avoid rounding in JSON
 		ID:      fmt.Sprintf("%d", ar.ID()),
 		GroupID: fmt.Sprintf("%d", ar.GroupID),
 	}
-	if ar.lastExecError != nil {
-		r.LastError = ar.lastExecError.Error()
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
 		r.Health = "err"
 	}
 	// satisfy APIRule.State logic
@@ -456,14 +517,14 @@ func (ar *AlertingRule) ToAPI() APIRule {
 // AlertsToAPI generates list of APIAlert objects from existing alerts
 func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
-	ar.mu.RLock()
+	ar.alertsMu.RLock()
 	for _, a := range ar.alerts {
 		if a.State == notifier.StateInactive {
 			continue
 		}
 		alerts = append(alerts, ar.newAlertAPI(*a))
 	}
-	ar.mu.RUnlock()
+	ar.alertsMu.RUnlock()
 	return alerts
 }
 
@@ -546,7 +607,10 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	}
 
 	ts := time.Now()
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res, err
+	}
 
 	// account for external labels in filter
 	var labelsFilter string
@@ -554,12 +618,9 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}
 
-	// Get the last data point in range via MetricsQL `last_over_time`.
-	// We don't use plain PromQL since Prometheus doesn't support
-	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr, ts)
+	qMetrics, _, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return err
 	}

app/vmalert/alerting_test.go

@@ -304,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -624,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO(), time.Now())
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -640,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -649,6 +649,50 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	}
 }
 
+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
 func TestAlertingRule_Template(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
@@ -656,14 +700,26 @@ func TestAlertingRule_Template(t *testing.T) {
 		expAlerts map[uint64]*notifier.Alert
 	}{
 		{
-			newTestRuleWithLabels("common", "region", "east"),
+			&AlertingRule{
+				Name: "common",
+				Labels: map[string]string{
+					"region": "east",
+				},
+				Annotations: map[string]string{
+					"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
+			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "instance", "foo"),
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "foo"`,
+					},
 					Labels: map[string]string{
 						alertNameLabel: "common",
 						"region":       "east",
@@ -671,7 +727,9 @@ func TestAlertingRule_Template(t *testing.T) {
 					},
 				},
 				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "bar"`,
+					},
 					Labels: map[string]string{
 						alertNameLabel: "common",
 						"region":       "east",
@@ -687,14 +745,15 @@ func TestAlertingRule_Template(t *testing.T) {
 					"instance": "{{ $labels.instance }}",
 				},
 				Annotations: map[string]string{
-					"summary":     `Too high connection number for "{{ $labels.instance }}"`,
+					"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
 					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "instance", "foo", alertNameLabel, "override"),
-				metricWithValueAndLabels(t, 10, "instance", "bar", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
@@ -703,7 +762,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "foo",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "foo"`,
+						"summary":     `first: Too high connection number for "foo"`,
 						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
@@ -713,7 +772,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "bar",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "bar"`,
+						"summary":     `second: Too high connection number for "bar"`,
 						"description": `override: It is 10 connections for "bar"`,
 					},
 				},
@@ -730,6 +789,7 @@ func TestAlertingRule_Template(t *testing.T) {
 					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1,
@@ -761,7 +821,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -871,5 +931,11 @@ func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 }
 
 func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
-	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor, EvalInterval: waitFor}
+	return &AlertingRule{
+		Name:         name,
+		For:          waitFor,
+		EvalInterval: waitFor,
+		alerts:       make(map[uint64]*notifier.Alert),
+		state:        newRuleState(),
+	}
 }

app/vmalert/config/config.go

@@ -4,16 +4,14 @@ import (
 	"crypto/md5"
 	"fmt"
 	"hash/fnv"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -23,17 +21,13 @@ import (
 // Group contains list of Rules grouped into
 // entity with one name and evaluation interval
 type Group struct {
-	Type        datasource.Type `yaml:"type,omitempty"`
+	Type        Type `yaml:"type,omitempty"`
 	File        string
 	Name        string              `yaml:"name"`
 	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
-	// ExtraFilterLabels is a list label filters applied to every rule
-	// request withing a group. Is compatible only with VM datasources.
-	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
-	// DEPRECATED: use Params field instead
-	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
 	Labels map[string]string `yaml:"labels"`
@@ -42,6 +36,8 @@ type Group struct {
 	Checksum string
 	// Optional HTTP URL parameters added to each rule request
 	Params url.Values `yaml:"params"`
+	// Headers contains optional HTTP headers added to each rule request
+	Headers []Header `yaml:"headers,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -59,23 +55,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	// change default value to prometheus datasource.
 	if g.Type.Get() == "" {
-		g.Type.Set(datasource.NewPrometheusType())
-	}
-
-	// backward compatibility with deprecated `ExtraFilterLabels` param
-	if len(g.ExtraFilterLabels) > 0 {
-		if g.Params == nil {
-			g.Params = url.Values{}
-		}
-		// Sort extraFilters for consistent order for query args across runs.
-		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
-		for k, v := range g.ExtraFilterLabels {
-			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
-		}
-		sort.Strings(extraFilters)
-		for _, extraFilter := range extraFilters {
-			g.Params.Add("extra_label", extraFilter)
-		}
+		g.Type.Set(NewPrometheusType())
 	}
 
 	h := md5.New()
@@ -85,7 +65,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // Validate check for internal Group or Rule configuration errors
-func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
+func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
@@ -97,7 +77,7 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			ruleName = r.Alert
 		}
 		if _, ok := uniqueRules[r.ID]; ok {
-			return fmt.Errorf("rule %q duplicate", ruleName)
+			return fmt.Errorf("%q is a duplicate within the group %q", r.String(), g.Name)
 		}
 		uniqueRules[r.ID] = struct{}{}
 		if err := r.Validate(); err != nil {
@@ -111,11 +91,11 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
-		if validateAnnotations {
-			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+		if validateTplFn != nil {
+			if err := validateTplFn(r.Annotations); err != nil {
 				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
 			}
-			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+			if err := validateTplFn(r.Labels); err != nil {
 				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
@@ -133,6 +113,7 @@ type Rule struct {
 	For         *promutils.Duration `yaml:"for,omitempty"`
 	Labels      map[string]string   `yaml:"labels,omitempty"`
 	Annotations map[string]string   `yaml:"annotations,omitempty"`
+	Debug       bool                `yaml:"debug,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -156,6 +137,32 @@ func (r *Rule) Name() string {
 	return r.Alert
 }
 
+// String implements Stringer interface
+func (r *Rule) String() string {
+	ruleType := "recording"
+	if r.Alert != "" {
+		ruleType = "alerting"
+	}
+	b := strings.Builder{}
+	b.WriteString(fmt.Sprintf("%s rule %q", ruleType, r.Name()))
+	b.WriteString(fmt.Sprintf("; expr: %q", r.Expr))
+
+	kv := sortMap(r.Labels)
+	for i := range kv {
+		if i == 0 {
+			b.WriteString("; labels:")
+		}
+		b.WriteString(" ")
+		b.WriteString(kv[i].key)
+		b.WriteString("=")
+		b.WriteString(kv[i].value)
+		if i < len(kv)-1 {
+			b.WriteString(",")
+		}
+	}
+	return b.String()
+}
+
 // HashRule hashes significant Rule fields into
 // unique hash that supposed to define Rule uniqueness
 func HashRule(r Rule) uint64 {
@@ -188,8 +195,11 @@ func (r *Rule) Validate() error {
 	return checkOverflow(r.XXX, "rule")
 }
 
+// ValidateTplFn must validate the given annotations
+type ValidateTplFn func(annotations map[string]string) error
+
 // Parse parses rule configs from given file patterns
-func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	var fp []string
 	for _, pattern := range pathPatterns {
 		matches, err := filepath.Glob(pattern)
@@ -199,7 +209,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
-	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -209,7 +218,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			continue
 		}
 		for _, g := range gr {
-			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
+			if err := g.Validate(validateTplFn, validateExpressions); err != nil {
 				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
 				continue
 			}
@@ -219,9 +228,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
-			if len(g.ExtraFilterLabels) > 0 {
-				isExtraFilterLabelsUsed = true
-			}
 			groups = append(groups, g)
 		}
 	}
@@ -231,14 +237,11 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
-	if isExtraFilterLabelsUsed {
-		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
-	}
 	return groups, nil
 }
 
 func parseFile(path string) ([]Group, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("error reading alert rule file: %w", err)
 	}

app/vmalert/config/config_test.go

@@ -9,19 +9,20 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
 func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
 		t.Errorf("error parsing files %s", err)
 	}
 }
@@ -32,7 +33,7 @@ func TestParseBad(t *testing.T) {
 		expErr string
 	}{
 		{
-			[]string{"testdata/rules0-bad.rules"},
+			[]string{"testdata/rules/rules0-bad.rules"},
 			"unexpected token",
 		},
 		{
@@ -56,12 +57,16 @@ func TestParseBad(t *testing.T) {
 			"either `record` or `alert` must be set",
 		},
 		{
-			[]string{"testdata/rules1-bad.rules"},
+			[]string{"testdata/rules/rules1-bad.rules"},
 			"bad graphite expr",
 		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
 	}
 	for _, tc := range testCases {
-		_, err := Parse(tc.path, true, true)
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
 			t.Errorf("expected to get error")
 			return
@@ -219,7 +224,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test thanos",
-				Type: datasource.NewRawType("thanos"),
+				Type: NewRawType("thanos"),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -231,7 +236,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test graphite",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "some-description",
@@ -243,7 +248,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test prometheus",
-				Type: datasource.NewPrometheusType(),
+				Type: NewPrometheusType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -256,7 +261,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite inherit",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
@@ -271,7 +276,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite prometheus bad expr",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
@@ -285,8 +290,13 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "invalid rule",
 		},
 	}
+
 	for _, tc := range testCases {
-		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
 		if err == nil {
 			if tc.expErr != "" {
 				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
@@ -491,6 +501,55 @@ params:
 rules:
   - alert: foo
     expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+limit: 5
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+limit: 10
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`debug` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+    debug: true
 `)
 	})
 }
@@ -528,30 +587,4 @@ rules:
     expr: sum by(job) (up == 1)
 `, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
 	})
-
-	t.Run("extra labels", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-  env: prod
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
-	})
-
-	t.Run("extra labels and params", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-params:
-  nocache: ["1"]
-  extra_label: ["env=prod"]
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
-	})
 }

app/vmalert/config/testdata/dir/rules6-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    headers:
+    - 'foobar'
+    rules:
+      - alert: rows
+        expr: vm_rows > 0

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -2,6 +2,7 @@ groups:
   - name: ReplayGroup
     interval: 1m
     concurrency: 1
+    limit: 1000
     rules:
       - record: type:vm_cache_entries:rate5m
         expr: sum(rate(vm_cache_entries[5m])) by (type)

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -1,18 +1,20 @@
 groups:
   - name: TestGroup
-    interval: 2s
+    interval: 5s
     concurrency: 2
-    extra_filter_labels: # deprecated param, use `params` instead
-        job: victoriametrics
+    limit: 1000
+    headers:
+      - "MyHeader: foo"
     params:
       denyPartialResponse: ["true"]
-      extra_label: ["env=dev"]
     rules:
       - alert: Conns
-        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        expr: vm_tcplistener_conns > 0
         for: 3m
+        debug: true
         annotations:
-          summary: Too high connection number for {{$labels.instance}}
+          labels: "Available labels: {{ $labels }}"
+          summary: Too high connection number for {{ $labels.instance }}
             {{ with printf "sum(vm_tcplistener_conns{instance=%q})" .Labels.instance | query }}
               {{ . | first | value }}
             {{ end }}
@@ -49,4 +51,12 @@ groups:
         expr: |2
             sum(code:requests:rate5m{code="200"})
           /
-            sum(code:requests:rate5m)
+            sum(code:requests:rate5m)
+      - record: code:requests:slo
+        labels:
+           recording: true
+        expr: 0.95
+      - record: time:current
+        labels:
+           recording: true
+        expr: time()

app/vmalert/config/testdata/templates/templates0-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template0" }}
+Visit {{ externalURL }}
+{{ end }}

app/vmalert/config/testdata/templates/templates1-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template1" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates2-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template2" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates3-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ printf "%s to %s!" "welcome" "hell" | toUpper }}
+{{ end }}

app/vmalert/config/testdata/templates/templates4-good-tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ 1230912039102391023.0 | humanizeDuration }}
+{{ end }}

app/vmalert/config/types.go

@@ -1,7 +1,8 @@
-package datasource
+package config
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
 	"github.com/VictoriaMetrics/metricsql"
@@ -9,45 +10,45 @@ import (
 
 // Type represents data source type
 type Type struct {
-	name string
+	Name string
 }
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
 	return Type{
-		name: "prometheus",
+		Name: "prometheus",
 	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
 	return Type{
-		name: "graphite",
+		Name: "graphite",
 	}
 }
 
 // NewRawType returns datasource type from raw string
 // without validation.
 func NewRawType(d string) Type {
-	return Type{name: d}
+	return Type{Name: d}
 }
 
 // Get returns datasource type
 func (t *Type) Get() string {
-	return t.name
+	return t.Name
 }
 
 // Set changes datasource type
 func (t *Type) Set(d Type) {
-	t.name = d.name
+	t.Name = d.Name
 }
 
 // String implements String interface with default value.
 func (t Type) String() string {
-	if t.name == "" {
+	if t.Name == "" {
 		return "prometheus"
 	}
-	return t.name
+	return t.Name
 }
 
 // ValidateExpr validates query expression with datasource ql.
@@ -62,7 +63,7 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	default:
-		return fmt.Errorf("unknown datasource type=%q", t.name)
+		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}
 	return nil
 }
@@ -81,11 +82,35 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	default:
 		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
-	t.name = s
+	t.Name = s
 	return nil
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
 func (t Type) MarshalYAML() (interface{}, error) {
-	return t.name, nil
+	return t.Name, nil
+}
+
+// Header is a Key - Value struct for holding an HTTP header.
+type Header struct {
+	Key   string
+	Value string
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	if s == "" {
+		return nil
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, s)
+	}
+	h.Key = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
 }

app/vmalert/datasource/datasource.go

@@ -2,26 +2,37 @@ package datasource
 
 import (
 	"context"
+	"net/http"
 	"net/url"
 	"time"
 )
 
 // Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string, ts time.Time) ([]Metric, error)
+	// Query executes instant request with the given query at the given ts.
+	// It returns list of Metric in response, the http.Request used for sending query
+	// and error if any. Returned http.Request can't be reused and its body is already read.
+	// Query should stop once ctx is cancelled.
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error)
+	// QueryRange executes range request with the given query on the given time range.
+	// It returns list of Metric in response and error if any.
+	// QueryRange should stop once ctx is cancelled.
 	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
 }
 
 // QuerierBuilder builds Querier with given params.
 type QuerierBuilder interface {
+	// BuildWithParams creates a new Querier object with the given params
 	BuildWithParams(params QuerierParams) Querier
 }
 
 // QuerierParams params for Querier.
 type QuerierParams struct {
-	DataSourceType     *Type
+	DataSourceType     string
 	EvaluationInterval time.Duration
 	QueryParams        url.Values
+	Headers            map[string]string
+	Debug              bool
 }
 
 // Metric is the basic entity which should be return by datasource

app/vmalert/datasource/init.go

@@ -6,14 +6,22 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
-	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
-		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
+		"E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.")
+	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to show -datasource.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("datasource.headers", "", "Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. "+
+		"For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. "+
+		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
@@ -35,9 +43,9 @@ var (
 	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
-	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
-		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
-		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
+		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
+		"If set to 0, rule's evaluation interval will be used instead.")
 	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
 		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
@@ -47,6 +55,13 @@ var (
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showDatasourceURL {
+		flagutil.RegisterSecretFlag("datasource.url")
+	}
+}
+
 // Param represents an HTTP GET param
 type Param struct {
 	Key, Value string
@@ -80,7 +95,8 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
@@ -92,7 +108,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,
 		queryStep:        *queryStep,
-		dataSourceType:   NewPrometheusType(),
+		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,
 	}, nil
 }

app/vmalert/datasource/vm.go

@@ -3,15 +3,30 @@ package datasource
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
+type datasourceType string
+
+const (
+	datasourcePrometheus datasourceType = "prometheus"
+	datasourceGraphite   datasourceType = "graphite"
+)
+
+func toDatasourceType(s string) datasourceType {
+	if s == string(datasourceGraphite) {
+		return datasourceGraphite
+	}
+	return datasourcePrometheus
+}
+
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
@@ -21,33 +36,46 @@ type VMStorage struct {
 	lookBack         time.Duration
 	queryStep        time.Duration
 
-	dataSourceType     Type
+	dataSourceType     datasourceType
 	evaluationInterval time.Duration
 	extraParams        url.Values
-	disablePathAppend  bool
+	extraHeaders       []keyValue
+
+	// whether to print additional log messages
+	// for each sent request
+	debug bool
+}
+
+type keyValue struct {
+	key   string
+	value string
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                 s.c,
-		authCfg:           s.authCfg,
-		datasourceURL:     s.datasourceURL,
-		lookBack:          s.lookBack,
-		queryStep:         s.queryStep,
-		appendTypePrefix:  s.appendTypePrefix,
-		dataSourceType:    s.dataSourceType,
-		disablePathAppend: s.disablePathAppend,
+		c:                s.c,
+		authCfg:          s.authCfg,
+		datasourceURL:    s.datasourceURL,
+		lookBack:         s.lookBack,
+		queryStep:        s.queryStep,
+		appendTypePrefix: s.appendTypePrefix,
+		dataSourceType:   s.dataSourceType,
 	}
 }
 
 // ApplyParams - changes given querier params.
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
-	if params.DataSourceType != nil {
-		s.dataSourceType = *params.DataSourceType
-	}
+	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
 	s.extraParams = params.QueryParams
+	s.debug = params.Debug
+	if params.Headers != nil {
+		for key, value := range params.Headers {
+			kv := keyValue{key: key, value: value}
+			s.extraHeaders = append(s.extraHeaders, kv)
+		}
+	}
 	return s
 }
 
@@ -57,56 +85,56 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
 	return &VMStorage{
-		c:                 c,
-		authCfg:           authCfg,
-		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix:  appendTypePrefix,
-		lookBack:          lookBack,
-		queryStep:         queryStep,
-		dataSourceType:    NewPrometheusType(),
-		disablePathAppend: disablePathAppend,
+		c:                c,
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix: appendTypePrefix,
+		lookBack:         lookBack,
+		queryStep:        queryStep,
+		dataSourceType:   datasourcePrometheus,
 	}
 }
 
 // Query executes the given query and returns parsed response
-func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, error) {
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error) {
 	req, err := s.newRequestPOST()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	switch s.dataSourceType.String() {
-	case "prometheus":
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case "graphite":
+	case datasourceGraphite:
 		s.setGraphiteReqParams(req, query, ts)
 	default:
-		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
+		return nil, nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
 	}
 
 	resp, err := s.do(ctx, req)
 	if err != nil {
-		return nil, err
+		return nil, req, err
 	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
 
 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != "prometheus" {
+	if s.dataSourceType != datasourcePrometheus {
 		parseFn = parseGraphiteResponse
 	}
-	return parseFn(req, resp)
+	result, err := parseFn(req, resp)
+	return result, req, err
 }
 
 // QueryRange executes the given query on the given time range.
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != "prometheus" {
-		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
+	if s.dataSourceType != datasourcePrometheus {
+		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
 	}
 	req, err := s.newRequestPOST()
 	if err != nil {
@@ -130,12 +158,15 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 }
 
 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	if s.debug {
+		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
+	}
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
 	if resp.StatusCode != http.StatusOK {
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
 	}
@@ -149,9 +180,10 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
-		if auth := s.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		s.authCfg.SetHeaders(req, true)
+	}
+	for _, h := range s.extraHeaders {
+		req.Header.Set(h.key, h.value)
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_prom_api.go

@@ -2,12 +2,18 @@ package datasource
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"strconv"
 	"time"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+
+		"to the configured -datasource.url and -remoteRead.url")
+)
+
 type promResponse struct {
 	Status    string `json:"status"`
 	ErrorType string `json:"errorType"`
@@ -25,13 +31,6 @@ type promInstant struct {
 	} `json:"result"`
 }
 
-type promRange struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
-	} `json:"result"`
-}
-
 func (r promInstant) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -50,6 +49,13 @@ func (r promInstant) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promRange struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TVs    [][2]interface{}  `json:"values"`
+	} `json:"result"`
+}
+
 func (r promRange) metrics() ([]Metric, error) {
 	var result []Metric
 	for i, res := range r.Result {
@@ -74,9 +80,22 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
+type promScalar [2]interface{}
+
+func (r promScalar) metrics() ([]Metric, error) {
+	var m Metric
+	f, err := strconv.ParseFloat(r[1].(string), 64)
+	if err != nil {
+		return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", r, r[1], err)
+	}
+	m.Values = append(m.Values, f)
+	m.Timestamps = append(m.Timestamps, int64(r[0].(float64)))
+	return []Metric{m}, nil
+}
+
 const (
-	statusSuccess, statusError = "success", "error"
-	rtVector, rtMatrix         = "vector", "matrix"
+	statusSuccess, statusError  = "success", "error"
+	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
 )
 
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
@@ -103,23 +122,23 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric,
 			return nil, err
 		}
 		return pr.metrics()
+	case rScalar:
+		var ps promScalar
+		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
+			return nil, err
+		}
+		return ps.metrics()
 	default:
 		return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
 	}
 }
 
-const (
-	prometheusInstantPath = "/api/v1/query"
-	prometheusRangePath   = "/api/v1/query_range"
-	prometheusPrefix      = "/prometheus"
-)
-
 func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusInstantPath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
@@ -136,10 +155,10 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 
 func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	if s.appendTypePrefix {
-		r.URL.Path += prometheusPrefix
+		r.URL.Path += "/prometheus"
 	}
-	if !s.disablePathAppend {
-		r.URL.Path += prometheusRangePath
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))

app/vmalert/datasource/vm_test.go

@@ -37,7 +37,7 @@ func TestVMInstantQuery(t *testing.T) {
 	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
 		c++
 		switch c {
-		case 7:
+		case 8:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -75,41 +75,38 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
 		case 6:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests"},"value":[1583786140,"2000"]}]}}`))
+		case 7:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	p := datasourcePrometheus
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
 	ts := time.Now()
 
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected connection error got nil")
+	expErr := func(err string) {
+		if _, _, err := pq.Query(ctx, query, ts); err == nil {
+			t.Fatalf("expected %q got nil", err)
+		}
 	}
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected invalid response status error got nil")
-	}
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected response body error got nil")
-	}
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected error status got nil")
-	}
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected unknown status got nil")
-	}
-	if _, err := pq.Query(ctx, query, ts); err == nil {
-		t.Fatalf("expected non-vector resultType error  got nil")
-	}
-	m, err := pq.Query(ctx, query, ts)
+
+	expErr("connection error")              // 0
+	expErr("invalid response status error") // 1
+	expErr("response body error")           // 2
+	expErr("error status")                  // 3
+	expErr("unknown status")                // 4
+	expErr("non-vector resultType error")   // 5
+
+	m, _, err := pq.Query(ctx, query, ts) // 6 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -132,10 +129,29 @@ func TestVMInstantQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m, expected)
 	}
 
-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	m, req, err := pq.Query(ctx, query, ts) // 7 - scalar
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if req == nil {
+		t.Fatalf("expected request to be non-nil")
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metrics got %d in %+v", len(m), m)
+	}
+	expected = []Metric{
+		{
+			Timestamps: []int64{1583786142},
+			Values:     []float64{1},
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m, expected)
+	}
 
-	m, err = gq.Query(ctx, queryRender, ts)
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
+
+	m, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -192,14 +208,13 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
-	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
 
-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})
 
 	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
@@ -225,15 +240,14 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
 
-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
 	_, err = gq.QueryRange(ctx, queryRender, start, end)
 	expectError(t, err, "is not supported")
 }
 
 func TestRequestParams(t *testing.T) {
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
@@ -249,95 +263,49 @@ func TestRequestParams(t *testing.T) {
 			"prometheus path",
 			false,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus path with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
 			},
 		},
 		{
 			"prometheus prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus prefix with disablePathAppend",
-			false,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
 			},
 		},
 		{
 			"prometheus range path",
 			true,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range path with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "", r.URL.Path)
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
 			"prometheus range prefix",
 			true,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
-			},
-		},
-		{
-			"prometheus range prefix with disablePathAppend",
-			true,
-			&VMStorage{
-				dataSourceType:    NewPrometheusType(),
-				appendTypePrefix:  true,
-				disablePathAppend: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, prometheusPrefix, r.URL.Path)
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
 			},
 		},
 		{
 			"graphite path",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 			},
 			func(t *testing.T, r *http.Request) {
 				checkEqualString(t, graphitePath, r.URL.Path)
@@ -347,7 +315,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewGraphiteType(),
+				dataSourceType:   datasourceGraphite,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
@@ -485,7 +453,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite extra params",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 				extraParams: url.Values{
 					"nocache":      {"1"},
 					"max_lookback": {"1h"},
@@ -504,14 +472,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.String() {
-			case "prometheus":
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case "graphite":
+			case datasourceGraphite:
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -519,7 +487,7 @@ func TestRequestParams(t *testing.T) {
 	}
 }
 
-func TestAuthConfig(t *testing.T) {
+func TestHeaders(t *testing.T) {
 	var testCases = []struct {
 		name    string
 		vmFn    func() *VMStorage
@@ -559,6 +527,40 @@ func TestAuthConfig(t *testing.T) {
 				checkEqualString(t, "foo", token)
 			},
 		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
 	}
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {

app/vmalert/group.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -18,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // Group is an entity for grouping rules
@@ -27,14 +28,16 @@ type Group struct {
 	Name           string
 	File           string
 	Rules          []Rule
-	Type           datasource.Type
+	Type           config.Type
 	Interval       time.Duration
+	Limit          int
 	Concurrency    int
 	Checksum       string
 	LastEvaluation time.Time
 
-	Labels map[string]string
-	Params url.Values
+	Labels  map[string]string
+	Params  url.Values
+	Headers map[string]string
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -49,14 +52,21 @@ type groupMetrics struct {
 	iterationTotal    *utils.Counter
 	iterationDuration *utils.Summary
 	iterationMissed   *utils.Counter
+	iterationInterval *utils.Gauge
 }
 
-func newGroupMetrics(name, file string) *groupMetrics {
+func newGroupMetrics(g *Group) *groupMetrics {
 	m := &groupMetrics{}
-	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
+	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
 	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		i := g.Interval.Seconds()
+		g.mu.RUnlock()
+		return i
+	})
 	return m
 }
 
@@ -83,22 +93,27 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:        cfg.Name,
 		File:        cfg.File,
 		Interval:    cfg.Interval.Duration(),
+		Limit:       cfg.Limit,
 		Concurrency: cfg.Concurrency,
 		Checksum:    cfg.Checksum,
 		Params:      cfg.Params,
+		Headers:     make(map[string]string),
 		Labels:      cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
 		updateCh:   make(chan *Group),
 	}
-	g.metrics = newGroupMetrics(g.Name, g.File)
 	if g.Interval == 0 {
 		g.Interval = defaultInterval
 	}
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
+	for _, h := range cfg.Headers {
+		g.Headers[h.Key] = h.Value
+	}
+	g.metrics = newGroupMetrics(g)
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
 		var extraLabels map[string]string
@@ -207,7 +222,9 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
 	g.Params = newGroup.Params
+	g.Headers = newGroup.Headers
 	g.Labels = newGroup.Labels
+	g.Limit = newGroup.Limit
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
@@ -222,6 +239,8 @@ func (g *Group) close() {
 
 	g.metrics.iterationDuration.Unregister()
 	g.metrics.iterationTotal.Unregister()
+	g.metrics.iterationMissed.Unregister()
+	g.metrics.iterationInterval.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -237,8 +256,6 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		notifiers:                nts,
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
 
-	evalTS := time.Now()
-
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
 		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
@@ -259,6 +276,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 	}
 
+	evalTS := time.Now()
+
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 
 	eval := func(ts time.Time) {
@@ -273,7 +292,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
-		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
 		for err := range errs {
 			if err != nil {
 				logger.Errorf("group %q: %s", g.Name, err)
@@ -303,6 +322,10 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				g.mu.Unlock()
 				continue
 			}
+
+			// ensure that staleness is tracked or existing rules only
+			e.purgeStaleSeries(g.Rules)
+
 			if g.Interval != ng.Interval {
 				g.Interval = ng.Interval
 				t.Stop()
@@ -347,12 +370,12 @@ type executor struct {
 	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, ts, resolveDuration)
+			res <- e.exec(ctx, rule, ts, resolveDuration, limit)
 		}
 		close(res)
 		return res
@@ -365,7 +388,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.T
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, ts, resolveDuration)
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -386,10 +409,10 @@ var (
 	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration, limit int) error {
 	execTotal.Inc()
 
-	tss, err := rule.Exec(ctx, ts)
+	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
@@ -421,11 +444,17 @@ func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDur
 		return nil
 	}
 
+	wg := sync.WaitGroup{}
 	for _, nt := range e.notifiers() {
-		if err := nt.Send(ctx, alerts); err != nil {
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
-		}
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
 	}
+	wg.Wait()
 	return errGr.Err()
 }
 
@@ -457,6 +486,30 @@ func (e *executor) getStaleSeries(rule Rule, tss []prompbmarshal.TimeSeries, tim
 	return staleS
 }
 
+// purgeStaleSeries deletes references in tracked
+// previouslySentSeriesToRW list to Rules which aren't present
+// in the given activeRules list. The method is used when the list
+// of loaded rules has changed and executor has to remove
+// references to non-existing rules.
+func (e *executor) purgeStaleSeries(activeRules []Rule) {
+	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
+
+	e.previouslySentSeriesToRWMu.Lock()
+
+	for _, rule := range activeRules {
+		id := rule.ID()
+		prev, ok := e.previouslySentSeriesToRW[id]
+		if ok {
+			// keep previous series for staleness detection
+			newPreviouslySentSeriesToRW[id] = prev
+		}
+	}
+	e.previouslySentSeriesToRW = nil
+	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
+
+	e.previouslySentSeriesToRWMu.Unlock()
+}
+
 func labelsToString(labels []prompbmarshal.Label) string {
 	var b strings.Builder
 	b.WriteRune('{')

app/vmalert/group_test.go

@@ -148,7 +148,7 @@ func TestUpdateWith(t *testing.T) {
 					t.Fatalf("expected to have rule %q; got %q", want, got)
 				}
 				if err := compareRules(t, got, want); err != nil {
-					t.Fatalf("comparsion error: %s", err)
+					t.Fatalf("comparison error: %s", err)
 				}
 			}
 		})
@@ -157,7 +157,7 @@ func TestUpdateWith(t *testing.T) {
 
 func TestGroupStart(t *testing.T) {
 	// TODO: make parsing from string instead of file
-	groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
@@ -355,3 +355,100 @@ func TestGetStaleSeries(t *testing.T) {
 		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
 		nil)
 }
+
+func TestPurgeStaleSeries(t *testing.T) {
+	ts := time.Now()
+	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
+	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
+
+	f := func(curRules, newRules, expStaleRules []Rule) {
+		t.Helper()
+		e := &executor{
+			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		}
+		// seed executor with series for
+		// current rules
+		for _, rule := range curRules {
+			e.getStaleSeries(rule, tss, ts)
+		}
+
+		e.purgeStaleSeries(newRules)
+
+		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expStaleRules), len(e.previouslySentSeriesToRW))
+		}
+
+		for _, exp := range expStaleRules {
+			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
+				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
+			}
+		}
+	}
+
+	f(nil, nil, nil)
+	f(
+		nil,
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+	)
+}
+
+func TestFaultyNotifier(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := newTestAlertingRule("instant", 0)
+	r.q = fq
+
+	fn := &fakeNotifier{}
+	e := &executor{
+		notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&faultyNotifier{},
+				fn,
+			}
+		},
+	}
+	delay := 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), delay)
+	defer cancel()
+
+	go func() {
+		_ = e.exec(ctx, r, time.Now(), 0, 10)
+	}()
+
+	tn := time.Now()
+	deadline := tn.Add(delay / 2)
+	for {
+		if fn.getCounter() > 0 {
+			return
+		}
+		if tn.After(deadline) {
+			break
+		}
+		tn = time.Now()
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Fatalf("alive notifier didn't receive notification by %v", deadline)
+}

app/vmalert/helpers_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"reflect"
 	"sort"
 	"sync"
@@ -44,18 +45,20 @@ func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Qu
 }
 
 func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
-	return fq.Query(ctx, q, time.Now())
+	req, _, err := fq.Query(ctx, q, time.Now())
+	return req, err
 }
 
-func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
-		return nil, fq.err
+		return nil, nil, fq.err
 	}
 	cp := make([]datasource.Metric, len(fq.metrics))
 	copy(cp, fq.metrics)
-	return cp, nil
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	return cp, req, nil
 }
 
 type fakeNotifier struct {
@@ -87,6 +90,18 @@ func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	return fn.alerts
 }
 
+type faultyNotifier struct {
+	fakeNotifier
+}
+
+func (fn *faultyNotifier) Send(ctx context.Context, _ []notifier.Alert) error {
+	d, ok := ctx.Deadline()
+	if ok {
+		time.Sleep(time.Until(d))
+	}
+	return fmt.Errorf("send failed")
+}
+
 func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
 	return metricWithValuesAndLabels(t, []float64{value}, labels...)
 }
@@ -152,7 +167,7 @@ func compareGroups(t *testing.T, a, b *Group) {
 			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
 		}
 		if err := compareRules(t, want, got); err != nil {
-			t.Fatalf("comparsion error: %s", err)
+			t.Fatalf("comparison error: %s", err)
 		}
 	}
 }

app/vmalert/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -22,11 +23,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules.
+	rulePath = flagutil.NewArrayString("rule", `Path to the file with alert rules.
 Supports patterns. Flag can be specified multiple times.
 Examples:
  -rule="/path/to/file". Path to a single file with alerting rules
@@ -34,6 +36,13 @@ Examples:
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
+	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
+	for rules annotations templating. Flag can be specified multiple times.
+Examples:
+ -rule.templates="/path/to/file". Path to a single file with go templates
+ -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+absolute path to all .tpl files in root.`)
+
 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
 		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
 
@@ -50,9 +59,12 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor
 	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
 
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
-	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
-	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
+	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
+		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
+		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]' . `+
+		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used`)
+	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
@@ -61,7 +73,7 @@ eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{
 
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 
-	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
 var alertURLGeneratorFn notifier.AlertURLGenerator
@@ -71,13 +83,20 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
+
+	err := templates.Load(*ruleTemplatesPath, true)
+	if err != nil {
+		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
+	}
 
 	if *dryRun {
-		u, _ := url.Parse("https://victoriametrics.com/")
-		notifier.InitTemplateFunc(u)
-		groups, err := config.Parse(*rulePath, true, true)
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -91,12 +110,17 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init `external.url`: %s", err)
 	}
-	notifier.InitTemplateFunc(eu)
+
 	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
 	if err != nil {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
 	}
 
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
@@ -105,8 +129,7 @@ func main() {
 		if rw == nil {
 			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
-		notifier.InitTemplateFunc(eu)
-		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
 		}
@@ -127,9 +150,8 @@ func main() {
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
-
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
-	groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 	if err != nil {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
@@ -170,7 +192,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	labels := make(map[string]string, 0)
+	labels := make(map[string]string)
 	for _, s := range *externalLabels {
 		if len(s) == 0 {
 			continue
@@ -228,8 +250,9 @@ func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL
 
 func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
 	if externalAlertSource == "" {
-		return func(alert notifier.Alert) string {
-			return fmt.Sprintf("%s/api/v1/%s/%s/status", externalURL, strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10))
+		return func(a notifier.Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
 		}, nil
 	}
 	if validateTemplate {
@@ -243,7 +266,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, nil, m)
+		templated, err := alert.ExecTemplate(nil, alert.Labels, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
@@ -273,6 +296,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		defer ticker.Stop()
 	}
 
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	// init reload metrics with positive values to improve alerting conditions
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
@@ -281,7 +309,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		case <-ctx.Done():
 			return
 		case <-sighupCh:
-			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
+			tmplMsg := ""
+			if len(*ruleTemplatesPath) > 0 {
+				tmplMsg = fmt.Sprintf("and templates %q ", *ruleTemplatesPath)
+			}
+			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
@@ -291,7 +323,14 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		err := templates.Load(*ruleTemplatesPath, false)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to load new templates: %s", err)
+			continue
+		}
+		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
@@ -299,6 +338,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
 			// set success to 1 since previous reload
 			// could have been unsuccessful
 			configSuccess.Set(1)
@@ -311,6 +351,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		templates.Reload()
 		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())

app/vmalert/main_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"testing"
@@ -35,24 +34,25 @@ func TestGetExternalURL(t *testing.T) {
 }
 
 func TestGetAlertURLGenerator(t *testing.T) {
-	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4}
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/api/v1/42/2/status"; exp != fn(testAlert) {
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
+	if exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Errorf("expected tempalte validation error got nil")
+		t.Errorf("expected template validation error got nil")
 	}
-	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true)
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/foo?query=4"; exp != fn(testAlert) {
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
@@ -86,7 +86,7 @@ groups:
 `
 	)
 
-	f, err := ioutil.TempFile("", "")
+	f, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +153,7 @@ groups:
 
 func writeToFile(t *testing.T, file, b string) {
 	t.Helper()
-	err := ioutil.WriteFile(file, []byte(b), 0644)
+	err := os.WriteFile(file, []byte(b), 0644)
 	if err != nil {
 		t.Fatal(err)
 	}

app/vmalert/manager.go

@@ -30,6 +30,23 @@ type manager struct {
 	groups   map[uint64]*Group
 }
 
+// RuleAPI generates APIRule object from alert by its ID(hash)
+func (m *manager) RuleAPI(gID, rID uint64) (APIRule, error) {
+	m.groupsMu.RLock()
+	defer m.groupsMu.RUnlock()
+
+	g, ok := m.groups[gID]
+	if !ok {
+		return APIRule{}, fmt.Errorf("can't find group with id %d", gID)
+	}
+	for _, rule := range g.Rules {
+		if rule.ID() == rID {
+			return rule.ToAPI(), nil
+		}
+	}
+	return APIRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
+}
+
 // AlertAPI generates APIAlert object from alert by its ID(hash)
 func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 	m.groupsMu.RLock()
@@ -70,9 +87,9 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
 		if err != nil {
 			if !*remoteReadIgnoreRestoreErrors {
-				return fmt.Errorf("failed to restore state for group %q: %w", group.Name, err)
+				return fmt.Errorf("failed to restore ruleState for group %q: %w", group.Name, err)
 			}
-			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
+			logger.Errorf("error while restoring ruleState for group %q: %s", group.Name, err)
 		}
 	}
 
@@ -170,6 +187,7 @@ func (g *Group) toAPI() APIGroup {
 		LastEvaluation: g.LastEvaluation,
 		Concurrency:    g.Concurrency,
 		Params:         urlValuesToStrings(g.Params),
+		Headers:        headersToStrings(g.Headers),
 		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
@@ -198,3 +216,23 @@ func urlValuesToStrings(values url.Values) []string {
 	}
 	return res
 }
+
+func headersToStrings(headers map[string]string) []string {
+	if len(headers) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(headers))
+	for k := range headers {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		v := headers[k]
+		res = append(res, fmt.Sprintf("%s: %s", k, v))
+	}
+
+	return res
+}

app/vmalert/manager_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"math/rand"
-	"net/url"
 	"os"
 	"strings"
 	"sync"
@@ -11,14 +10,15 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
@@ -29,7 +29,7 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
 	cfg := loadCfg(t, []string{"foo/bar"}, true, true)
 	if err := m.update(context.Background(), cfg, false); err != nil {
-		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
+		t.Fatalf("expected to load successfully with empty rules dir; got err instead: %v", err)
 	}
 }
 
@@ -47,9 +47,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 		"config/testdata/dir/rules0-bad.rules",
 		"config/testdata/dir/rules1-good.rules",
 		"config/testdata/dir/rules1-bad.rules",
-		"config/testdata/rules0-good.rules",
-		"config/testdata/rules1-good.rules",
-		"config/testdata/rules2-good.rules",
+		"config/testdata/rules/rules0-good.rules",
+		"config/testdata/rules/rules1-good.rules",
+		"config/testdata/rules/rules2-good.rules",
 	}
 	evalInterval := *evaluationInterval
 	defer func() { *evaluationInterval = evalInterval }()
@@ -68,7 +68,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			defer wg.Done()
 			for i := 0; i < iterations; i++ {
 				rnd := rand.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, true, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -125,13 +125,13 @@ func TestManagerUpdate(t *testing.T) {
 	}{
 		{
 			name:       "update good rules",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules1-good.rules",
 			want: []*Group{
 				{
 					File:     "config/testdata/dir/rules1-good.rules",
 					Name:     "duplicatedGroupDiffFiles",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules: []Rule{
 						&AlertingRule{
@@ -150,20 +150,20 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update good rules from 1 to 2 groups",
-			initPath:   "config/testdata/dir/rules1-good.rules",
-			updatePath: "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/dir/rules/rules1-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules:    []Rule{VMRows},
 					Interval: defaultEvalInterval,
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -172,21 +172,21 @@ func TestManagerUpdate(t *testing.T) {
 		},
 		{
 			name:       "update with one bad rule file",
-			initPath:   "config/testdata/rules0-good.rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules2-bad.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Name:     "TestGroup",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -196,19 +196,19 @@ func TestManagerUpdate(t *testing.T) {
 		{
 			name:       "update empty dir rules from 0 to 2 groups",
 			initPath:   "config/testdata/empty/*",
-			updatePath: "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
 			want: []*Group{
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
-					File:     "config/testdata/rules0-good.rules",
+					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -231,7 +231,7 @@ func TestManagerUpdate(t *testing.T) {
 				t.Fatalf("failed to complete initial rules update: %s", err)
 			}
 
-			cfgUpdate, err := config.Parse([]string{tc.updatePath}, true, true)
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
 			if err == nil { // update can fail and that's expected
 				_ = m.update(ctx, cfgUpdate, false)
 			}
@@ -329,7 +329,11 @@ func TestManagerUpdateNegative(t *testing.T) {
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
-	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)
+	var validateTplFn config.ValidateTplFn
+	if validateAnnotations {
+		validateTplFn = notifier.ValidateTemplates
+	}
+	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {
 		t.Fatal(err)
 	}

app/vmalert/multiarch/Dockerfile

@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
 ARG TARGETARCH
-COPY vmalert-${TARGETARCH}-prod ./vmalert-prod
+COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod

app/vmalert/notifier/alert.go

@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"text/template"
+	textTpl "text/template"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -73,9 +74,12 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Labels map[string]string
-	Value  float64
-	Expr   string
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
 }
 
 var tplHeaders = []string{
@@ -84,32 +88,47 @@ var tplHeaders = []string{
 	"{{ $expr := .Expr }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
+	"{{ $alertID := .AlertID }}",
+	"{{ $groupID := .GroupID }}",
+	"{{ $activeAt := .ActiveAt }}",
 }
 
 // ExecTemplate executes the Alert template for given
 // map of annotations.
 // Every alert could have a different datasource, so function
 // requires a queryFunction as an argument.
-func (a *Alert) ExecTemplate(q QueryFn, labels, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
-	return templateAnnotations(annotations, tplData, funcsWithQuery(q), true)
+func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr, AlertID: a.ID, GroupID: a.GroupID, ActiveAt: a.ActiveAt}
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error getting a template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ExecTemplate executes the given template for given annotations map.
-func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) {
-	return templateAnnotations(annotations, tpl, funcsWithQuery(q), true)
+func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error cloning template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
 }
 
 // ValidateTemplates validate annotations for possible template error, uses empty data for template population
 func ValidateTemplates(annotations map[string]string) error {
-	_, err := templateAnnotations(annotations, AlertTplData{
+	tmpl, err := templates.Get()
+	if err != nil {
+		return err
+	}
+	_, err = templateAnnotations(annotations, AlertTplData{
 		Labels: map[string]string{},
 		Value:  0,
-	}, tmplFunc, false)
+	}, tmpl, false)
 	return err
 }
 
-func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap, execute bool) (map[string]string, error) {
+func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
@@ -122,7 +141,7 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 		builder.Grow(len(header) + len(text))
 		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), tData, funcs, execute); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, tmpl, execute); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -138,11 +157,14 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap, execute bool) error {
-	t := template.New("").Funcs(funcs).Option("missingkey=zero")
-	tpl, err := t.Parse(text)
+func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
+	tpl, err := tmpl.Clone()
 	if err != nil {
-		return fmt.Errorf("error parsing annotation: %w", err)
+		return fmt.Errorf("error cloning template before parse annotation: %w", err)
+	}
+	tpl, err = tpl.Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation template: %w", err)
 	}
 	if !execute {
 		return nil

app/vmalert/notifier/alert_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -11,7 +12,7 @@ import (
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
-	extLabels := make(map[string]string, 0)
+	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
 		extDC      = "east"
@@ -109,6 +110,65 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
 			},
 		},
+		{
+			name: "alert and group IDs",
+			alert: &Alert{
+				ID:      42,
+				GroupID: 24,
+			},
+			annotations: map[string]string{
+				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+			},
+			expTpl: map[string]string{
+				"url": "/api/v1/alert?alertID=42&groupID=24",
+			},
+		},
+		{
+			name: "ActiveAt time",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+			},
+			expTpl: map[string]string{
+				"diagram": "![](http://example.com?render=1660941298",
+			},
+		},
+		{
+			name:  "ActiveAt time is nil",
+			alert: &Alert{},
+			annotations: map[string]string{
+				"default_time": "{{$activeAt}}",
+			},
+			expTpl: map[string]string{
+				"default_time": "0001-01-01 00:00:00 +0000 UTC",
+			},
+		},
+		{
+			name: "ActiveAt custome format",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+			},
+			expTpl: map[string]string{
+				"fire_time": "2022/08/19 20:34:58",
+			},
+		},
+		{
+			name: "ActiveAt query range",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+			},
+			expTpl: map[string]string{
+				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+			},
+		},
 	}
 
 	qFn := func(q string) ([]datasource.Metric, error) {

app/vmalert/notifier/alertmanager.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -79,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	req = req.WithContext(ctx)
 
 	if am.authCfg != nil {
-		if auth := am.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		am.authCfg.SetHeaders(req, true)
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -91,7 +89,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	defer func() { _ = resp.Body.Close() }()
 
 	if resp.StatusCode != http.StatusOK {
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
 		}

app/vmalert/notifier/config.go

@@ -4,8 +4,8 @@ import (
 	"crypto/md5"
 	"fmt"
 	"gopkg.in/yaml.v2"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -63,10 +63,13 @@ type Config struct {
 }
 
 // StaticConfig contains list of static targets in the following form:
-// 	targets:
-//  	[ - '<host>' ]
+//
+//	targets:
+//	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
+	// HTTPClientConfig contains HTTP configuration for the Targets
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -103,7 +106,7 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 func parseConfig(path string) (*Config, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("error reading config file: %w", err)
 	}

app/vmalert/notifier/config_watcher.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
@@ -161,12 +162,13 @@ func (cw *configWatcher) start() error {
 	if len(cw.cfg.StaticConfigs) > 0 {
 		var targets []Target
 		for _, cfg := range cw.cfg.StaticConfigs {
+			httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig)
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
@@ -252,3 +254,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
 	cw.targets[key] = targets
 	cw.targetsMu.Unlock()
 }
+
+// mergeHTTPClientConfigs merges fields between child and parent params
+// by populating child from parent params if they're missing.
+func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig {
+	if child.Authorization == nil {
+		child.Authorization = parent.Authorization
+	}
+	if child.BasicAuth == nil {
+		child.BasicAuth = parent.BasicAuth
+	}
+	if child.BearerToken == nil {
+		child.BearerToken = parent.BearerToken
+	}
+	if child.BearerTokenFile == "" {
+		child.BearerTokenFile = parent.BearerTokenFile
+	}
+	if child.OAuth2 == nil {
+		child.OAuth2 = parent.OAuth2
+	}
+	if child.TLSConfig == nil {
+		child.TLSConfig = parent.TLSConfig
+	}
+	if child.Headers == nil {
+		child.Headers = parent.Headers
+	}
+	return child
+}

app/vmalert/notifier/config_watcher_test.go

@@ -2,17 +2,18 @@ package notifier
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"sync"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 func TestConfigWatcherReload(t *testing.T) {
-	f, err := ioutil.TempFile("", "")
+	f, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -34,7 +35,7 @@ static_configs:
 		t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns)
 	}
 
-	f2, err := ioutil.TempFile("", "")
+	f2, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +62,7 @@ func TestConfigWatcherStart(t *testing.T) {
 	consulSDServer := newFakeConsulServer()
 	defer consulSDServer.Close()
 
-	consulSDFile, err := ioutil.TempFile("", "")
+	consulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -107,7 +108,7 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) {
 	consulSDServer2 := newFakeConsulServer()
 	defer consulSDServer2.Close()
 
-	consulSDFile, err := ioutil.TempFile("", "")
+	consulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -123,7 +124,7 @@ consul_sd_configs:
       - consul
 `, consulSDServer1.URL, consulSDServer2.URL))
 
-	staticAndConsulSDFile, err := ioutil.TempFile("", "")
+	staticAndConsulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -175,7 +176,7 @@ consul_sd_configs:
 
 func writeToFile(t *testing.T, file, b string) {
 	t.Helper()
-	checkErr(t, ioutil.WriteFile(file, []byte(b), 0644))
+	checkErr(t, os.WriteFile(file, []byte(b), 0644))
 }
 
 func checkErr(t *testing.T, err error) {
@@ -299,3 +300,20 @@ func newFakeConsulServer() *httptest.Server {
 
 	return httptest.NewServer(mux)
 }
+
+func TestMergeHTTPClientConfigs(t *testing.T) {
+	cfg1 := promauth.HTTPClientConfig{Headers: []string{"Header:Foo"}}
+	cfg2 := promauth.HTTPClientConfig{BasicAuth: &promauth.BasicAuthConfig{
+		Username: "foo",
+		Password: promauth.NewSecret("bar"),
+	}}
+
+	result := mergeHTTPClientConfigs(cfg1, cfg2)
+
+	if result.Headers == nil {
+		t.Fatalf("expected Headers to be inherited")
+	}
+	if result.BasicAuth == nil {
+		t.Fatalf("expected BasicAuth tp be present")
+	}
+}

app/vmalert/notifier/init.go

@@ -3,9 +3,11 @@ package notifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -15,32 +17,32 @@ var (
 	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
 	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
 
-	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	addrs = flagutil.NewArrayString("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
 
-	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
-	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
+	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
-	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
-	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
+	bearerToken     = flagutil.NewArrayString("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArrayString("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
 
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
-	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
-	tlsKeyFile            = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
-	tlsCAFile             = flagutil.NewArray("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
+	tlsCertFile           = flagutil.NewArrayString("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
+	tlsKeyFile            = flagutil.NewArrayString("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
+	tlsCAFile             = flagutil.NewArrayString("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
 		"By default system CA is used")
-	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
+	tlsServerName = flagutil.NewArrayString("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")
 
-	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+	oauth2ClientID = flagutil.NewArrayString("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+	oauth2ClientSecret = flagutil.NewArrayString("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+	oauth2ClientSecretFile = flagutil.NewArrayString("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+	oauth2TokenURL = flagutil.NewArrayString("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+	oauth2Scopes = flagutil.NewArrayString("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )
 
@@ -72,9 +74,10 @@ var (
 
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
-//   * configuration via flags (for backward compatibility). Is always static
+//   - configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
-//   * configuration via file. Supports live reloads and service discovery.
+//   - configuration via file. Supports live reloads and service discovery.
+//
 // Init returns an error if both mods are used.
 func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
 	if externalLabels != nil || externalURL != "" {
@@ -83,6 +86,12 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 
 	externalURL = extURL
 	externalLabels = extLabels
+	eu, err := url.Parse(externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse external URL: %s", err)
+	}
+
+	templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
 
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
@@ -102,7 +111,6 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 		return staticNotifiersFn, nil
 	}
 
-	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %s", err)
@@ -138,7 +146,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}
 
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
 		if err != nil {
 			return nil, err
 		}

app/vmalert/notifier/package_test.go

@@ -1,13 +1,14 @@
 package notifier
 
 import (
-	"net/url"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"os"
 	"testing"
 )
 
 func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	InitTemplateFunc(u)
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }

app/vmalert/notifier/testdata/static.good.yaml

@@ -1,7 +1,21 @@
+headers:
+  - 'CustomHeader: foo'
+
 static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
+    basic_auth:
+      username: foo
+      password: bar
+
+  - targets:
+      - localhost:9096
+      - localhost:9097
+    basic_auth:
+      username: foo
+      password: baz
+
 alert_relabel_configs:
   - target_label: "foo"
-    replacement: "aaa"
+    replacement: "aaa"

app/vmalert/recording.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -18,7 +17,7 @@ import (
 // to evaluate configured Expression and
 // return TimeSeries as result.
 type RecordingRule struct {
-	Type    datasource.Type
+	Type    config.Type
 	RuleID  uint64
 	Name    string
 	Expr    string
@@ -27,19 +26,9 @@ type RecordingRule struct {
 
 	q datasource.Querier
 
-	// guard status fields
-	mu sync.RWMutex
-	// stores last moment of time Exec was called
-	lastExecTime time.Time
-	// stores the duration of the last Exec call
-	lastExecDuration time.Duration
-	// stores last error that happened in Exec func
-	// resets on every successful Exec
-	// may be used as Health state
-	lastExecError error
-	// stores the number of samples returned during
-	// the last evaluation
-	lastExecSamples int
+	// state stores recent state changes
+	// during evaluations
+	state *ruleState
 
 	metrics *recordingRuleMetrics
 }
@@ -69,28 +58,28 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		Labels:  cfg.Labels,
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
+		state:   newRuleState(),
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
 		}),
 	}
 
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
 	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
-			rr.mu.RLock()
-			defer rr.mu.RUnlock()
-			if rr.lastExecError == nil {
+			e := rr.state.getLast()
+			if e.err == nil {
 				return 0
 			}
 			return 1
 		})
 	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
-			rr.mu.RLock()
-			defer rr.mu.RUnlock()
-			return float64(rr.lastExecSamples)
+			e := rr.state.getLast()
+			return float64(e.samples)
 		})
 	return rr
 }
@@ -124,17 +113,30 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }
 
 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, req, err := rr.q.Query(ctx, rr.Expr, ts)
+	curState := ruleStateEntry{
+		time:     start,
+		at:       ts,
+		duration: time.Since(start),
+		samples:  len(qMetrics),
+		req:      req,
+	}
+
+	defer func() {
+		rr.state.add(curState)
+	}()
 
-	rr.lastExecTime = ts
-	rr.lastExecDuration = time.Since(ts)
-	rr.lastExecError = err
-	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
-		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+		curState.err = fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+		return nil, curState.err
+	}
+
+	numSeries := len(qMetrics)
+	if limit > 0 && numSeries > limit {
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d series", limit, numSeries)
+		return nil, curState.err
 	}
 
 	duplicates := make(map[string]struct{}, len(qMetrics))
@@ -143,8 +145,8 @@ func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarsha
 		ts := rr.toTimeSeries(r)
 		key := stringifyLabels(ts)
 		if _, ok := duplicates[key]; ok {
-			rr.lastExecError = errDuplicate
-			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
+			curState.err = fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
+			return nil, curState.err
 		}
 		duplicates[key] = struct{}{}
 		tss = append(tss, ts)
@@ -199,23 +201,25 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 // ToAPI returns Rule's representation in form
 // of APIRule
 func (rr *RecordingRule) ToAPI() APIRule {
+	lastState := rr.state.getLast()
 	r := APIRule{
 		Type:           "recording",
 		DatasourceType: rr.Type.String(),
 		Name:           rr.Name,
 		Query:          rr.Expr,
 		Labels:         rr.Labels,
-		LastEvaluation: rr.lastExecTime,
-		EvaluationTime: rr.lastExecDuration.Seconds(),
+		LastEvaluation: lastState.time,
+		EvaluationTime: lastState.duration.Seconds(),
 		Health:         "ok",
-		LastSamples:    rr.lastExecSamples,
+		LastSamples:    lastState.samples,
+		Updates:        rr.state.getAll(),
+
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", rr.ID()),
 		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
-
-	if rr.lastExecError != nil {
-		r.LastError = rr.lastExecError.Error()
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
 		r.Health = "err"
 	}
 	return r

app/vmalert/recording_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestRecoridngRule_Exec(t *testing.T) {
+func TestRecordingRule_Exec(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -19,7 +19,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 		expTS   []prompbmarshal.TimeSeries
 	}{
 		{
-			&RecordingRule{Name: "foo"},
+			&RecordingRule{Name: "foo", state: newRuleState()},
 			[]datasource.Metric{metricWithValueAndLabels(t, 10,
 				"__name__", "bar",
 			)},
@@ -30,7 +30,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			&RecordingRule{Name: "foobarbaz"},
+			&RecordingRule{Name: "foobarbaz", state: newRuleState()},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
@@ -52,9 +52,12 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			&RecordingRule{Name: "job:foo", Labels: map[string]string{
-				"source": "test",
-			}},
+			&RecordingRule{
+				Name:  "job:foo",
+				state: newRuleState(),
+				Labels: map[string]string{
+					"source": "test",
+				}},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
@@ -77,7 +80,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO(), time.Now())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now(), 0)
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -88,7 +91,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecRange(t *testing.T) {
+func TestRecordingRule_ExecRange(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -169,16 +172,61 @@ func TestRecoridngRule_ExecRange(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecNegative(t *testing.T) {
-	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
-		"job": "test",
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
+		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+	}
+	rule := &RecordingRule{Name: "job:foo", state: newRuleState(), Labels: map[string]string{
+		"source": "test_limit",
 	}}
+	var err error
+	for _, testCase := range testCases {
+		fq := &fakeQuerier{}
+		fq.add(testMetrics...)
+		rule.q = fq
+		_, err = rule.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRecordingRule_ExecNegative(t *testing.T) {
+	rr := &RecordingRule{
+		Name:  "job:foo",
+		state: newRuleState(),
+		Labels: map[string]string{
+			"job": "test",
+		},
+	}
 
 	fq := &fakeQuerier{}
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO(), time.Now())
+	_, err := rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +241,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO(), time.Now())
+	_, err = rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remoteread/init.go

@@ -7,12 +7,21 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
-		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+		"Remote read is used to restore alerts state."+
+		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+		"E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
+
+	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("remoteRead.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteRead.url. "+
+		"For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. "+
+		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
@@ -34,10 +43,15 @@ var (
 	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
 	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
 	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
-
-	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteReadURL {
+		flagutil.RegisterSecretFlag("remoteRead.url")
+	}
+}
+
 // Init creates a Querier from provided flag values.
 // Returns nil if addr flag wasn't set.
 func Init() (datasource.QuerierBuilder, error) {
@@ -52,10 +66,11 @@ func Init() (datasource.QuerierBuilder, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c), nil
 }

app/vmalert/remotewrite/init.go

@@ -7,12 +7,19 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
-		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("remoteWrite.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
@@ -39,10 +46,15 @@ var (
 	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
 	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
 	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
-
-	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init creates Client object from given flags.
 // Returns nil if addr flag wasn't set.
 func Init(ctx context.Context) (*Client, error) {
@@ -58,7 +70,8 @@ func Init(ctx context.Context) (*Client, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/remotewrite/remotewrite.go

@@ -3,8 +3,9 @@ package remotewrite
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"strings"
@@ -19,17 +20,20 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
+var (
+	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
+)
+
 // Client is an asynchronous HTTP client for writing
 // timeseries via remote write protocol.
 type Client struct {
-	addr              string
-	c                 *http.Client
-	authCfg           *promauth.Config
-	input             chan prompbmarshal.TimeSeries
-	flushInterval     time.Duration
-	maxBatchSize      int
-	maxQueueSize      int
-	disablePathAppend bool
+	addr          string
+	c             *http.Client
+	authCfg       *promauth.Config
+	input         chan prompbmarshal.TimeSeries
+	flushInterval time.Duration
+	maxBatchSize  int
+	maxQueueSize  int
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
@@ -70,8 +74,6 @@ const (
 	defaultWriteTimeout  = 30 * time.Second
 )
 
-const writePath = "/api/v1/write"
-
 // NewClient returns asynchronous client for
 // writing timeseries via remotewrite protocol.
 func NewClient(ctx context.Context, cfg Config) (*Client, error) {
@@ -102,14 +104,13 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
-		addr:              strings.TrimSuffix(cfg.Addr, "/"),
-		authCfg:           cfg.AuthCfg,
-		flushInterval:     cfg.FlushInterval,
-		maxBatchSize:      cfg.MaxBatchSize,
-		maxQueueSize:      cfg.MaxQueueSize,
-		doneCh:            make(chan struct{}),
-		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
-		disablePathAppend: cfg.DisablePathAppend,
+		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		authCfg:       cfg.AuthCfg,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		maxQueueSize:  cfg.MaxQueueSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 	}
 
 	for i := 0; i < cc; i++ {
@@ -235,13 +236,19 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
+
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+
 	if c.authCfg != nil {
-		if auth := c.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		c.authCfg.SetHeaders(req, true)
 	}
-	if !c.disablePathAppend {
-		req.URL.Path = path.Join(req.URL.Path, writePath)
+	if !*disablePathAppend {
+		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
@@ -250,7 +257,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
 			resp.StatusCode, req.URL.Redacted(), body)
 	}

app/vmalert/remotewrite/remotewrite_test.go

@@ -3,7 +3,7 @@ package remotewrite
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
@@ -80,7 +80,23 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("bad method %q", r.Method))
 		return
 	}
-	data, err := ioutil.ReadAll(r.Body)
+
+	h := r.Header.Get("Content-Encoding")
+	if h != "snappy" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	}
+
+	h = r.Header.Get("Content-Type")
+	if h != "application/x-protobuf" {
+		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
+	}
+
+	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
+	if h != "0.1.0" {
+		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	}
+
+	data, err := io.ReadAll(r.Body)
 	if err != nil {
 		rw.err(w, fmt.Errorf("body read err: %w", err))
 		return

app/vmalert/replay.go

@@ -88,6 +88,10 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 		"\nrequests to make: \t%d"+
 		"\nmax range per request: \t%v\n",
 		g.Name, g.Interval, iterations, step)
+	if g.Limit > 0 {
+		fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
+			g.Limit)
+	}
 	for _, rule := range g.Rules {
 		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
 		var bar *pb.ProgressBar

app/vmalert/rule.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"errors"
+	"net/http"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -15,9 +17,10 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context at the given timestamp
-	Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error)
-	// ExecRange executes the rule on the given time range
+	// Exec executes the rule with given context at the given timestamp and limit.
+	// returns an err if number of resulting time series exceeds the limit.
+	Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error)
+	// ExecRange executes the rule on the given time range.
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
@@ -30,3 +33,74 @@ type Rule interface {
 }
 
 var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
+
+type ruleState struct {
+	sync.RWMutex
+	entries []ruleStateEntry
+	cur     int
+}
+
+type ruleStateEntry struct {
+	// stores last moment of time rule.Exec was called
+	time time.Time
+	// stores the timesteamp with which rule.Exec was called
+	at time.Time
+	// stores the duration of the last rule.Exec call
+	duration time.Duration
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health ruleState
+	err error
+	// stores the number of samples returned during
+	// the last evaluation
+	samples int
+	// stores the HTTP request used by datasource during rule.Exec
+	req *http.Request
+}
+
+const defaultStateEntriesLimit = 20
+
+func newRuleState() *ruleState {
+	return &ruleState{
+		entries: make([]ruleStateEntry, defaultStateEntriesLimit),
+	}
+}
+
+func (s *ruleState) getLast() ruleStateEntry {
+	s.RLock()
+	defer s.RUnlock()
+	return s.entries[s.cur]
+}
+
+func (s *ruleState) getAll() []ruleStateEntry {
+	entries := make([]ruleStateEntry, 0)
+
+	s.RLock()
+	defer s.RUnlock()
+
+	cur := s.cur
+	for {
+		e := s.entries[cur]
+		if !e.time.IsZero() || !e.at.IsZero() {
+			entries = append(entries, e)
+		}
+		cur--
+		if cur < 0 {
+			cur = cap(s.entries) - 1
+		}
+		if cur == s.cur {
+			return entries
+		}
+	}
+}
+
+func (s *ruleState) add(e ruleStateEntry) {
+	s.Lock()
+	defer s.Unlock()
+
+	s.cur++
+	if s.cur > cap(s.entries)-1 {
+		s.cur = 0
+	}
+	s.entries[s.cur] = e
+}

app/vmalert/rule_test.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestRule_state(t *testing.T) {
+	state := newRuleState()
+	e := state.getLast()
+	if !e.at.IsZero() {
+		t.Fatalf("expected entry to be zero")
+	}
+
+	now := time.Now()
+	state.add(ruleStateEntry{at: now})
+
+	e = state.getLast()
+	if e.at != now {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, now)
+	}
+
+	time.Sleep(time.Millisecond)
+	now2 := time.Now()
+	state.add(ruleStateEntry{at: now2})
+
+	e = state.getLast()
+	if e.at != now2 {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, now2)
+	}
+
+	if len(state.getAll()) != 2 {
+		t.Fatalf("expected for state to have 2 entries only; got %d",
+			len(state.getAll()),
+		)
+	}
+
+	var last time.Time
+	for i := 0; i < defaultStateEntriesLimit*2; i++ {
+		last = time.Now()
+		state.add(ruleStateEntry{at: last})
+	}
+
+	e = state.getLast()
+	if e.at != last {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, last)
+	}
+
+	if len(state.getAll()) != defaultStateEntriesLimit {
+		t.Fatalf("expected for state to have %d entries only; got %d",
+			defaultStateEntriesLimit, len(state.getAll()),
+		)
+	}
+}
+
+// TestRule_stateConcurrent supposed to test concurrent
+// execution of state updates.
+// Should be executed with -race flag
+func TestRule_stateConcurrent(t *testing.T) {
+	state := newRuleState()
+
+	const workers = 50
+	const iterations = 100
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				state.add(ruleStateEntry{at: time.Now()})
+				state.getAll()
+				state.getLast()
+			}
+		}()
+	}
+	wg.Wait()
+}

app/vmalert/templates/template.go

@@ -11,26 +11,118 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package notifier
+package templates
 
 import (
 	"errors"
 	"fmt"
+	htmlTpl "html/template"
+	"io"
 	"math"
 	"net"
 	"net/url"
+	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	htmlTpl "html/template"
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
+// go template execution fails when it's tree is empty
+const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
+
+var tplMu sync.RWMutex
+
+type textTemplate struct {
+	current     *textTpl.Template
+	replacement *textTpl.Template
+}
+
+var masterTmpl textTemplate
+
+func newTemplate() *textTpl.Template {
+	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(defaultTemplate))
+}
+
+// Load func loads templates from multiple globs specified in pathPatterns and either
+// sets them directly to current template if it's undefined or with overwrite=true
+// or sets replacement templates and adds templates with new names to a current
+func Load(pathPatterns []string, overwrite bool) error {
+	var err error
+	tmpl := newTemplate()
+	for _, tp := range pathPatterns {
+		p, err := filepath.Glob(tp)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
+		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseGlob(tp)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	if len(tmpl.Templates()) > 0 {
+		err := tmpl.Execute(io.Discard, nil)
+		if err != nil {
+			return fmt.Errorf("failed to execute template: %w", err)
+		}
+	}
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.current == nil || overwrite {
+		masterTmpl.replacement = nil
+		masterTmpl.current = newTemplate()
+	} else {
+		masterTmpl.replacement = newTemplate()
+		if err = copyTemplates(tmpl, masterTmpl.replacement, overwrite); err != nil {
+			return err
+		}
+	}
+	return copyTemplates(tmpl, masterTmpl.current, overwrite)
+}
+
+func copyTemplates(from *textTpl.Template, to *textTpl.Template, overwrite bool) error {
+	if from == nil {
+		return nil
+	}
+	if to == nil {
+		to = newTemplate()
+	}
+	tmpl, err := from.Clone()
+	if err != nil {
+		return err
+	}
+	for _, t := range tmpl.Templates() {
+		if to.Lookup(t.Name()) == nil || overwrite {
+			to, err = to.AddParseTree(t.Name(), t.Tree)
+			if err != nil {
+				return fmt.Errorf("failed to add template %q: %w", t.Name(), err)
+			}
+		}
+	}
+	return nil
+}
+
+// Reload func replaces current template with a replacement template
+// which was set by Load with override=false
+func Reload() {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.replacement != nil {
+		masterTmpl.current = masterTmpl.replacement
+		masterTmpl.replacement = nil
+	}
+}
+
 // metric is private copy of datasource.Metric,
 // it is used for templating annotations,
 // Labels as map simplifies templates evaluation.
@@ -60,12 +152,62 @@ func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
 // for templating functions.
 type QueryFn func(query string) ([]datasource.Metric, error)
 
-var tmplFunc textTpl.FuncMap
+// UpdateWithFuncs updates existing or sets a new function map for a template
+func UpdateWithFuncs(funcs textTpl.FuncMap) {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	masterTmpl.current = masterTmpl.current.Funcs(funcs)
+}
 
-// InitTemplateFunc initiates template helper functions
-func InitTemplateFunc(externalURL *url.URL) {
+// GetWithFuncs returns a copy of current template with additional FuncMap
+// provided with funcs argument
+func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	tmpl, err := masterTmpl.current.Clone()
+	if err != nil {
+		return nil, err
+	}
+	return tmpl.Funcs(funcs), nil
+}
+
+// Get returns a copy of a template
+func Get() (*textTpl.Template, error) {
+	tplMu.RLock()
+	defer tplMu.RUnlock()
+	return masterTmpl.current.Clone()
+}
+
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
+}
+
+// FuncsWithExternalURL returns a function map that depends on externalURL value
+func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
+}
+
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
 	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	tmplFunc = textTpl.FuncMap{
+	return textTpl.FuncMap{
 		/* Strings */
 
 		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
@@ -113,13 +255,26 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return d.Seconds(), nil
 		},
 
+		// same with parseDuration but returns a time.Duration
+		"parseDurationTime": func(s string) (time.Duration, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d, nil
+		},
+
 		/* Numbers */
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(v float64) string {
+		"humanize": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				prefix := ""
@@ -130,7 +285,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 					prefix = p
 					v /= 1000
 				}
-				return fmt.Sprintf("%.4g%s", v, prefix)
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -140,13 +295,17 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(v float64) string {
+		"humanize1024": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
@@ -156,16 +315,20 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v /= 1024
 			}
-			return fmt.Sprintf("%.4g%s", v, prefix)
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
 		},
 
 		// humanizeDuration converts given seconds to a human readable duration
-		"humanizeDuration": func(v float64) string {
+		"humanizeDuration": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
 			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+				return fmt.Sprintf("%.4g", v), nil
 			}
 			if v == 0 {
-				return fmt.Sprintf("%.4gs", v)
+				return fmt.Sprintf("%.4gs", v), nil
 			}
 			if math.Abs(v) >= 1 {
 				sign := ""
@@ -179,16 +342,16 @@ func InitTemplateFunc(externalURL *url.URL) {
 				days := int64(v) / 60 / 60 / 24
 				// For days to minutes, we display seconds as an integer.
 				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
 				}
 				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
 				}
 				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
 				}
 				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v)
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
 			}
 			prefix := ""
 			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
@@ -198,33 +361,64 @@ func InitTemplateFunc(externalURL *url.URL) {
 				prefix = p
 				v *= 1000
 			}
-			return fmt.Sprintf("%.4g%ss", v, prefix)
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(v float64) string {
-			return fmt.Sprintf("%.4g%%", v*100)
+		"humanizePercentage": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(v float64) string {
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v)
+		"humanizeTimestamp": func(i interface{}) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
 			}
-			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
-			return fmt.Sprint(t)
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return fmt.Sprint(t), nil
+		},
+
+		// toTime converts given timestamp to a time.Time.
+		"toTime": func(i interface{}) (time.Time, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return time.Time{}, err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return t, nil
 		},
 
 		/* URLs */
 
 		// externalURL returns value of `external.url` flag
 		"externalURL": func() string {
-			return externalURL.String()
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathPrefix returns a Path segment from the URL value in `external.url` flag
 		"pathPrefix": func() string {
-			return externalURL.Path
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
 		},
 
 		// pathEscape escapes the string so it can be safely placed inside a URL path segment,
@@ -259,7 +453,7 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// execute "/api/v1/query?query=foo" request and will return
 		// the first value in response.
 		"query": func(q string) ([]metric, error) {
-			// query function supposed to be substituted at funcsWithQuery().
+			// query function supposed to be substituted at FuncsWithQuery().
 			// it is present here only for validation purposes, when there is no
 			// provided datasource.
 			//
@@ -316,29 +510,13 @@ func InitTemplateFunc(externalURL *url.URL) {
 	}
 }
 
-func funcsWithQuery(query QueryFn) textTpl.FuncMap {
-	fm := make(textTpl.FuncMap)
-	for k, fn := range tmplFunc {
-		fm[k] = fn
-	}
-	fm["query"] = func(q string) ([]metric, error) {
-		result, err := query(q)
-		if err != nil {
-			return nil, err
-		}
-		return datasourceMetricsToTemplateMetrics(result), nil
-	}
-	return fm
-}
-
 // Time is the number of milliseconds since the epoch
 // (1970-01-01 00:00 UTC) excluding leap seconds.
 type Time int64
 
-// TimeFromUnixNano returns the Time equivalent to the Unix Time
-// t provided in nanoseconds.
-func TimeFromUnixNano(t int64) Time {
-	return Time(t / nanosPerTick)
+// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
+func timeFromUnixTimestamp(t float64) Time {
+	return Time(t * 1e3)
 }
 
 // The number of nanoseconds per minimum tick.
@@ -355,3 +533,28 @@ const second = int64(time.Second / minimumTick)
 func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
+
+func toFloat64(v interface{}) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
+}

app/vmalert/templates/template_test.go

@@ -0,0 +1,275 @@
+package templates
+
+import (
+	"strings"
+	"testing"
+	textTpl "text/template"
+)
+
+func mkTemplate(current, replacement interface{}) textTemplate {
+	tmpl := textTemplate{}
+	if current != nil {
+		switch val := current.(type) {
+		case string:
+			tmpl.current = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	if replacement != nil {
+		switch val := replacement.(type) {
+		case string:
+			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	return tmpl
+}
+
+func equalTemplates(tmpls ...*textTpl.Template) bool {
+	var cmp *textTpl.Template
+	for i, tmpl := range tmpls {
+		if i == 0 {
+			cmp = tmpl
+		} else {
+			if cmp == nil || tmpl == nil {
+				if cmp != tmpl {
+					return false
+				}
+				continue
+			}
+			if len(tmpl.Templates()) != len(cmp.Templates()) {
+				return false
+			}
+			for _, t := range tmpl.Templates() {
+				tp := cmp.Lookup(t.Name())
+				if tp == nil {
+					return false
+				}
+				if tp.Root.String() != t.Root.String() {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+func TestTemplates_Load(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		pathPatterns     []string
+		overwrite        bool
+		expectedTemplate textTemplate
+		expErr           string
+	}{
+		{
+			"non existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"non existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`, nil),
+			"",
+		},
+		{
+			"existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				`, `
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`),
+			"",
+		},
+		{
+			"load template with syntax error",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/bad0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			"failed to parse template glob",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			err := Load(tc.pathPatterns, tc.overwrite)
+			if tc.expErr == "" && err != nil {
+				t.Error("happened error that wasn't expected: %w", err)
+			}
+			if tc.expErr != "" && err == nil {
+				t.Error("%+w", err)
+				t.Error("expected error that didn't happened")
+			}
+			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
+				t.Error("%+w", err)
+				t.Error("expected string doesn't exist in error message")
+			}
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}
+
+func TestTemplates_Reload(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		expectedTemplate textTemplate
+	}{
+		{
+			"empty current and replacement templates",
+			mkTemplate(nil, nil),
+			mkTemplate(nil, nil),
+		},
+		{
+			"empty current template only",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"empty replacement template only",
+			mkTemplate(nil, `
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"defined both templates",
+			mkTemplate(`
+				{{- define "test.0" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{- printf "before" -}}
+				{{- end -}}
+			`, `
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`, nil),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			Reload()
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
+}

app/vmalert/templates/templates/other/nested/bad0-test.tpl

@@ -0,0 +1,3 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL" }}
+{{- end -}}

app/vmalert/templates/templates/other/nested/good0-test.tpl

@@ -0,0 +1,9 @@
+{{- define "test.1" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.0" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.3" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}

app/vmalert/templates/templates/test/good0-test.tpl

@@ -0,0 +1,9 @@
+{{- define "test.2" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.0" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}
+{{- define "test.3" -}}
+    {{ printf "Hello %s!" externalURL }}
+{{- end -}}

app/vmalert/tpl/footer.qtpl

@@ -1,7 +1,15 @@
-{% func Footer() %}
+{% import (
+    "net/http"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+) %}
+
+
+{% func Footer(r *http.Request) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
         </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="{%s prefix %}static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="{%s prefix %}static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
         <script type="text/javascript">
             function expandAll() {
                $('.collapse').addClass('show');
@@ -18,14 +26,14 @@
 
               $(".group-heading").click(function(e) {
                  let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                   new bootstrap.Collapse(el, {
                     toggle: true
                   });
               });
 
               var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
               if (group.length > 0) {
                 group.click();
               }

app/vmalert/tpl/footer.qtpl.go

@@ -5,25 +5,46 @@
 package tpl
 
 //line app/vmalert/tpl/footer.qtpl:1
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+//line app/vmalert/tpl/footer.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/tpl/footer.qtpl:1
+//line app/vmalert/tpl/footer.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/tpl/footer.qtpl:1
-func StreamFooter(qw422016 *qt422016.Writer) {
-//line app/vmalert/tpl/footer.qtpl:1
+//line app/vmalert/tpl/footer.qtpl:8
+func StreamFooter(qw422016 *qt422016.Writer, r *http.Request) {
+//line app/vmalert/tpl/footer.qtpl:8
+	qw422016.N().S(`
+    `)
+//line app/vmalert/tpl/footer.qtpl:9
+	prefix := utils.Prefix(r.URL.Path)
+
+//line app/vmalert/tpl/footer.qtpl:9
 	qw422016.N().S(`
         </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="`)
+//line app/vmalert/tpl/footer.qtpl:11
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/footer.qtpl:11
+	qw422016.N().S(`static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="`)
+//line app/vmalert/tpl/footer.qtpl:12
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/footer.qtpl:12
+	qw422016.N().S(`static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
         <script type="text/javascript">
             function expandAll() {
                $('.collapse').addClass('show');
@@ -40,14 +61,14 @@ func StreamFooter(qw422016 *qt422016.Writer) {
 
               $(".group-heading").click(function(e) {
                  let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                   new bootstrap.Collapse(el, {
                     toggle: true
                   });
               });
 
               var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
               if (group.length > 0) {
                 group.click();
               }
@@ -56,31 +77,31 @@ func StreamFooter(qw422016 *qt422016.Writer) {
     </body>
 </html>
 `)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }
 
-//line app/vmalert/tpl/footer.qtpl:36
-func WriteFooter(qq422016 qtio422016.Writer) {
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+func WriteFooter(qq422016 qtio422016.Writer, r *http.Request) {
+//line app/vmalert/tpl/footer.qtpl:44
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/footer.qtpl:36
-	StreamFooter(qw422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+	StreamFooter(qw422016, r)
+//line app/vmalert/tpl/footer.qtpl:44
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }
 
-//line app/vmalert/tpl/footer.qtpl:36
-func Footer() string {
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+func Footer(r *http.Request) string {
+//line app/vmalert/tpl/footer.qtpl:44
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/footer.qtpl:36
-	WriteFooter(qb422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+	WriteFooter(qb422016, r)
+//line app/vmalert/tpl/footer.qtpl:44
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 	return qs422016
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }

app/vmalert/tpl/header.qtpl

@@ -1,9 +1,19 @@
-{% func Header(title string, pages []NavItem) %}
+{% import (
+    "strings"
+    "net/http"
+    "path"
+    "net/url"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+) %}
+
+{% func Header(r *http.Request, navItems []NavItem, title string) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="{%s prefix %}static/css/bootstrap.min.css" rel="stylesheet" />
     <style>
         body{
           min-height: 75rem;
@@ -49,9 +59,53 @@
           background-color: rgba(0,0,0,.5);
           -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
         }
+        textarea.curl-area{
+            width: 100%;
+            line-height: 1;
+            font-size: 12px;
+            border: none;
+            margin: 0;
+            padding: 0;
+            overflow: scroll;
+        }
     </style>
 </head>
 <body>
-    {%= PrintNavItems(title, pages) %}
+    {%= printNavItems(r, title, navItems) %}
     <main class="px-2">
 {% endfunc %}
+
+
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func printNavItems(r *http.Request, current string, items []NavItem) %}
+{%code
+    prefix := "/vmalert/"
+    if strings.HasPrefix(r.URL.Path, prefix) {
+        prefix = ""
+    }
+%}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    {% code
+                    u, _ := url.Parse(item.Url)
+                    %}
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}"
+                       href="{% if u.IsAbs() %}{%s item.Url %}{% else %}{%s path.Join(prefix,item.Url) %}{% endif %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -5,37 +5,57 @@
 package tpl
 
 //line app/vmalert/tpl/header.qtpl:1
+import (
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+//line app/vmalert/tpl/header.qtpl:10
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/tpl/header.qtpl:1
+//line app/vmalert/tpl/header.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/tpl/header.qtpl:1
-func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
-//line app/vmalert/tpl/header.qtpl:1
+//line app/vmalert/tpl/header.qtpl:10
+func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string) {
+//line app/vmalert/tpl/header.qtpl:10
+	qw422016.N().S(`
+    `)
+//line app/vmalert/tpl/header.qtpl:11
+	prefix := utils.Prefix(r.URL.Path)
+
+//line app/vmalert/tpl/header.qtpl:11
 	qw422016.N().S(`
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <title>vmalert`)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	if title != "" {
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 		qw422016.N().S(` - `)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 		qw422016.E().S(title)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	}
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	qw422016.N().S(`</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="`)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.N().S(`static/css/bootstrap.min.css" rel="stylesheet" />
     <style>
         body{
           min-height: 75rem;
@@ -81,41 +101,152 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
           background-color: rgba(0,0,0,.5);
           -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
         }
+        textarea.curl-area{
+            width: 100%;
+            line-height: 1;
+            font-size: 12px;
+            border: none;
+            margin: 0;
+            padding: 0;
+            overflow: scroll;
+        }
     </style>
 </head>
 <body>
     `)
-//line app/vmalert/tpl/header.qtpl:55
-	StreamPrintNavItems(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:55
+//line app/vmalert/tpl/header.qtpl:74
+	streamprintNavItems(qw422016, r, title, navItems)
+//line app/vmalert/tpl/header.qtpl:74
 	qw422016.N().S(`
     <main class="px-2">
 `)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 }
 
-//line app/vmalert/tpl/header.qtpl:57
-func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string) {
+//line app/vmalert/tpl/header.qtpl:76
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:57
-	StreamHeader(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+	StreamHeader(qw422016, r, navItems, title)
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 }
 
-//line app/vmalert/tpl/header.qtpl:57
-func Header(title string, pages []NavItem) string {
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+func Header(r *http.Request, navItems []NavItem, title string) string {
+//line app/vmalert/tpl/header.qtpl:76
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:57
-	WriteHeader(qb422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+	WriteHeader(qb422016, r, navItems, title)
+//line app/vmalert/tpl/header.qtpl:76
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+}
+
+//line app/vmalert/tpl/header.qtpl:80
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/header.qtpl:86
+func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem) {
+//line app/vmalert/tpl/header.qtpl:86
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:88
+	prefix := "/vmalert/"
+	if strings.HasPrefix(r.URL.Path, prefix) {
+		prefix = ""
+	}
+
+//line app/vmalert/tpl/header.qtpl:92
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/header.qtpl:97
+	for _, item := range items {
+//line app/vmalert/tpl/header.qtpl:97
+		qw422016.N().S(`
+                <li class="nav-item">
+                    `)
+//line app/vmalert/tpl/header.qtpl:100
+		u, _ := url.Parse(item.Url)
+
+//line app/vmalert/tpl/header.qtpl:101
+		qw422016.N().S(`
+                    <a class="nav-link`)
+//line app/vmalert/tpl/header.qtpl:102
+		if current == item.Name {
+//line app/vmalert/tpl/header.qtpl:102
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/header.qtpl:102
+		}
+//line app/vmalert/tpl/header.qtpl:102
+		qw422016.N().S(`"
+                       href="`)
+//line app/vmalert/tpl/header.qtpl:103
+		if u.IsAbs() {
+//line app/vmalert/tpl/header.qtpl:103
+			qw422016.E().S(item.Url)
+//line app/vmalert/tpl/header.qtpl:103
+		} else {
+//line app/vmalert/tpl/header.qtpl:103
+			qw422016.E().S(path.Join(prefix, item.Url))
+//line app/vmalert/tpl/header.qtpl:103
+		}
+//line app/vmalert/tpl/header.qtpl:103
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/header.qtpl:104
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/header.qtpl:104
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/header.qtpl:107
+	}
+//line app/vmalert/tpl/header.qtpl:107
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/header.qtpl:111
+}
+
+//line app/vmalert/tpl/header.qtpl:111
+func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem) {
+//line app/vmalert/tpl/header.qtpl:111
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:111
+	streamprintNavItems(qw422016, r, current, items)
+//line app/vmalert/tpl/header.qtpl:111
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:111
+}
+
+//line app/vmalert/tpl/header.qtpl:111
+func printNavItems(r *http.Request, current string, items []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:111
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:111
+	writeprintNavItems(qb422016, r, current, items)
+//line app/vmalert/tpl/header.qtpl:111
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:111
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:111
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:111
 }

app/vmalert/tpl/nav.qtpl

@@ -1,25 +0,0 @@
-{% code
-type NavItem struct {
-	Name string
-	Url  string
-}
-%}
-
-{% func PrintNavItems(current string, items []NavItem) %}
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
-  <div class="container-fluid">
-    <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
-            {% for _, item := range items  %}
-                <li class="nav-item">
-                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
-                        {%s item.Name %}
-                    </a>
-                </li>
-            {% endfor %}
-        </ul>
-  </div>
-</nav>
-{% endfunc %}
-
-

app/vmalert/tpl/nav.qtpl.go

@@ -1,96 +0,0 @@
-// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vmalert/tpl/nav.qtpl:1
-package tpl
-
-//line app/vmalert/tpl/nav.qtpl:1
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vmalert/tpl/nav.qtpl:1
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vmalert/tpl/nav.qtpl:2
-type NavItem struct {
-	Name string
-	Url  string
-}
-
-//line app/vmalert/tpl/nav.qtpl:8
-func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
-//line app/vmalert/tpl/nav.qtpl:8
-	qw422016.N().S(`
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
-  <div class="container-fluid">
-    <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
-            `)
-//line app/vmalert/tpl/nav.qtpl:13
-	for _, item := range items {
-//line app/vmalert/tpl/nav.qtpl:13
-		qw422016.N().S(`
-                <li class="nav-item">
-                    <a class="nav-link`)
-//line app/vmalert/tpl/nav.qtpl:15
-		if current == item.Name {
-//line app/vmalert/tpl/nav.qtpl:15
-			qw422016.N().S(` active`)
-//line app/vmalert/tpl/nav.qtpl:15
-		}
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.N().S(`" href="`)
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.E().S(item.Url)
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.N().S(`">
-                        `)
-//line app/vmalert/tpl/nav.qtpl:16
-		qw422016.E().S(item.Name)
-//line app/vmalert/tpl/nav.qtpl:16
-		qw422016.N().S(`
-                    </a>
-                </li>
-            `)
-//line app/vmalert/tpl/nav.qtpl:19
-	}
-//line app/vmalert/tpl/nav.qtpl:19
-	qw422016.N().S(`
-        </ul>
-  </div>
-</nav>
-`)
-//line app/vmalert/tpl/nav.qtpl:23
-}
-
-//line app/vmalert/tpl/nav.qtpl:23
-func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
-//line app/vmalert/tpl/nav.qtpl:23
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/nav.qtpl:23
-	StreamPrintNavItems(qw422016, current, items)
-//line app/vmalert/tpl/nav.qtpl:23
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/nav.qtpl:23
-}
-
-//line app/vmalert/tpl/nav.qtpl:23
-func PrintNavItems(current string, items []NavItem) string {
-//line app/vmalert/tpl/nav.qtpl:23
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/nav.qtpl:23
-	WritePrintNavItems(qb422016, current, items)
-//line app/vmalert/tpl/nav.qtpl:23
-	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/nav.qtpl:23
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/nav.qtpl:23
-	return qs422016
-//line app/vmalert/tpl/nav.qtpl:23
-}