app/vmselect/vmui: make vmui-update after a54987f671

docs/CHANGELOG.md: cut v1.82.0
docs/CHANGELOG.md: typo fix
2026-06-09 11:54:31 +03:00 · 2022-10-07 03:31:17 +03:00 · 2022-10-07 03:15:23 +03:00 · 2022-10-07 03:11:22 +03:00 · 2022-10-07 02:50:59 +03:00 · 2022-10-07 01:23:52 +03:00
2082 changed files with 286666 additions and 150689 deletions
--- a/.github/workflows/check-licenses.yml
+++ b/.github/workflows/check-licenses.yml
@@ -17,7 +17,7 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@main
        with:
-          go-version: 1.17
+          go-version: 1.19.1
        id: go
      - name: Code checkout
        uses: actions/checkout@master
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -19,7 +19,7 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@main
        with:
-          go-version: 1.17
+          go-version: 1.19.1
        id: go
      - name: Code checkout
        uses: actions/checkout@master
@@ -29,8 +29,6 @@ jobs:
          make install-errcheck
          make install-golangci-lint
      - name: Build
-        env:
-          GO111MODULE: on
        run: |
          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
          make check-all
@@ -38,30 +36,8 @@ jobs:
          make test-full
          make test-pure
          make test-full-386
-          make victoria-metrics
-          make victoria-metrics-pure
-          make victoria-metrics-arm
-          make victoria-metrics-arm64
-          make vmutils
-          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=freebsd go build -mod=vendor ./app/vmagent
-          GOOS=freebsd go build -mod=vendor ./app/vmalert
-          GOOS=freebsd go build -mod=vendor ./app/vmbackup
-          GOOS=freebsd go build -mod=vendor ./app/vmrestore
-          GOOS=freebsd go build -mod=vendor ./app/vmctl
-          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
-          GOOS=openbsd go build -mod=vendor ./app/vmagent
-          GOOS=openbsd go build -mod=vendor ./app/vmalert
-          GOOS=openbsd go build -mod=vendor ./app/vmbackup
-          GOOS=openbsd go build -mod=vendor ./app/vmrestore
-          GOOS=openbsd go build -mod=vendor ./app/vmctl
-          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
-          GOOS=darwin go build -mod=vendor ./app/vmagent
-          GOOS=darwin go build -mod=vendor ./app/vmalert
-          GOOS=darwin go build -mod=vendor ./app/vmbackup
-          GOOS=darwin go build -mod=vendor ./app/vmrestore
-          GOOS=darwin go build -mod=vendor ./app/vmctl
-          CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
+          make victoria-metrics-crossbuild
+          make vmuitils-crossbuild
      - name: Publish coverage
        uses: codecov/codecov-action@v3
        with:
--- a/.gitignore
+++ b/.gitignore
@@ -19,4 +19,5 @@
 .DS_store
 Gemfile.lock
 /_site
-_site
+_site
+*.tmp
--- a/342
+++ b/342
@@ -13,6 +13,11 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA

 .PHONY: $(MAKECMDGOALS)

+include app/*/Makefile
+include deployment/*/Makefile
+include snap/local/Makefile
+include package/release/Makefile
+
 all: \
 	victoria-metrics-prod \
 	vmagent-prod \
@@ -22,15 +27,10 @@ all: \
 	vmrestore-prod \
 	vmctl-prod

-include app/*/Makefile
-include deployment/*/Makefile
-include snap/local/Makefile
-
-
 clean:
 	rm -rf bin/*

-publish: \
+publish: docker-scan \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
@@ -64,21 +64,77 @@ vmutils-pure: \
 	vmrestore-pure \
 	vmctl-pure

-vmutils-arm64: \
-	vmagent-arm64 \
-	vmalert-arm64 \
-	vmauth-arm64 \
-	vmbackup-arm64 \
-	vmrestore-arm64 \
-	vmctl-arm64
+vmutils-linux-amd64: \
+	vmagent-linux-amd64 \
+	vmalert-linux-amd64 \
+	vmauth-linux-amd64 \
+	vmbackup-linux-amd64 \
+	vmrestore-linux-amd64 \
+	vmctl-linux-amd64

-vmutils-arm: \
-	vmagent-arm \
-	vmalert-arm \
-	vmauth-arm \
-	vmbackup-arm \
-	vmrestore-arm \
-	vmctl-arm
+vmutils-linux-arm64: \
+	vmagent-linux-arm64 \
+	vmalert-linux-arm64 \
+	vmauth-linux-arm64 \
+	vmbackup-linux-arm64 \
+	vmrestore-linux-arm64 \
+	vmctl-linux-arm64
+
+vmutils-linux-arm: \
+	vmagent-linux-arm \
+	vmalert-linux-arm \
+	vmauth-linux-arm \
+	vmbackup-linux-arm \
+	vmrestore-linux-arm \
+	vmctl-linux-arm
+
+vmutils-linux-386: \
+	vmagent-linux-386 \
+	vmalert-linux-386 \
+	vmauth-linux-386 \
+	vmbackup-linux-386 \
+	vmrestore-linux-386 \
+	vmctl-linux-386
+
+vmutils-linux-ppc64le: \
+	vmagent-linux-ppc64le \
+	vmalert-linux-ppc64le \
+	vmauth-linux-ppc64le \
+	vmbackup-linux-ppc64le \
+	vmrestore-linux-ppc64le \
+	vmctl-linux-ppc64le
+
+vmutils-darwin-amd64: \
+	vmagent-darwin-amd64 \
+	vmalert-darwin-amd64 \
+	vmauth-darwin-amd64 \
+	vmbackup-darwin-amd64 \
+	vmrestore-darwin-amd64 \
+	vmctl-darwin-amd64
+
+vmutils-darwin-arm64: \
+	vmagent-darwin-arm64 \
+	vmalert-darwin-arm64 \
+	vmauth-darwin-arm64 \
+	vmbackup-darwin-arm64 \
+	vmrestore-darwin-arm64 \
+	vmctl-darwin-arm64
+
+vmutils-freebsd-amd64: \
+	vmagent-freebsd-amd64 \
+	vmalert-freebsd-amd64 \
+	vmauth-freebsd-amd64 \
+	vmbackup-freebsd-amd64 \
+	vmrestore-freebsd-amd64 \
+	vmctl-freebsd-amd64
+
+vmutils-openbsd-amd64: \
+	vmagent-openbsd-amd64 \
+	vmalert-openbsd-amd64 \
+	vmauth-openbsd-amd64 \
+	vmbackup-openbsd-amd64 \
+	vmrestore-openbsd-amd64 \
+	vmctl-openbsd-amd64

 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
@@ -86,6 +142,28 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64

+victoria-metrics-crossbuild: \
+	victoria-metrics-linux-amd64 \
+	victoria-metrics-linux-arm64 \
+	victoria-metrics-linux-arm \
+	victoria-metrics-linux-386 \
+	victoria-metrics-linux-ppc64le \
+	victoria-metrics-darwin-amd64 \
+	victoria-metrics-darwin-arm64 \
+	victoria-metrics-freebsd-amd64 \
+	victoria-metrics-openbsd-amd64
+
+vmutils-crossbuild: \
+	vmutils-linux-amd64 \
+	vmutils-linux-arm64 \
+	vmutils-linux-arm \
+	vmutils-linux-386 \
+	vmutils-linux-ppc64le \
+	vmutils-darwin-amd64 \
+	vmutils-darwin-arm64 \
+	vmutils-freebsd-amd64 \
+	vmutils-openbsd-amd64 \
+	vmutils-windows-amd64

 publish-release:
 	git checkout $(TAG) && $(MAKE) release publish && \
@@ -98,86 +176,110 @@ release: \
 	release-vmutils

 release-victoria-metrics: \
-	release-victoria-metrics-amd64 \
-	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64 \
+	release-victoria-metrics-linux-amd64 \
+	release-victoria-metrics-linux-arm \
+	release-victoria-metrics-linux-arm64 \
 	release-victoria-metrics-darwin-amd64 \
-	release-victoria-metrics-darwin-arm64
+	release-victoria-metrics-darwin-arm64 \
+	release-victoria-metrics-freebsd-amd64 \
+	release-victoria-metrics-openbsd-amd64

-release-victoria-metrics-amd64:
-	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch

-release-victoria-metrics-arm:
-	OSARCH=arm $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-victoria-metrics-goos-goarch

-release-victoria-metrics-arm64:
-	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic
+release-victoria-metrics-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch

 release-victoria-metrics-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch

 release-victoria-metrics-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch

-release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
+release-victoria-metrics-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
+
+release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod

 release-vmutils: \
-	release-vmutils-amd64 \
-	release-vmutils-arm64 \
-	release-vmutils-arm \
-	release-vmutils-darwin-amd64  \
+	release-vmutils-linux-amd64 \
+	release-vmutils-linux-arm64 \
+	release-vmutils-linux-arm \
+	release-vmutils-darwin-amd64 \
 	release-vmutils-darwin-arm64 \
+	release-vmutils-freebsd-amd64 \
+	release-vmutils-openbsd-amd64 \
 	release-vmutils-windows-amd64

-release-vmutils-amd64:
-	OSARCH=amd64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch

-release-vmutils-arm64:
-	OSARCH=arm64 $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch

-release-vmutils-arm:
-	OSARCH=arm $(MAKE) release-vmutils-generic
+release-vmutils-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-vmutils-goos-goarch

 release-vmutils-darwin-amd64:
-	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch

 release-vmutils-darwin-arm64:
-	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
+
+release-vmutils-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch

 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic
+	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch

-release-vmutils-generic: \
-	vmagent-$(OSARCH)-prod \
-	vmalert-$(OSARCH)-prod \
-	vmauth-$(OSARCH)-prod \
-	vmbackup-$(OSARCH)-prod \
-	vmrestore-$(OSARCH)-prod \
-	vmctl-$(OSARCH)-prod
+release-vmutils-goos-goarch: \
+	vmagent-$(GOOS)-$(GOARCH)-prod \
+	vmalert-$(GOOS)-$(GOARCH)-prod \
+	vmauth-$(GOOS)-$(GOARCH)-prod \
+	vmbackup-$(GOOS)-$(GOARCH)-prod \
+	vmrestore-$(GOOS)-$(GOARCH)-prod \
+	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(OSARCH)-prod \
-			vmalert-$(OSARCH)-prod \
-			vmauth-$(OSARCH)-prod \
-			vmbackup-$(OSARCH)-prod \
-			vmrestore-$(OSARCH)-prod \
-			vmctl-$(OSARCH)-prod \
-			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(GOOS)-$(GOARCH)-prod \
+			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmauth-$(GOOS)-$(GOARCH)-prod \
+			vmbackup-$(GOOS)-$(GOARCH)-prod \
+			vmrestore-$(GOOS)-$(GOARCH)-prod \
+			vmctl-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-$(GOOS)-$(GOARCH)-prod \
+		vmalert-$(GOOS)-$(GOARCH)-prod \
+		vmauth-$(GOOS)-$(GOARCH)-prod \
+		vmbackup-$(GOOS)-$(GOARCH)-prod \
+		vmrestore-$(GOOS)-$(GOARCH)-prod \
+		vmctl-$(GOOS)-$(GOARCH)-prod

-release-vmutils-windows-generic: \
+release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
@@ -194,112 +296,122 @@ release-vmutils-windows-generic: \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vmagent-windows-$(GOARCH)-prod.exe \
+		vmalert-windows-$(GOARCH)-prod.exe \
+		vmauth-windows-$(GOARCH)-prod.exe \
+		vmctl-windows-$(GOARCH)-prod.exe


 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)

 fmt:
-	GO111MODULE=on gofmt -l -w -s ./lib
-	GO111MODULE=on gofmt -l -w -s ./app
+	gofmt -l -w -s ./lib
+	gofmt -l -w -s ./app

 vet:
-	GO111MODULE=on go vet -mod=vendor ./lib/...
-	GO111MODULE=on go vet -mod=vendor ./app/...
+	go vet ./lib/...
+	go vet ./app/...

 lint: install-golint
 	golint lib/...
 	golint app/...

 install-golint:
-	which golint || GO111MODULE=off go get golang.org/x/lint/golint
+	which golint || go install golang.org/x/lint/golint@latest

 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
+	errcheck -exclude=errcheck_excludes.txt ./app/...

 install-errcheck:
-	which errcheck || GO111MODULE=off go get github.com/kisielk/errcheck
+	which errcheck || go install github.com/kisielk/errcheck@latest

-check-all: fmt vet lint errcheck golangci-lint
+check-all: fmt vet lint errcheck golangci-lint govulncheck

 test:
-	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+	go test ./lib/... ./app/...

 test-race:
-	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...

 test-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...

 test-full:
-	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...

 test-full-386:
-	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...

 benchmark:
-	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+	go test -bench=. ./lib/...
+	go test -bench=. ./app/...

 benchmark-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
-	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./app/...

 vendor-update:
-	GO111MODULE=on go get -u -d ./lib/...
-	GO111MODULE=on go get -u -d ./app/...
-	GO111MODULE=on go mod tidy -compat=1.17
-	GO111MODULE=on go mod vendor
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.19
+	go mod vendor

 app-local:
-	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)

 app-local-pure:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)

-app-local-with-goarch:
-	GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-goos-goarch:
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)

-app-local-windows-with-goarch:
-	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+app-local-windows-goarch:
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)

 quicktemplate-gen: install-qtc
 	qtc

 install-qtc:
-	which qtc || GO111MODULE=off go get github.com/valyala/quicktemplate/qtc
+	which qtc || go install github.com/valyala/quicktemplate/qtc@latest


 golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m

 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
+
+govulncheck: install-govulncheck
+	govulncheck ./...
+
+install-govulncheck:
+	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest

 install-wwhrd:
-	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
+	which wwhrd || go install github.com/frapposelli/wwhrd@latest

 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml

 copy-docs:
-	echo "---\nsort: ${ORDER}\n---\n" > ${DST}
+	echo '' > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "---\nsort: ${ORDER}\n---\n" > ${DST}; \
+	fi
 	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp

 # Copies docs for all components and adds the order tag.
+# For ORDER=0 it adds no order tag.
+# Images starting with <img src="docs/ are replaced with <img src="
 # Cluster docs are supposed to be ordered as 9th.
-# For The rest of docs is ordered manually.t
+# The rest of docs is ordered manually.
 docs-sync:
-	cp README.md docs/README.md
+	SRC=README.md DST=docs/README.md ORDER=0 $(MAKE) copy-docs
 	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md ORDER=1 $(MAKE) copy-docs
 	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
 	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
--- a/README.md
+++ b/README.md
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+|---------|--------------------|
+| 1.81.x  | :white_check_mark: |
+| 1.80.x  | :x:                |
+| 1.79.x  | :white_check_mark: |
+| < 1.78  | :x:                |
+
+## Reporting a Vulnerability
+
+Please report any security issues to security@victoriametrics.com
--- a/app/victoria-metrics/Makefile
+++ b/app/victoria-metrics/Makefile
@@ -12,20 +12,20 @@ victoria-metrics-prod:
 victoria-metrics-pure-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure

-victoria-metrics-amd64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-amd64
+victoria-metrics-linux-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-amd64

-victoria-metrics-arm-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+victoria-metrics-linux-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm

-victoria-metrics-arm64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+victoria-metrics-linux-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-arm64

-victoria-metrics-ppc64le-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+victoria-metrics-linux-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-ppc64le

-victoria-metrics-386-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+victoria-metrics-linux-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-386

 victoria-metrics-darwin-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ victoria-metrics-darwin-amd64-prod:
 victoria-metrics-darwin-arm64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64

+victoria-metrics-freebsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-freebsd-amd64
+
+victoria-metrics-openbsd-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker

@@ -64,60 +70,68 @@ run-victoria-metrics:
 	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
 	$(MAKE) run-via-docker

-victoria-metrics-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-amd64 ./app/victoria-metrics
+victoria-metrics-linux-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

-victoria-metrics-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+victoria-metrics-linux-arm:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch

-victoria-metrics-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
+victoria-metrics-linux-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch

-victoria-metrics-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
+victoria-metrics-linux-ppc64le:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch

-victoria-metrics-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+victoria-metrics-linux-386:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-darwin-arm64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-freebsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-metrics-openbsd-amd64:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch

 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure

 ### Packaging as DEB - amd64
-victoria-metrics-package-deb: victoria-metrics-prod
+victoria-metrics-package-deb-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_deb.sh amd64

 ### Packaging as DEB - arm64
-victoria-metrics-package-deb-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-deb-arm: victoria-metrics-linux-arm-prod
+	./package/package_deb.sh arm
+
+### Packaging as DEB - arm64
+victoria-metrics-package-deb-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_deb.sh arm64

 ### Packaging as DEB - all
-victoria-metrics-package-deb-all: \
-        victoria-metrics-package-deb \
+victoria-metrics-package-deb: \
+        victoria-metrics-package-deb-amd64 \
+        victoria-metrics-package-deb-arm \
        victoria-metrics-package-deb-arm64

 ### Packaging as RPM - amd64
-victoria-metrics-package-rpm: victoria-metrics-prod
+victoria-metrics-package-rpm-amd64: victoria-metrics-linux-amd64-prod
 	./package/package_rpm.sh amd64

 ### Packaging as RPM - arm64
-victoria-metrics-package-rpm-arm64: victoria-metrics-arm64-prod
+victoria-metrics-package-rpm-arm64: victoria-metrics-linux-arm64-prod
 	./package/package_rpm.sh arm64

 ### Packaging as RPM - all
-victoria-metrics-package-rpm-all: \
-        victoria-metrics-package-rpm \
+victoria-metrics-package-rpm: \
+        victoria-metrics-package-rpm-amd64 \
        victoria-metrics-package-rpm-arm64

 ### Packaging as both DEB and RPM - all
-victoria-metrics-package-deb-rpm-all: \
+victoria-metrics-package-deb-rpm: \
        victoria-metrics-package-deb \
-        victoria-metrics-package-deb-arm64 \
-        victoria-metrics-package-rpm \
-        victoria-metrics-package-rpm-arm64
-
-### Packaging as snap
-victoria-metrics-package-snap:
-	which snapcraft || snap install snapcraft
-	which multipass || snap install multipass
-	snapcraft
-
+        victoria-metrics-package-rpm
--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -19,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )

@@ -37,6 +38,7 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()

 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -92,7 +94,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"vmui", "Web UI"},
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
--- a/app/victoria-metrics/main_test.go
+++ b/app/victoria-metrics/main_test.go
@@ -6,7 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -139,7 +138,7 @@ func setUp() {
 		if err != nil {
 			return false
 		}
-		resp.Body.Close()
+		_ = resp.Body.Close()
 		return resp.StatusCode == 200
 	}
 	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
@@ -308,7 +307,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 		if filepath.Ext(path) != ".json" {
 			return nil
 		}
-		b, err := ioutil.ReadFile(path)
+		b, err := os.ReadFile(path)
 		s.noError(err)
 		item := test{}
 		s.noError(json.Unmarshal(b, &item))
@@ -338,7 +337,9 @@ func tcpWrite(t *testing.T, address string, data string) {
 	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
 	s.noError(err)
-	defer conn.Close()
+	defer func() {
+		_ = conn.Close()
+	}()
 	n, err := conn.Write([]byte(data))
 	s.noError(err)
 	s.equalInt(n, len(data))
@@ -349,7 +350,9 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
 	for dec := json.NewDecoder(resp.Body); dec.More(); {
@@ -364,7 +367,9 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
--- a/app/victoria-metrics/multiarch/Dockerfile
+++ b/app/victoria-metrics/multiarch/Dockerfile
@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
 ARG TARGETARCH
-COPY victoria-metrics-${TARGETARCH}-prod ./victoria-metrics-prod
+COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod
--- a/app/victoria-metrics/self_scraper.go
+++ b/app/victoria-metrics/self_scraper.go
@@ -6,8 +6,8 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -60,7 +60,7 @@ func selfScraper(scrapeInterval time.Duration) {
 			currentTimestamp = currentTime.UnixNano() / 1e6
 		}
 		bb.Reset()
-		httpserver.WritePrometheusMetrics(&bb)
+		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
 		rows.Unmarshal(s)
@@ -85,7 +85,9 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr.Timestamp = currentTimestamp
 			mr.Value = r.Value
 		}
-		vmstorage.AddRows(mrs)
+		if err := vmstorage.AddRows(mrs); err != nil {
+			logger.Errorf("cannot store self-scraped metrics: %s", err)
+		}
 	}
 }

--- a/app/vmagent/Makefile
+++ b/app/vmagent/Makefile
@@ -12,20 +12,20 @@ vmagent-prod:
 vmagent-pure-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-pure

-vmagent-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+vmagent-linux-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-amd64

-vmagent-arm-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+vmagent-linux-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm

-vmagent-arm64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+vmagent-linux-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm64

-vmagent-ppc64le-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+vmagent-linux-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-ppc64le

-vmagent-386-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-386
+vmagent-linux-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386

 vmagent-darwin-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmagent-darwin-amd64-prod:
 vmagent-darwin-arm64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64

+vmagent-freebsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-freebsd-amd64
+
+vmagent-openbsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-openbsd-amd64
+
 vmagent-windows-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64

@@ -67,26 +73,35 @@ run-vmagent:
 	APP_NAME=vmagent \
 	$(MAKE) run-via-docker

-vmagent-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-amd64:
+	APP_NAME=vmagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

-vmagent-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch

-vmagent-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch

-vmagent-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmagent-local-with-goarch
+vmagent-linux-ppc64le:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch

-vmagent-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmagent-local-with-goarch
+vmagent-linux-386:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch

-vmagent-local-with-goarch:
-	APP_NAME=vmagent $(MAKE) app-local-with-goarch
+vmagent-darwin-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-darwin-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmagent-freebsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-openbsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-goarch

 vmagent-pure:
 	APP_NAME=vmagent $(MAKE) app-local-pure
-
-vmagent-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
--- a/app/vmagent/csvimport/request_handler.go
+++ b/app/vmagent/csvimport/request_handler.go
@@ -67,7 +67,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))
--- a/app/vmagent/datadog/request_handler.go
+++ b/app/vmagent/datadog/request_handler.go
@@ -1,9 +1,7 @@
 package datadog

 import (
-	"fmt"
 	"net/http"
-	"strings"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
@@ -59,12 +57,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 			Value: ss.Host,
 		})
 		for _, tag := range ss.Tags {
-			n := strings.IndexByte(tag, ':')
-			if n < 0 {
-				return fmt.Errorf("cannot find ':' in tag %q", tag)
-			}
-			name := tag[:n]
-			value := tag[n+1:]
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -89,7 +82,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
--- a/app/vmagent/graphite/request_handler.go
+++ b/app/vmagent/graphite/request_handler.go
@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
--- a/app/vmagent/influx/request_handler.go
+++ b/app/vmagent/influx/request_handler.go
@@ -134,7 +134,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -1,6 +1,7 @@
 package main

 import (
+	"embed"
 	"flag"
 	"fmt"
 	"io"
@@ -23,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -35,6 +37,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -63,6 +66,12 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )

+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -71,6 +80,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()

 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -104,10 +114,12 @@ func main() {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, httpInsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, httpInsertHandler)
 	}

 	promscrape.Init(remotewrite.Push)
@@ -149,6 +161,40 @@ func main() {
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }

+func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
+	if !remotewrite.MultitenancyEnabled() {
+		return func(req *http.Request) error {
+			path := strings.Replace(req.URL.Path, "//", "/", -1)
+			if path != "/api/put" {
+				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
+			}
+			return opentsdbhttp.InsertHandler(nil, req)
+		}
+	}
+	return func(req *http.Request) error {
+		path := strings.Replace(req.URL.Path, "//", "/", -1)
+		at, err := getAuthTokenFromPath(path)
+		if err != nil {
+			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
+		}
+		return opentsdbhttp.InsertHandler(at, req)
+	}
+}
+
+func getAuthTokenFromPath(path string) (*auth.Token, error) {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
+	}
+	if p.Prefix != "insert" {
+		return nil, fmt.Errorf(`unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+	}
+	if p.Suffix != "opentsdb/api/put" {
+		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
+	}
+	return auth.NewToken(p.AuthToken)
+}
+
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
 		if r.Method != "GET" {
@@ -159,7 +205,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -170,8 +217,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}

 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
 	switch path {
-	case "/api/v1/write":
+	case "/prometheus/api/v1/write", "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -180,7 +232,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import":
+	case "/prometheus/api/v1/import", "/api/v1/import":
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
@@ -189,7 +241,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/csv":
+	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
@@ -198,7 +250,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/prometheus":
+	case "/prometheus/api/v1/import/prometheus", "/api/v1/import/prometheus":
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(nil, r); err != nil {
 			prometheusimportErrors.Inc()
@@ -207,7 +259,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/api/v1/import/native":
+	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
@@ -216,7 +268,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/write", "/api/v2/write":
+	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
@@ -225,7 +277,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/query":
+	case "/influx/query", "/query":
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
@@ -254,16 +306,31 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/intake/":
+	case "/datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
-	case "/targets":
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "/prometheus/targets", "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
-	case "/target_response":
+	case "/prometheus/service-discovery", "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
+	case "/prometheus/api/v1/targets", "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
+		return true
+	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
 			promscrapeTargetResponseErrors.Inc()
@@ -271,7 +338,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
-	case "/config":
+	case "/prometheus/config", "/config":
 		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
 			err := &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
@@ -284,13 +351,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
-	case "/api/v1/targets":
-		promscrapeAPIV1TargetsRequests.Inc()
+	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		state := r.FormValue("state")
-		promscrape.WriteAPIV1Targets(w, state)
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
-	case "/-/reload":
+	case "/prometheus/-/reload", "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -305,11 +382,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			w.Write([]byte("OK"))
 		}
 		return true
+	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+		if remotewrite.MultitenancyEnabled() {
+			return processMultitenantRequest(w, r, path)
+		}
+		return false
 	}
-	if remotewrite.MultitenancyEnabled() {
-		return processMultitenantRequest(w, r, path)
-	}
-	return false
 }

 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
@@ -327,6 +409,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
+	if strings.HasPrefix(p.Suffix, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
+	}
 	switch p.Suffix {
 	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -410,11 +497,16 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/intake/":
+	case "datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -447,15 +539,18 @@ var (

 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)

-	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)

 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)

-	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)

 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
--- a/app/vmagent/multiarch/Dockerfile
+++ b/app/vmagent/multiarch/Dockerfile
@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-COPY vmagent-${TARGETARCH}-prod ./vmagent-prod
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod
--- a/app/vmagent/native/request_handler.go
+++ b/app/vmagent/native/request_handler.go
@@ -87,6 +87,6 @@ func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }
--- a/app/vmagent/opentsdb/request_handler.go
+++ b/app/vmagent/opentsdb/request_handler.go
@@ -58,7 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
--- a/app/vmagent/opentsdbhttp/request_handler.go
+++ b/app/vmagent/opentsdbhttp/request_handler.go
@@ -5,6 +5,7 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
@@ -19,19 +20,19 @@ var (

 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }

-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

@@ -65,7 +66,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
--- a/app/vmagent/prometheusimport/request_handler.go
+++ b/app/vmagent/prometheusimport/request_handler.go
@@ -82,7 +82,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))
--- a/app/vmagent/promremotewrite/request_handler.go
+++ b/app/vmagent/promremotewrite/request_handler.go
@@ -81,7 +81,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
--- a/app/vmagent/remotewrite/client.go
+++ b/app/vmagent/remotewrite/client.go
@@ -3,7 +3,7 @@ package remotewrite
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -20,60 +20,50 @@ import (
 )

 var (
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to -remoteWrite.url. "+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to -remoteWrite.url")
-	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. "+
-		"Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")

-	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsCAFile = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
-		"By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to -remoteWrite.url. "+
-		"By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")

-	basicAuthUsername = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for -remoteWrite.url. "+
-		"The file is re-read every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for -remoteWrite.url. "+
-		"The token is re-read from the file every second. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")

-	oauth2ClientID = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecret = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2TokenURL = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -remoteWrite.url. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
+	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+		"The token is re-read from the file every second")

-	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for -remoteWrite.url. "+
-		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
-	awsService = flagutil.NewArray("remoteWrite.aws.serice", "Optional AWS Service to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url. Defaults to \"aps\".")
-	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+
+	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
+	awsEC2Endpoint = flagutil.NewArrayString("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSTSEndpoint = flagutil.NewArrayString("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRegion      = flagutil.NewArrayString("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN     = flagutil.NewArrayString("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey   = flagutil.NewArrayString("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService     = flagutil.NewArrayString("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"Defaults to \"aps\"")
+	awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
 )

 type client struct {
@@ -166,6 +156,9 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
+	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(*queues)
+	})
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -183,6 +176,11 @@ func (c *client) MustStop() {
 }

 func getAuthConfig(argIdx int) (*promauth.Config, error) {
+	headersValue := headers.GetOptionalArg(argIdx)
+	var hdrs []string
+	if headersValue != "" {
+		hdrs = strings.Split(headersValue, "^^")
+	}
 	username := basicAuthUsername.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
 	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
@@ -219,7 +217,15 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}

-	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		OAuth2:          oauth2Cfg,
+		TLSConfig:       tlsCfg,
+		Headers:         hdrs,
+	}
+	authCfg, err := opts.NewConfig()
 	if err != nil {
 		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
@@ -230,12 +236,14 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	if !awsUseSigv4.GetOptionalArg(argIdx) {
 		return nil, nil
 	}
+	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
 	region := awsRegion.GetOptionalArg(argIdx)
 	roleARN := awsRoleARN.GetOptionalArg(argIdx)
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
 	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, service)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
 	if err != nil {
 		return nil, err
 	}
@@ -301,14 +309,12 @@ again:
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
+	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
 	h.Set("Content-Encoding", "snappy")
 	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	if ah := c.authCfg.GetAuthHeader(); ah != "" {
-		req.Header.Set("Authorization", ah)
-	}
 	if c.awsCfg != nil {
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
 			// there is no need in retry, request will be rejected by client.Do and retried by code below
@@ -345,15 +351,14 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 		if err != nil {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
 				"failed to read response body: %s",
 				len(block), c.sanitizedURL, statusCode, err)
 		} else {
-			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
 				len(block), c.sanitizedURL, statusCode, string(body))
 		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
@@ -370,7 +375,7 @@ again:
 	if retryDuration > time.Minute {
 		retryDuration = time.Minute
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
 		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
@@ -390,6 +395,8 @@ again:
 	goto again
 }

+var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
 type rateLimiter struct {
 	perSecondLimit int64

--- a/app/vmagent/remotewrite/pendingseries_test.go
+++ b/app/vmagent/remotewrite/pendingseries_test.go
@@ -0,0 +1,62 @@
+package remotewrite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+func TestPushWriteRequest(t *testing.T) {
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
+			testPushWriteRequest(t, rowsCount)
+		})
+	}
+}
+
+func testPushWriteRequest(t *testing.T, rowsCount int) {
+	wr := newTestWriteRequest(rowsCount, 10)
+	pushBlockLen := 0
+	pushBlock := func(block []byte) {
+		if pushBlockLen > 0 {
+			panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+		}
+		pushBlockLen = len(block)
+	}
+	pushWriteRequest(wr, pushBlock)
+	b := prompbmarshal.MarshalWriteRequest(nil, wr)
+	zb := snappy.Encode(nil, b)
+	maxPushBlockLen := len(zb)
+	minPushBlockLen := maxPushBlockLen / 2
+	if pushBlockLen < minPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be at least %d bytes", pushBlockLen, minPushBlockLen)
+	}
+	if pushBlockLen > maxPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be smaller or equal to %d bytes", pushBlockLen, maxPushBlockLen)
+	}
+}
+
+func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
+	var wr prompbmarshal.WriteRequest
+	for i := 0; i < seriesCount; i++ {
+		var labels []prompbmarshal.Label
+		for j := 0; j < labelsCount; j++ {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  fmt.Sprintf("label_%d_%d", i, j),
+				Value: fmt.Sprintf("value_%d_%d", i, j),
+			})
+		}
+		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
+			Labels: labels,
+			Samples: []prompbmarshal.Sample{
+				{
+					Value:     float64(i),
+					Timestamp: 1000 * int64(i),
+				},
+			},
+		})
+	}
+	return &wr
+}
--- a/app/vmagent/remotewrite/pendingseries_timing_test.go
+++ b/app/vmagent/remotewrite/pendingseries_timing_test.go
@@ -0,0 +1,36 @@
+package remotewrite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+	"github.com/klauspost/compress/s2"
+)
+
+func BenchmarkCompressWriteRequestSnappy(b *testing.B) {
+	b.Run("snappy", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, snappy.Encode)
+	})
+	b.Run("s2", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, s2.EncodeSnappy)
+	})
+}
+
+func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []byte) []byte) {
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
+			wr := newTestWriteRequest(rowsCount, 10)
+			data := prompbmarshal.MarshalWriteRequest(nil, wr)
+			b.ReportAllocs()
+			b.SetBytes(int64(rowsCount))
+			b.RunParallel(func(pb *testing.PB) {
+				var zb []byte
+				for pb.Next() {
+					zb = compressFunc(zb[:cap(zb)], data)
+				}
+			})
+		})
+	}
+}
--- a/app/vmagent/remotewrite/relabel.go
+++ b/app/vmagent/remotewrite/relabel.go
@@ -13,18 +13,22 @@ import (
 )

 var (
-	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
 		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
 		"The path can point either to local file or to http url")
 	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
+
+	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
+		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
+		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
 )

 var labelsGlobal []prompbmarshal.Label
@@ -107,6 +111,18 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 				labels = append(labels, *extraLabel)
 			}
 		}
+		if *usePromCompatibleNaming {
+			// Replace unsupported Prometheus chars in label names and metric names with underscores.
+			tmpLabels := labels[labelsLen:]
+			for j := range tmpLabels {
+				label := &tmpLabels[j]
+				if label.Name == "__name__" {
+					label.Value = promrelabel.SanitizeName(label.Value)
+				} else {
+					label.Name = promrelabel.SanitizeName(label.Name)
+				}
+			}
+		}
 		labels = pcs.Apply(labels, labelsLen, true)
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
--- a/app/vmagent/remotewrite/remotewrite.go
+++ b/app/vmagent/remotewrite/remotewrite.go
@@ -22,14 +22,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
-	xxhash "github.com/cespare/xxhash/v2"
+	"github.com/cespare/xxhash/v2"
 )

 var (
-	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
-	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
@@ -38,7 +38,7 @@ var (
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
-	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
 		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
@@ -234,15 +234,11 @@ func Stop() {

 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// Note that wr may be modified by Push due to relabeling and rounding.
-func Push(wr *prompbmarshal.WriteRequest) {
-	PushWithAuthToken(nil, wr)
-}
-
-// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
+// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
+// If at isn't nil, the the data is pushed to the configured `-remoteWrite.multitenantURL`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
-func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
 		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
 		at = defaultAuthToken
@@ -252,7 +248,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rwctxs = rwctxsDefault
 	} else {
 		if len(*remoteWriteMultitenantURLs) == 0 {
-			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
 		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
@@ -440,7 +436,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
 	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
--- a/app/vmagent/static/css/bootstrap.min.css
+++ b/app/vmagent/static/css/bootstrap.min.css
--- a/app/vmagent/vmimport/request_handler.go
+++ b/app/vmagent/vmimport/request_handler.go
@@ -88,7 +88,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
--- a/app/vmalert/Makefile
+++ b/app/vmalert/Makefile
@@ -12,20 +12,20 @@ vmalert-prod:
 vmalert-pure-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-pure

-vmalert-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+vmalert-linux-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-amd64

-vmalert-arm-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+vmalert-linux-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm

-vmalert-arm64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+vmalert-linux-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm64

-vmalert-ppc64le-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+vmalert-linux-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-ppc64le

-vmalert-386-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-386
+vmalert-linux-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386

 vmalert-darwin-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmalert-darwin-amd64-prod:
 vmalert-darwin-arm64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64

+vmalert-freebsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-freebsd-amd64
+
+vmalert-openbsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-openbsd-amd64
+
 vmalert-windows-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64

@@ -88,7 +94,7 @@ run-vmalert-sd: vmalert
 		-configCheckInterval=10s

 replay-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
@@ -96,26 +102,35 @@ replay-vmalert: vmalert
 		-replay.timeFrom=2021-05-11T07:21:43Z \
 		-replay.timeTo=2021-05-29T18:40:43Z

-vmalert-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-amd64:
+	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

-vmalert-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch

-vmalert-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch

-vmalert-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmalert-local-with-goarch
+vmalert-linux-ppc64le:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch

-vmalert-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmalert-local-with-goarch
+vmalert-linux-386:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch

-vmalert-local-with-goarch:
-	APP_NAME=vmalert $(MAKE) app-local-with-goarch
+vmalert-darwin-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-darwin-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-freebsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-openbsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-goarch

 vmalert-pure:
 	APP_NAME=vmalert $(MAKE) app-local-pure
-
-vmalert-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -2,7 +2,7 @@

 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured `-datasource.url`. For sending alerting notifications
+rules against configured `-datasource.url` compatible with Prometheus HTTP API. For sending alerting notifications
 vmalert relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag.
 Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
 protocol and require `-remoteWrite.url` to be configured.
@@ -20,7 +20,7 @@ implementation and aims to be compatible with its syntax.
 * Keeps the alerts [state on restarts](#alerts-state-on-restarts);
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
-* Lightweight without extra dependencies.
+* Lightweight and without extra dependencies.
 * Supports [reusable templates](#reusable-templates) for annotations.

 ## Limitations
@@ -36,7 +36,7 @@ implementation and aims to be compatible with its syntax.

 To build `vmalert` from sources:

-```bash
+```console
 git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
@@ -47,28 +47,29 @@ The build binary will be placed in `VictoriaMetrics/bin` folder.
 To start using `vmalert` you will need the following things:

 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable MetricsQL endpoint to run queries against;
+* datasource address - reachable endpoint with [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) support for running queries against;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
  aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
  [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-  compatible storage to persist rules and alerts state info;
+  compatible storage to persist rules and alerts state info. To persist results to multiple destinations use vmagent
+  configured with multiple remote writes as a proxy;
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.

 Then configure `vmalert` accordingly:

-```bash
+```console
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
-    -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
+    -datasource.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource
    -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
    -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
-    -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
+    -remoteRead.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource to restore alerts state from
    -external.label=cluster=east-1 \         # External label to be applied for each rule
    -external.label=replica=a                # Multiple external labels may be set
 ```

-Note there's a separate `remoteRead.url` to allow writing results of
+Note there's a separate `remoteWrite.url` to allow writing results of
 alerting/recording rules into a different storage than the initial data that's
 queried. This allows using `vmalert` to aggregate data from a short-term,
 high-frequency, high-cardinality storage into a long-term storage with
@@ -101,6 +102,10 @@ name: <string>
 # How often rules in the group are evaluated.
 [ interval: <duration> | default = -evaluationInterval flag ]

+# Limit the number of alerts an alerting rule and series a recording
+# rule can produce. 0 is no limit.
+[ limit: <int> | default = 0 ]
+
 # How many rules execute at once within a group. Increasing concurrency may speed
 # up round execution speed.
 [ concurrency: <integer> | default = 1 ]
@@ -109,24 +114,27 @@ name: <string>
 # By default "prometheus" type is used.
 [ type: <string> ]

-# Warning: DEPRECATED
-# Please use `params` instead:
-#   params:
-#     extra_label: ["job=nodeexporter", "env=prod"]
-extra_filter_labels:
-  [ <labelname>: <labelvalue> ... ]
-
 # Optional list of HTTP URL parameters
 # applied for all rules requests within a group
 # For example:
 #  params:
 #    nocache: ["1"]                # disable caching for vmselect
 #    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
-#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests
 # see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
 params:
  [ <string>: [<string>, ...]]

+# Optional list of HTTP headers in form `header-name: value`
+# applied for all rules requests within a group
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+# Headers set via this param have priority over headers set via `-datasource.headers` flag.
+headers:
+  [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
@@ -141,7 +149,7 @@ rules:
 ### Rules

 Every rule contains `expr` field for [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
-or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. Vmalert will execute the configured
+or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. `vmalert` will execute the configured
 expression and then act according to the Rule type.

 There are two types of Rules:
@@ -166,7 +174,7 @@ The syntax for alerting rule is the following:
 alert: <string>

 # The expression to evaluate. The expression language depends on the type value.
-# By default PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
+# By default, PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>

@@ -176,6 +184,13 @@ expr: <string>
 # as firing once they return.
 [ for: <duration> | default = 0s ]

+# Whether to print debug information into logs.
+# Information includes alerts state changes and requests sent to the datasource.
+# Please note, that if rule's query params contain sensitive
+# information - it will be printed to logs.
+# Is applicable to alerting rules only.
+[ debug: <bool> | default = false ]
+
 # Labels to add or overwrite for each alert.
 labels:
  [ <labelname>: <tmpl_string> ]
@@ -185,14 +200,31 @@ annotations:
  [ <labelname>: <tmpl_string> ]
 ```

-It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations to format data, iterate over it or execute expressions.
+#### Templating
+
+It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations to format data, iterate over
+or execute expressions.
+The following variables are available in templating:
+
+| Variable                           | Description                                                                                               | Example                                                                                                                                                                |
+|------------------------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| $value or .Value                   | The current alert's value. Avoid using value in labels, it may cause unexpected issues.                   | {% raw %}Number of connections is {{ $value }}{% endraw %}                                                                                                             |
+| $activeAt or .ActiveAt             | The moment of [time](https://pkg.go.dev/time) when alert became active (`pending` or `firing`).           | {% raw %}http://vm-grafana.com/panelId=xx?from={{($activeAt.Add (parseDurationTime \"1h\")).Unix}}&to={{($activeAt.Add (parseDurationTime \"-1h\")).Unix}}{% endraw %} |
+| $labels or .Labels                 | The list of labels of the current alert. Use as ".Labels.<label_name>".                                   | {% raw %}Too high number of connections for {{ .Labels.instance }}{% endraw %}                                                                                         |
+| $alertID or .AlertID               | The current alert's ID generated by vmalert.                                                              | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %}                                                                                   |
+| $groupID or .GroupID               | The current alert's group ID generated by vmalert.                                                        | {% raw %}Link: vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}{% endraw %}                                                                                   |
+| $expr or .Expr                     | Alert's expression. Can be used for generating links to Grafana or other systems.                         | {% raw %}/api/v1/query?query={{ $expr&#124;quotesEscape&#124;queryEscape }}{% endraw %}                                                                                |
+| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag.                                        | {% raw %}Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }}){% endraw %}                                                                          |
+| $externalURL or .ExternalURL       | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | {% raw %}Visit {{ $externalURL }} for more details{% endraw %}                                                                                                         |
+
 Additionally, `vmalert` provides some extra templating functions
-listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/template_func.go) and [reusable templates](#reusable-templates).
+listed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/templates/template.go)
+and [reusable templates](#reusable-templates).

 #### Reusable templates

 Like in Alertmanager you can define [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates)
-to share same templates across annotations. Just define the templates in a file and 
+to share same templates across annotations. Just define the templates in a file and
 set the path via `-rule.templates` flag.

 For example, template `grafana.filter` can be defined as following:
@@ -241,7 +273,7 @@ The syntax for recording rules is following:
 record: <string>

 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If group.type="graphite", then the expression
+# By default, MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>

@@ -299,6 +331,10 @@ groups:
    # Rules for accountID=456, projectID=789
 ```

+The results of alerting and recording rules contain `vm_account_id` and `vm_project_id` labels
+if `-clusterMode` is enabled. These labels can be used during [templating](https://docs.victoriametrics.com/vmalert.html#templating),
+and help to identify to which account or project the triggered alert or produced recording belongs.
+
 If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
 contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
 `vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
@@ -385,6 +421,10 @@ Alertmanagers.
 To avoid recording rules results and alerts state duplication in VictoriaMetrics server
 don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
 The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert's `evaluation_interval`.
+If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
+command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
+But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
+which would affect deduplication logic and result into "jumping" datapoints.

 Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
 all `vmalert`s are having the same config.
@@ -397,6 +437,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics)

 #### Downsampling and aggregation via vmalert

+`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries
+via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`.
+This ability allows to aggregate data. For example, the following rule will calculate the average value for
+metric `http_requests` on the `5m` interval:
+
+```yaml
+  - record: http_requests:avg5m
+    expr: avg_over_time(http_requests[5m])
+```
+
+Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m`
+to the configured `-remoteWrite.url`.
+
+`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval`
+or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series.
+This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`:
+
+```yaml
+groups:
+  - name: my_group
+    interval: 5m
+    rules:
+    - record: http_requests:avg5m
+      expr: avg_over_time(http_requests[5m])
+```
+
+Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows
+reading data from one data source and backfilling results to another. This helps to build a system
+for aggregating and downsampling the data.
+
 The following example shows how to build a topology where `vmalert` will process data from one cluster
 and write results into another. Such clusters may be called as "hot" (low retention,
 high-speed disks, used for operative monitoring) and "cold" (long term retention,
@@ -408,7 +478,7 @@ or reducing resolution) and push results to "cold" cluster.

 ```
 ./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
-    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recording rules expressions
    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus    # vminsert addr to persist recording rules results
 ```

@@ -420,6 +490,21 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec

 See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).

+#### Multiple remote writes
+
+For persisting recording or alerting rule results `vmalert` requires `-remoteWrite.url` to be set.
+But this flag supports only one destination. To persist rule results to multiple destinations
+we recommend using [vmagent](https://docs.victoriametrics.com/vmagent.html) as fan-out proxy:
+
+<img alt="vmalert multiple remote write destinations" src="vmalert_multiple_rw.png">
+
+In this topology, `vmalert` is configured to persist rule results to `vmagent`. And `vmagent`
+is configured to fan-out received data to two or more destinations.
+Using `vmagent` as a proxy provides additional benefits such as
+[data persisting when storage is unreachable](https://docs.victoriametrics.com/vmagent.html#replication-and-high-availability),
+or time series modification via [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling).
+
+
 ### Web

 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -427,11 +512,21 @@ See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
 * `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
-* `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status"` - get alert status by ID.
+* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in JSON format.
  Used as alert source in AlertManager.
+* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in web UI.
+* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in web UI.
 * `http://<vmalert-addr>/metrics` - application metrics.
 * `http://<vmalert-addr>/-/reload` - hot configuration reload.

+`vmalert` web UI can be accessed from [single-node version of VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html)
+and from [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+This may be used for better integraion with Grafana unified alerting system. See the following docs for details:
+
+* [How to query vmalert from single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmalert)
+* [How to query vmalert from VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmalert)
+
+
 ## Graphite

 vmalert sends requests to `<-datasource.url>/render?format=json` during evaluation of alerting and recording rules
@@ -454,7 +549,7 @@ To run vmalert in `replay` mode:

 ```
 ./bin/vmalert -rule=path/to/your.rules \        # path to files with rules you usually use with vmalert
-    -datasource.url=http://localhost:8428 \     # PromQL/MetricsQL compatible datasource
+    -datasource.url=http://localhost:8428 \     # Prometheus HTTP API compatible datasource
    -remoteWrite.url=http://localhost:8428 \    # remote write compatible storage to persist results
    -replay.timeFrom=2021-05-11T07:21:43Z \     # time from begin replay
    -replay.timeTo=2021-05-29T18:40:43Z         # time to finish replay
@@ -490,7 +585,7 @@ max range per request:  8h20m0s

 In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
 executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
-to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
+to [/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) endpoint
 of the configured `-datasource.url`. Returned data is then processed according to the rule type and
 backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
@@ -525,16 +620,17 @@ There are following non-required `replay` flags:
  (rules which depend on each other) rules. It is expected, that remote storage will be able to persist
  previously accepted data during the delay, so data will be available for the subsequent queries.
  Keep it equal or bigger than `-remoteWrite.flushInterval`.
-* `replay.disableProgressBar` - whether to disable progress bar which shows progress work.
+* `-replay.disableProgressBar` - whether to disable progress bar which shows progress work.
  Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
  It could break logs parsing by external system and generate additional load on it.

-See full description for these flags in `./vmalert --help`.
+See full description for these flags in `./vmalert -help`.

 ### Limitations

 * Graphite engine isn't supported yet;
 * `query` template function is disabled for performance reasons (might be changed in future);
+* `limit` group's param has no effect during replay (might be changed in future);

 ## Monitoring

@@ -546,6 +642,95 @@ Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/1495
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.

+## Troubleshooting
+
+vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed,
+the data is already present in configured `-datasource.url`:
+
+<img alt="vmalert expected evaluation" src="vmalert_ts_normal.gif">
+
+Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations
+may get empty response from datasource and produce empty recording rules or reset alerts state:
+
+<img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
+
+_By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
+This behavior is controlled by `-search.latencyOffset` command-line flag on vmselect. Usually, this results into
+a 30s shift for recording rules results.
+Note that too small value passed to `-search.latencyOffset` may lead to incomplete query results._
+
+Try the following recommendations in such cases:
+
+* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics
+are delivered to the datasource;
+* If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback`
+command-line flag to add a time shift for evaluations;
+* If time intervals between datapoints in datasource are irregular - try changing vmalert's `-datasource.queryStep`
+command-line flag to specify how far search query can lookback for the recent datapoint. By default, this value
+is equal to group's evaluation interval.
+
+Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that
+alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only
+after multiple consecutive evaluations, and at each evaluation their expression must be true. If at least one evaluation
+becomes false, then alert's state resets to the initial state.
+
+If `-remoteWrite.url` command-line flag is configured, vmalert will persist alert's state in form of time series
+`ALERTS` and `ALERTS_FOR_STATE` to the specified destination. Such time series can be then queried via
+[vmui](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) or Grafana to track how alerts state
+changed in time.
+
+vmalert also stores last N state updates for each rule. To check updates, click on `Details` link next to rule's name
+on `/vmalert/groups` page and check the `Last updates` section:
+
+<img alt="vmalert state" src="vmalert_state.png">
+
+Rows in the section represent ordered rule evaluations and their results. The column `curl` contains an example of
+HTTP request sent by vmalert to the `-datasource.url` during evaluation. If specific state shows that there were
+no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
+moment when rule was evaluated.
+
+vmalert also alows configuring more detailed logging for specific rule. Just set `debug: true` in rule's configuration
+and vmalert will start printing additional log messages:
+```terminal
+2022-09-15T13:35:41.155Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 samples (elapsed: 5.896041ms)
+2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
+2022-09-15T13:35:56.178Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: query returned 1 samples (elapsed: 28.368208ms)
+2022-09-15T13:35:56.178Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29&step=15s&time=1663248945"
+2022-09-15T13:35:56.179Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} created in state PENDING
+...
+2022-09-15T13:36:56.153Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:36:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} PENDING => FIRING: 1m0s since becoming active at 2022-09-15 15:35:56.126006 +0200 CEST m=+39.384575417
+```
+
+
+## Profiling
+
+`vmalert` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
+
+```console
+curl http://0.0.0.0:8880/debug/pprof/heap > mem.pprof
+```
+
+</div>
+
+* CPU profile. It can be collected with the following command (replace `0.0.0.0` with hostname if needed):
+
+<div class="with-copy" markdown="1">
+
+```console
+curl http://0.0.0.0:8880/debug/pprof/profile > cpu.pprof
+```
+
+</div>
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+It is safe sharing the collected profiles from security point of view, since they do not contain sensitive information.
+
 ## Configuration

 ### Flags
@@ -554,10 +739,10 @@ Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.

 The shortlist of configuration flags is the following:
-
+{% raw  %}
 ```
  -clusterMode
-     If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy . This flag is available only in enterprise version of VictoriaMetrics
  -configCheckInterval duration
     Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.
  -datasource.appendTypePrefix
@@ -574,26 +759,30 @@ The shortlist of configuration flags is the following:
     Optional path to bearer token file to use for -datasource.url.
  -datasource.disableKeepAlive
     Whether to disable long-lived connections to the datasource. If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.
+  -datasource.headers string
+     Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'
  -datasource.lookback duration
     Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
  -datasource.maxIdleConnections int
     Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
  -datasource.oauth2.clientID string
-     Optional OAuth2 clientID to use for -datasource.url. 
+     Optional OAuth2 clientID to use for -datasource.url.
  -datasource.oauth2.clientSecret string
     Optional OAuth2 clientSecret to use for -datasource.url.
  -datasource.oauth2.clientSecretFile string
-     Optional OAuth2 clientSecretFile to use for -datasource.url. 
+     Optional OAuth2 clientSecretFile to use for -datasource.url.
  -datasource.oauth2.scopes string
     Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
  -datasource.oauth2.tokenUrl string
     Optional OAuth2 tokenURL to use for -datasource.url.
  -datasource.queryStep duration
-     queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+     How far a value can fallback to when evaluating queries. For example, if -datasource.queryStep=15s then param "step" with value "15s" will be added to every query. If set to 0, rule's evaluation interval will be used instead. (default 5m0s)
  -datasource.queryTimeAlignment
     Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
  -datasource.roundDigits int
     Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+  -datasource.showURL
+     Whether to show -datasource.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
  -datasource.tlsCAFile string
     Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
  -datasource.tlsCertFile string
@@ -605,14 +794,14 @@ The shortlist of configuration flags is the following:
  -datasource.tlsServerName string
     Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
  -datasource.url string
-     VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend
+     Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.
  -defaultTenant.graphite string
-     Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy .This flag is available only in enterprise version of VictoriaMetrics
  -defaultTenant.prometheus string
-     Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
+     Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy . This flag is available only in enterprise version of VictoriaMetrics
  -disableAlertgroupLabel
     Whether to disable adding group's Name as label to generated alerts and time series.
-  -dryRun -rule
+  -dryRun
     Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
  -enableTCP6
     Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
@@ -621,17 +810,19 @@ The shortlist of configuration flags is the following:
  -envflag.prefix string
     Prefix for environment variables if -envflag.enable is set
  -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in enterprise version of VictoriaMetrics
  -evaluationInterval duration
     How often to evaluate the rules (default 1m0s)
  -external.alert.source string
-     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-     eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
+     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used
+     If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
  -external.label array
     Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
     Supports an array of values separated by comma or specified via multiple flags.
  -external.url string
     External URL is used as alert's source for sent alerts to the notifier
+  -flagsAuthKey string
+     Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -fs.disableMmap
     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
  -http.connTimeout duration
@@ -672,7 +863,7 @@ The shortlist of configuration flags is the following:
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
  -metricsAuthKey string
-     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -notifier.basicAuth.password array
     Optional basic auth password for -notifier.url
     Supports an array of values separated by comma or specified via multiple flags.
@@ -726,17 +917,25 @@ The shortlist of configuration flags is the following:
     Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
     Supports an array of values separated by comma or specified via multiple flags.
  -pprofAuthKey string
-     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -promscrape.consul.waitTime duration
     Wait time used by Consul service discovery. Default value is used if not set
  -promscrape.consulSDCheckInterval duration
-     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
+     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs for details (default 30s)
  -promscrape.discovery.concurrency int
     The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
  -promscrape.discovery.concurrentWaitTime duration
     The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
  -promscrape.dnsSDCheckInterval duration
-     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
+     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#dns_sd_configs for details (default 30s)
+  -pushmetrics.extraLabel array
+     Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -pushmetrics.interval duration
+     Interval for pushing metrics to -pushmetrics.url (default 10s)
+  -pushmetrics.url array
+     Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default metrics exposed at /metrics page aren't pushed to any remote storage
+     Supports an array of values separated by comma or specified via multiple flags.
  -remoteRead.basicAuth.password string
     Optional basic auth password for -remoteRead.url
  -remoteRead.basicAuth.passwordFile string
@@ -749,6 +948,8 @@ The shortlist of configuration flags is the following:
     Optional path to bearer token file to use for -remoteRead.url.
  -remoteRead.disablePathAppend
     Whether to disable automatic appending of '/api/v1/query' path to the configured -datasource.url and -remoteRead.url
+  -remoteRead.headers string
+     Optional HTTP headers to send with each request to the corresponding -remoteRead.url. For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'
  -remoteRead.ignoreRestoreErrors
     Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
  -remoteRead.lookback duration
@@ -762,7 +963,9 @@ The shortlist of configuration flags is the following:
  -remoteRead.oauth2.scopes string
     Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
  -remoteRead.oauth2.tokenUrl string
-     Optional OAuth2 tokenURL to use for -remoteRead.url. 
+     Optional OAuth2 tokenURL to use for -remoteRead.url.
+  -remoteRead.showURL
+     Whether to show -remoteRead.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
  -remoteRead.tlsCAFile string
     Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
  -remoteRead.tlsCertFile string
@@ -774,7 +977,7 @@ The shortlist of configuration flags is the following:
  -remoteRead.tlsServerName string
     Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
  -remoteRead.url vmalert
-     Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
+     Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
  -remoteWrite.basicAuth.password string
     Optional basic auth password for -remoteWrite.url
  -remoteWrite.basicAuth.passwordFile string
@@ -791,6 +994,8 @@ The shortlist of configuration flags is the following:
     Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
  -remoteWrite.flushInterval duration
     Defines interval of flushes to remote write endpoint (default 5s)
+  -remoteWrite.headers string
+     Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'
  -remoteWrite.maxBatchSize int
     Defines defines max number of timeseries to be flushed at once (default 1000)
  -remoteWrite.maxQueueSize int
@@ -805,6 +1010,8 @@ The shortlist of configuration flags is the following:
     Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
  -remoteWrite.oauth2.tokenUrl string
     Optional OAuth2 tokenURL to use for -notifier.url.
+  -remoteWrite.showURL
+     Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
  -remoteWrite.tlsCAFile string
     Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
  -remoteWrite.tlsCertFile string
@@ -816,7 +1023,7 @@ The shortlist of configuration flags is the following:
  -remoteWrite.tlsServerName string
     Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
  -remoteWrite.url string
-     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
  -replay.disableProgressBar
     Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
  -replay.maxDatapointsPerQuery int
@@ -865,9 +1072,12 @@ The shortlist of configuration flags is the following:
     Supports an array of values separated by comma or specified via multiple flags.
  -tlsKeyFile string
     Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+  -tlsMinVersion string
+     Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
  -version
     Show VictoriaMetrics version
 ```
+{% endraw  %}

 ### Hot config reload

@@ -914,7 +1124,7 @@ and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configurati
 For example:

 ```
-static_configs: 
+static_configs:
  - targets:
      - localhost:9093
      - localhost:9095
@@ -923,7 +1133,7 @@ consul_sd_configs:
  - server: localhost:8500
    services:
      - alertmanager
-      
+
 dns_sd_configs:
  - names:
      - my.domain.com
@@ -951,7 +1161,7 @@ is the following:
 # password and password_file are mutually exclusive.
 basic_auth:
  [ username: <string> ]
-  [ password: <secret> ]
+  [ password: <string> ]
  [ password_file: <string> ]

 # Optional `Authorization` header configuration.
@@ -970,10 +1180,41 @@ authorization:
 tls_config:
  [ <tls_config> ]

+# Configures Bearer authentication token via string
+bearer_token: <string>
+# or by passing path to the file with token.
+bearer_token_file: <string>
+
+# Configures OAuth 2.0 authentication
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2
+oauth2:
+  [ <oauth2_config> ]
+  
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all requests to notifiers
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+headers:
+  [ <string>, ...]
+
 # List of labeled statically configured Notifiers.
+#
+# Each list of targets may be additionally instructed with
+# authorization params. Target's authorization params will
+# inherit params from global authorization params if there
+# are no conflicts.
 static_configs:
-  targets:
-    [ - '<host>' ]
+  [ - targets: ]
+      [ - '<host>' ]
+      [ oauth2 ]
+      [ basic_auth ]
+      [ authorization ]
+      [ tls_config ]
+      [ bearer_token ]
+      [ bearer_token_file ]
+      [ headers ]

 # List of Consul service discovery configurations.
 # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
@@ -1013,9 +1254,28 @@ It is recommended using

 * `vmalert` is located in `vmutils-*` archives there.

+### Docker image
+
+You can build `vmalert` docker image from source and push it to your own docker repository.
+Run the following commands from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics):
+```console
+make package-vmalert
+docker tag victoria-metrics/vmalert:version my-repo:my-version-name
+docker push my-repo:my-version-name
+```
+
+To run the built image in `victoria-metrics-k8s-stack` or `VMAlert` CR object apply the following config change:
+```yaml
+kind: VMAlert
+spec:
+  image:
+    repository: my-repo
+    tag: my-version-name
+```
+
 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.1.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmalert` binary and puts it into the `bin` folder.

@@ -1031,12 +1291,12 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b

 ### Development ARM build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
-2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.1.
+2. Run `make vmalert-linux-arm` or `make vmalert-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmalert-linux-arm` or `vmalert-linux-arm64` binary respectively and puts it into the `bin` folder.

 ### Production ARM build

 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmalert-arm-prod` or `make vmalert-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmalert-arm-prod` or `vmalert-arm64-prod` binary respectively and puts it into the `bin` folder.
+2. Run `make vmalert-linux-arm-prod` or `make vmalert-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmalert-linux-arm-prod` or `vmalert-linux-arm64-prod` binary respectively and puts it into the `bin` folder.
--- a/app/vmalert/alerting.go
+++ b/app/vmalert/alerting.go
@@ -6,6 +6,7 @@ import (
 	"hash/fnv"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"

@@ -20,7 +21,7 @@ import (

 // AlertingRule is basic alert entity
 type AlertingRule struct {
-	Type         datasource.Type
+	Type         config.Type
 	RuleID       uint64
 	Name         string
 	Expr         string
@@ -30,24 +31,17 @@ type AlertingRule struct {
 	GroupID      uint64
 	GroupName    string
 	EvalInterval time.Duration
+	Debug        bool

 	q datasource.Querier

-	// guard status fields
-	mu sync.RWMutex
+	alertsMu sync.RWMutex
 	// stores list of active alerts
 	alerts map[uint64]*notifier.Alert
-	// stores last moment of time Exec was called
-	lastExecTime time.Time
-	// stores the duration of the last Exec call
-	lastExecDuration time.Duration
-	// stores last error that happened in Exec func
-	// resets on every successful Exec
-	// may be used as Health state
-	lastExecError error
-	// stores the number of samples returned during
-	// the last evaluation
-	lastExecSamples int
+
+	// state stores recent state changes
+	// during evaluations
+	state *ruleState

 	metrics *alertingRuleMetrics
 }
@@ -71,20 +65,24 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupID:      group.ID(),
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
+		Debug:        cfg.Debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
+			Debug:              cfg.Debug,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
+		state:   newRuleState(),
 		metrics: &alertingRuleMetrics{},
 	}

 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StatePending {
@@ -95,8 +93,8 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		})
 	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StateFiring {
@@ -107,18 +105,16 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		})
 	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
-			if ar.lastExecError == nil {
+			e := ar.state.getLast()
+			if e.err == nil {
 				return 0
 			}
 			return 1
 		})
 	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
-			ar.mu.RLock()
-			defer ar.mu.RUnlock()
-			return float64(ar.lastExecSamples)
+			e := ar.state.getLast()
+			return float64(e.samples)
 		})
 	return ar
 }
@@ -142,12 +138,42 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }

+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
+	if !ar.Debug {
+		return
+	}
+	prefix := fmt.Sprintf("DEBUG rule %q:%q (%d) at %v: ",
+		ar.GroupName, ar.Name, ar.RuleID, at.Format(time.RFC3339))
+
+	if a != nil {
+		labelKeys := make([]string, len(a.Labels))
+		var i int
+		for k := range a.Labels {
+			labelKeys[i] = k
+			i++
+		}
+		sort.Strings(labelKeys)
+		labels := make([]string, len(labelKeys))
+		for i, l := range labelKeys {
+			labels[i] = fmt.Sprintf("%s=%q", l, a.Labels[l])
+		}
+		labelsStr := strings.Join(labels, ",")
+		prefix += fmt.Sprintf("alert %d {%s} ", a.ID, labelsStr)
+	}
+	msg := fmt.Sprintf(format, args...)
+	logger.Infof("%s", prefix+msg)
+}
+
 type labelSet struct {
-	// origin labels from series
-	// used for templating
+	// origin labels extracted from received time series
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, origin labels from time series preferred.
+	// used for templating annotations
 	origin map[string]string
-	// processed labels with additional data
-	// used as Alert labels
+	// processed labels includes origin labels
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, extra labels are preferred.
+	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
 	processed map[string]string
 }

@@ -155,15 +181,15 @@ type labelSet struct {
 // to labelSet which contains original and processed labels.
 func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
 	ls := &labelSet{
-		origin:    make(map[string]string, len(m.Labels)),
+		origin:    make(map[string]string),
 		processed: make(map[string]string),
 	}
 	for _, l := range m.Labels {
+		ls.origin[l.Name] = l.Value
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
 			continue
 		}
-		ls.origin[l.Name] = l.Value
 		ls.processed[l.Name] = l.Value
 	}

@@ -177,14 +203,23 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 	}
 	for k, v := range extraLabels {
 		ls.processed[k] = v
+		if _, ok := ls.origin[k]; !ok {
+			ls.origin[k] = v
+		}
 	}

 	// set additional labels to identify group and rule name
 	if ar.Name != "" {
 		ls.processed[alertNameLabel] = ar.Name
+		if _, ok := ls.origin[alertNameLabel]; !ok {
+			ls.origin[alertNameLabel] = ar.Name
+		}
 	}
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
 		ls.processed[alertGroupNameLabel] = ar.GroupName
+		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
+			ls.origin[alertGroupNameLabel] = ar.GroupName
+		}
 	}
 	return ls, nil
 }
@@ -240,41 +275,57 @@ const resolvedRetention = 15 * time.Minute

 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	start := time.Now()
-	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
-	ar.mu.Lock()
-	defer ar.mu.Unlock()
+	qMetrics, req, err := ar.q.Query(ctx, ar.Expr, ts)
+	curState := ruleStateEntry{
+		time:     start,
+		at:       ts,
+		duration: time.Since(start),
+		samples:  len(qMetrics),
+		err:      err,
+		req:      req,
+	}
+
+	defer func() {
+		ar.state.add(curState)
+	}()
+
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()

-	ar.lastExecTime = start
-	ar.lastExecDuration = time.Since(start)
-	ar.lastExecError = err
-	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}

+	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.samples, curState.duration)
+
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
 		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
+			ar.logDebugf(ts, a, "deleted as inactive")
 			delete(ar.alerts, h)
 		}
 	}

-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res, err
+	}
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
 		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
+			curState.err = fmt.Errorf("failed to expand labels: %s", err)
+			return nil, curState.err
 		}
 		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
 			// conflicting with the metric labels
-			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
-			return nil, ar.lastExecError
+			curState.err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, curState.err
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
@@ -284,30 +335,28 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 				// back to notifier.StatePending
 				a.State = notifier.StatePending
 				a.ActiveAt = ts
+				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
-			if a.Value != m.Values[0] {
-				// update Value field with latest value
-				a.Value = m.Values[0]
-				// and re-exec template since Value can be used
-				// in annotations
-				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
-				if err != nil {
-					return nil, err
-				}
+			a.Value = m.Values[0]
+			// re-exec template since Value or query can be used in annotations
+			a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+			if err != nil {
+				return nil, err
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
+		a, err := ar.newAlert(m, ls, start, qFn)
 		if err != nil {
-			ar.lastExecError = err
-			return nil, fmt.Errorf("failed to create alert: %w", err)
+			curState.err = fmt.Errorf("failed to create alert: %w", err)
+			return nil, curState.err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
 		a.ActiveAt = ts
 		ar.alerts[h] = a
+		ar.logDebugf(ts, a, "created in state PENDING")
 	}
-
+	var numActivePending int
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
@@ -316,20 +365,29 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 				// alert was in Pending state - it is not
 				// active anymore
 				delete(ar.alerts, h)
+				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
 			}
 			if a.State == notifier.StateFiring {
 				a.State = notifier.StateInactive
 				a.ResolvedAt = ts
+				ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
 			}
 			continue
 		}
+		numActivePending++
 		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
+			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
 		}
 	}
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+		return nil, curState.err
+	}
 	return ar.toTimeSeries(ts.Unix()), nil
 }

@@ -405,8 +463,8 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T

 // AlertAPI generates APIAlert object from alert by its id(hash)
 func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
-	ar.mu.RLock()
-	defer ar.mu.RUnlock()
+	ar.alertsMu.RLock()
+	defer ar.alertsMu.RUnlock()
 	a, ok := ar.alerts[id]
 	if !ok {
 		return nil
@@ -414,9 +472,10 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }

-// ToAPI returns Rule representation in form
-// of APIRule
+// ToAPI returns Rule representation in form of APIRule
+// Isn't thread-safe. Call must be protected by AlertingRule mutex.
 func (ar *AlertingRule) ToAPI() APIRule {
+	lastState := ar.state.getLast()
 	r := APIRule{
 		Type:           "alerting",
 		DatasourceType: ar.Type.String(),
@@ -425,19 +484,20 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		Duration:       ar.For.Seconds(),
 		Labels:         ar.Labels,
 		Annotations:    ar.Annotations,
-		LastEvaluation: ar.lastExecTime,
-		EvaluationTime: ar.lastExecDuration.Seconds(),
+		LastEvaluation: lastState.time,
+		EvaluationTime: lastState.duration.Seconds(),
 		Health:         "ok",
 		State:          "inactive",
 		Alerts:         ar.AlertsToAPI(),
-		LastSamples:    ar.lastExecSamples,
+		LastSamples:    lastState.samples,
+		Updates:        ar.state.getAll(),

 		// encode as strings to avoid rounding in JSON
 		ID:      fmt.Sprintf("%d", ar.ID()),
 		GroupID: fmt.Sprintf("%d", ar.GroupID),
 	}
-	if ar.lastExecError != nil {
-		r.LastError = ar.lastExecError.Error()
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
 		r.Health = "err"
 	}
 	// satisfy APIRule.State logic
@@ -457,14 +517,14 @@ func (ar *AlertingRule) ToAPI() APIRule {
 // AlertsToAPI generates list of APIAlert objects from existing alerts
 func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
-	ar.mu.RLock()
+	ar.alertsMu.RLock()
 	for _, a := range ar.alerts {
 		if a.State == notifier.StateInactive {
 			continue
 		}
 		alerts = append(alerts, ar.newAlertAPI(*a))
 	}
-	ar.mu.RUnlock()
+	ar.alertsMu.RUnlock()
 	return alerts
 }

@@ -547,7 +607,10 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	}

 	ts := time.Now()
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res, err
+	}

 	// account for external labels in filter
 	var labelsFilter string
@@ -555,12 +618,9 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}

-	// Get the last data point in range via MetricsQL `last_over_time`.
-	// We don't use plain PromQL since Prometheus doesn't support
-	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr, ts)
+	qMetrics, _, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return err
 	}
--- a/app/vmalert/alerting_test.go
+++ b/app/vmalert/alerting_test.go
@@ -304,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -624,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {

 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO(), time.Now())
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}

 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -640,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {

 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -649,6 +649,50 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	}
 }

+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
 func TestAlertingRule_Template(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
@@ -656,14 +700,26 @@ func TestAlertingRule_Template(t *testing.T) {
 		expAlerts map[uint64]*notifier.Alert
 	}{
 		{
-			newTestRuleWithLabels("common", "region", "east"),
+			&AlertingRule{
+				Name: "common",
+				Labels: map[string]string{
+					"region": "east",
+				},
+				Annotations: map[string]string{
+					"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
+			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "instance", "foo"),
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "foo"`,
+					},
 					Labels: map[string]string{
 						alertNameLabel: "common",
 						"region":       "east",
@@ -671,7 +727,9 @@ func TestAlertingRule_Template(t *testing.T) {
 					},
 				},
 				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "bar"`,
+					},
 					Labels: map[string]string{
 						alertNameLabel: "common",
 						"region":       "east",
@@ -687,14 +745,15 @@ func TestAlertingRule_Template(t *testing.T) {
 					"instance": "{{ $labels.instance }}",
 				},
 				Annotations: map[string]string{
-					"summary":     `Too high connection number for "{{ $labels.instance }}"`,
+					"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
 					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "instance", "foo", alertNameLabel, "override"),
-				metricWithValueAndLabels(t, 10, "instance", "bar", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
@@ -703,7 +762,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "foo",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "foo"`,
+						"summary":     `first: Too high connection number for "foo"`,
 						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
@@ -713,7 +772,7 @@ func TestAlertingRule_Template(t *testing.T) {
 						"instance":     "bar",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "bar"`,
+						"summary":     `second: Too high connection number for "bar"`,
 						"description": `override: It is 10 connections for "bar"`,
 					},
 				},
@@ -730,6 +789,7 @@ func TestAlertingRule_Template(t *testing.T) {
 					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
+				state:  newRuleState(),
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1,
@@ -761,7 +821,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -871,5 +931,11 @@ func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 }

 func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
-	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor, EvalInterval: waitFor}
+	return &AlertingRule{
+		Name:         name,
+		For:          waitFor,
+		EvalInterval: waitFor,
+		alerts:       make(map[uint64]*notifier.Alert),
+		state:        newRuleState(),
+	}
 }
--- a/app/vmalert/config/config.go
+++ b/app/vmalert/config/config.go
@@ -4,16 +4,14 @@ import (
 	"crypto/md5"
 	"fmt"
 	"hash/fnv"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"

 	"gopkg.in/yaml.v2"

-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -23,17 +21,13 @@ import (
 // Group contains list of Rules grouped into
 // entity with one name and evaluation interval
 type Group struct {
-	Type        datasource.Type `yaml:"type,omitempty"`
+	Type        Type `yaml:"type,omitempty"`
 	File        string
 	Name        string              `yaml:"name"`
 	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
-	// ExtraFilterLabels is a list label filters applied to every rule
-	// request withing a group. Is compatible only with VM datasources.
-	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
-	// DEPRECATED: use Params field instead
-	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
 	Labels map[string]string `yaml:"labels"`
@@ -42,6 +36,8 @@ type Group struct {
 	Checksum string
 	// Optional HTTP URL parameters added to each rule request
 	Params url.Values `yaml:"params"`
+	// Headers contains optional HTTP headers added to each rule request
+	Headers []Header `yaml:"headers,omitempty"`

 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -59,23 +55,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	// change default value to prometheus datasource.
 	if g.Type.Get() == "" {
-		g.Type.Set(datasource.NewPrometheusType())
-	}
-
-	// backward compatibility with deprecated `ExtraFilterLabels` param
-	if len(g.ExtraFilterLabels) > 0 {
-		if g.Params == nil {
-			g.Params = url.Values{}
-		}
-		// Sort extraFilters for consistent order for query args across runs.
-		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
-		for k, v := range g.ExtraFilterLabels {
-			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
-		}
-		sort.Strings(extraFilters)
-		for _, extraFilter := range extraFilters {
-			g.Params.Add("extra_label", extraFilter)
-		}
+		g.Type.Set(NewPrometheusType())
 	}

 	h := md5.New()
@@ -85,7 +65,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }

 // Validate check for internal Group or Rule configuration errors
-func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
+func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
@@ -97,7 +77,7 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			ruleName = r.Alert
 		}
 		if _, ok := uniqueRules[r.ID]; ok {
-			return fmt.Errorf("rule %q duplicate", ruleName)
+			return fmt.Errorf("%q is a duplicate within the group %q", r.String(), g.Name)
 		}
 		uniqueRules[r.ID] = struct{}{}
 		if err := r.Validate(); err != nil {
@@ -111,11 +91,11 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
-		if validateAnnotations {
-			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+		if validateTplFn != nil {
+			if err := validateTplFn(r.Annotations); err != nil {
 				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
 			}
-			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+			if err := validateTplFn(r.Labels); err != nil {
 				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
@@ -133,6 +113,7 @@ type Rule struct {
 	For         *promutils.Duration `yaml:"for,omitempty"`
 	Labels      map[string]string   `yaml:"labels,omitempty"`
 	Annotations map[string]string   `yaml:"annotations,omitempty"`
+	Debug       bool                `yaml:"debug,omitempty"`

 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -156,6 +137,32 @@ func (r *Rule) Name() string {
 	return r.Alert
 }

+// String implements Stringer interface
+func (r *Rule) String() string {
+	ruleType := "recording"
+	if r.Alert != "" {
+		ruleType = "alerting"
+	}
+	b := strings.Builder{}
+	b.WriteString(fmt.Sprintf("%s rule %q", ruleType, r.Name()))
+	b.WriteString(fmt.Sprintf("; expr: %q", r.Expr))
+
+	kv := sortMap(r.Labels)
+	for i := range kv {
+		if i == 0 {
+			b.WriteString("; labels:")
+		}
+		b.WriteString(" ")
+		b.WriteString(kv[i].key)
+		b.WriteString("=")
+		b.WriteString(kv[i].value)
+		if i < len(kv)-1 {
+			b.WriteString(",")
+		}
+	}
+	return b.String()
+}
+
 // HashRule hashes significant Rule fields into
 // unique hash that supposed to define Rule uniqueness
 func HashRule(r Rule) uint64 {
@@ -188,8 +195,11 @@ func (r *Rule) Validate() error {
 	return checkOverflow(r.XXX, "rule")
 }

+// ValidateTplFn must validate the given annotations
+type ValidateTplFn func(annotations map[string]string) error
+
 // Parse parses rule configs from given file patterns
-func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	var fp []string
 	for _, pattern := range pathPatterns {
 		matches, err := filepath.Glob(pattern)
@@ -199,7 +209,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
-	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -209,7 +218,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			continue
 		}
 		for _, g := range gr {
-			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
+			if err := g.Validate(validateTplFn, validateExpressions); err != nil {
 				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
 				continue
 			}
@@ -219,9 +228,6 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
-			if len(g.ExtraFilterLabels) > 0 {
-				isExtraFilterLabelsUsed = true
-			}
 			groups = append(groups, g)
 		}
 	}
@@ -231,14 +237,11 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
-	if isExtraFilterLabelsUsed {
-		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
-	}
 	return groups, nil
 }

 func parseFile(path string) ([]Group, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("error reading alert rule file: %w", err)
 	}
--- a/app/vmalert/config/config_test.go
+++ b/app/vmalert/config/config_test.go
@@ -9,7 +9,7 @@ import (

 	"gopkg.in/yaml.v2"

-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -22,7 +22,7 @@ func TestMain(m *testing.M) {
 }

 func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
 		t.Errorf("error parsing files %s", err)
 	}
 }
@@ -60,9 +60,13 @@ func TestParseBad(t *testing.T) {
 			[]string{"testdata/rules/rules1-bad.rules"},
 			"bad graphite expr",
 		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
 	}
 	for _, tc := range testCases {
-		_, err := Parse(tc.path, true, true)
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
 			t.Errorf("expected to get error")
 			return
@@ -220,7 +224,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test thanos",
-				Type: datasource.NewRawType("thanos"),
+				Type: NewRawType("thanos"),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -232,7 +236,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test graphite",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "some-description",
@@ -244,7 +248,7 @@ func TestGroup_Validate(t *testing.T) {
 		},
 		{
 			group: &Group{Name: "test prometheus",
-				Type: datasource.NewPrometheusType(),
+				Type: NewPrometheusType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -257,7 +261,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite inherit",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
@@ -272,7 +276,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite prometheus bad expr",
-				Type: datasource.NewGraphiteType(),
+				Type: NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
@@ -286,8 +290,13 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "invalid rule",
 		},
 	}
+
 	for _, tc := range testCases {
-		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
 		if err == nil {
 			if tc.expErr != "" {
 				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
@@ -492,6 +501,55 @@ params:
 rules:
  - alert: foo
    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+limit: 5
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+limit: 10
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`debug` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+    debug: true
 `)
 	})
 }
@@ -529,30 +587,4 @@ rules:
    expr: sum by(job) (up == 1)
 `, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
 	})
-
-	t.Run("extra labels", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-  env: prod
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
-	})
-
-	t.Run("extra labels and params", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-extra_filter_labels:
-  job: victoriametrics
-params:
-  nocache: ["1"]
-  extra_label: ["env=prod"]
-rules:
-  - alert: ExampleAlertAlwaysFiring
-    expr: sum by(job) (up == 1)
-`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
-	})
 }
--- a/app/vmalert/config/testdata/dir/rules6-bad.rules
+++ b/app/vmalert/config/testdata/dir/rules6-bad.rules
@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    headers:
+    - 'foobar'
+    rules:
+      - alert: rows
+        expr: vm_rows > 0
--- a/app/vmalert/config/testdata/rules/rules-replay-good.rules
+++ b/app/vmalert/config/testdata/rules/rules-replay-good.rules
@@ -2,6 +2,7 @@ groups:
  - name: ReplayGroup
    interval: 1m
    concurrency: 1
+    limit: 1000
    rules:
      - record: type:vm_cache_entries:rate5m
        expr: sum(rate(vm_cache_entries[5m])) by (type)
--- a/app/vmalert/config/testdata/rules/rules2-good.rules
+++ b/app/vmalert/config/testdata/rules/rules2-good.rules
@@ -1,16 +1,20 @@
 groups:
  - name: TestGroup
-    interval: 2s
+    interval: 5s
    concurrency: 2
+    limit: 1000
+    headers:
+      - "MyHeader: foo"
    params:
      denyPartialResponse: ["true"]
-      extra_label: ["env=dev"]
    rules:
      - alert: Conns
-        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        expr: vm_tcplistener_conns > 0
        for: 3m
+        debug: true
        annotations:
-          summary: Too high connection number for {{$labels.instance}}
+          labels: "Available labels: {{ $labels }}"
+          summary: Too high connection number for {{ $labels.instance }}
            {{ with printf "sum(vm_tcplistener_conns{instance=%q})" .Labels.instance | query }}
              {{ . | first | value }}
            {{ end }}
--- a/app/vmalert/datasource/type.go
+++ b/app/vmalert/datasource/type.go
@@ -1,7 +1,8 @@
-package datasource
+package config

 import (
 	"fmt"
+	"strings"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
 	"github.com/VictoriaMetrics/metricsql"
@@ -9,45 +10,45 @@ import (

 // Type represents data source type
 type Type struct {
-	name string
+	Name string
 }

 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
 	return Type{
-		name: "prometheus",
+		Name: "prometheus",
 	}
 }

 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
 	return Type{
-		name: "graphite",
+		Name: "graphite",
 	}
 }

 // NewRawType returns datasource type from raw string
 // without validation.
 func NewRawType(d string) Type {
-	return Type{name: d}
+	return Type{Name: d}
 }

 // Get returns datasource type
 func (t *Type) Get() string {
-	return t.name
+	return t.Name
 }

 // Set changes datasource type
 func (t *Type) Set(d Type) {
-	t.name = d.name
+	t.Name = d.Name
 }

 // String implements String interface with default value.
 func (t Type) String() string {
-	if t.name == "" {
+	if t.Name == "" {
 		return "prometheus"
 	}
-	return t.name
+	return t.Name
 }

 // ValidateExpr validates query expression with datasource ql.
@@ -62,7 +63,7 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	default:
-		return fmt.Errorf("unknown datasource type=%q", t.name)
+		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}
 	return nil
 }
@@ -81,11 +82,35 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	default:
 		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
-	t.name = s
+	t.Name = s
 	return nil
 }

 // MarshalYAML implements the yaml.Unmarshaler interface.
 func (t Type) MarshalYAML() (interface{}, error) {
-	return t.name, nil
+	return t.Name, nil
+}
+
+// Header is a Key - Value struct for holding an HTTP header.
+type Header struct {
+	Key   string
+	Value string
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	if s == "" {
+		return nil
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, s)
+	}
+	h.Key = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
 }
--- a/app/vmalert/datasource/datasource.go
+++ b/app/vmalert/datasource/datasource.go
@@ -2,26 +2,37 @@ package datasource

 import (
 	"context"
+	"net/http"
 	"net/url"
 	"time"
 )

 // Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string, ts time.Time) ([]Metric, error)
+	// Query executes instant request with the given query at the given ts.
+	// It returns list of Metric in response, the http.Request used for sending query
+	// and error if any. Returned http.Request can't be reused and its body is already read.
+	// Query should stop once ctx is cancelled.
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error)
+	// QueryRange executes range request with the given query on the given time range.
+	// It returns list of Metric in response and error if any.
+	// QueryRange should stop once ctx is cancelled.
 	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
 }

 // QuerierBuilder builds Querier with given params.
 type QuerierBuilder interface {
+	// BuildWithParams creates a new Querier object with the given params
 	BuildWithParams(params QuerierParams) Querier
 }

 // QuerierParams params for Querier.
 type QuerierParams struct {
-	DataSourceType     *Type
+	DataSourceType     string
 	EvaluationInterval time.Duration
 	QueryParams        url.Values
+	Headers            map[string]string
+	Debug              bool
 }

 // Metric is the basic entity which should be return by datasource
--- a/app/vmalert/datasource/init.go
+++ b/app/vmalert/datasource/init.go
@@ -6,14 +6,22 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )

 var (
-	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
-		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend")
-	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
+		"E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.")
+	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to show -datasource.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("datasource.headers", "", "Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. "+
+		"For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. "+
+		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")

 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
@@ -35,9 +43,9 @@ var (
 	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")

 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
-	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
-		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
-		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
+		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
+		"If set to 0, rule's evaluation interval will be used instead.")
 	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
 		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
@@ -47,6 +55,13 @@ var (
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )

+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showDatasourceURL {
+		flagutil.RegisterSecretFlag("datasource.url")
+	}
+}
+
 // Param represents an HTTP GET param
 type Param struct {
 	Key, Value string
@@ -80,7 +95,8 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
@@ -92,7 +108,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,
 		queryStep:        *queryStep,
-		dataSourceType:   NewPrometheusType(),
+		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,
 	}, nil
 }
--- a/app/vmalert/datasource/vm.go
+++ b/app/vmalert/datasource/vm.go
@@ -3,15 +3,30 @@ package datasource
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"

+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )

+type datasourceType string
+
+const (
+	datasourcePrometheus datasourceType = "prometheus"
+	datasourceGraphite   datasourceType = "graphite"
+)
+
+func toDatasourceType(s string) datasourceType {
+	if s == string(datasourceGraphite) {
+		return datasourceGraphite
+	}
+	return datasourcePrometheus
+}
+
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
@@ -21,9 +36,19 @@ type VMStorage struct {
 	lookBack         time.Duration
 	queryStep        time.Duration

-	dataSourceType     Type
+	dataSourceType     datasourceType
 	evaluationInterval time.Duration
 	extraParams        url.Values
+	extraHeaders       []keyValue
+
+	// whether to print additional log messages
+	// for each sent request
+	debug bool
+}
+
+type keyValue struct {
+	key   string
+	value string
 }

 // Clone makes clone of VMStorage, shares http client.
@@ -41,11 +66,16 @@ func (s *VMStorage) Clone() *VMStorage {

 // ApplyParams - changes given querier params.
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
-	if params.DataSourceType != nil {
-		s.dataSourceType = *params.DataSourceType
-	}
+	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
 	s.extraParams = params.QueryParams
+	s.debug = params.Debug
+	if params.Headers != nil {
+		for key, value := range params.Headers {
+			kv := keyValue{key: key, value: value}
+			s.extraHeaders = append(s.extraHeaders, kv)
+		}
+	}
 	return s
 }

@@ -63,47 +93,48 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 		appendTypePrefix: appendTypePrefix,
 		lookBack:         lookBack,
 		queryStep:        queryStep,
-		dataSourceType:   NewPrometheusType(),
+		dataSourceType:   datasourcePrometheus,
 	}
 }

 // Query executes the given query and returns parsed response
-func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, error) {
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error) {
 	req, err := s.newRequestPOST()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}

-	switch s.dataSourceType.String() {
-	case "prometheus":
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case "graphite":
+	case datasourceGraphite:
 		s.setGraphiteReqParams(req, query, ts)
 	default:
-		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
+		return nil, nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
 	}

 	resp, err := s.do(ctx, req)
 	if err != nil {
-		return nil, err
+		return nil, req, err
 	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()

 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != "prometheus" {
+	if s.dataSourceType != datasourcePrometheus {
 		parseFn = parseGraphiteResponse
 	}
-	return parseFn(req, resp)
+	result, err := parseFn(req, resp)
+	return result, req, err
 }

 // QueryRange executes the given query on the given time range.
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != "prometheus" {
-		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
+	if s.dataSourceType != datasourcePrometheus {
+		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
 	}
 	req, err := s.newRequestPOST()
 	if err != nil {
@@ -127,12 +158,15 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 }

 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	if s.debug {
+		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
+	}
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
 	if resp.StatusCode != http.StatusOK {
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
 	}
@@ -146,9 +180,10 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
-		if auth := s.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		s.authCfg.SetHeaders(req, true)
+	}
+	for _, h := range s.extraHeaders {
+		req.Header.Set(h.key, h.value)
 	}
 	return req, nil
 }
--- a/app/vmalert/datasource/vm_test.go
+++ b/app/vmalert/datasource/vm_test.go
@@ -83,18 +83,18 @@ func TestVMInstantQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()

-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
 	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())

-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	p := datasourcePrometheus
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
 	ts := time.Now()

 	expErr := func(err string) {
-		if _, err := pq.Query(ctx, query, ts); err == nil {
+		if _, _, err := pq.Query(ctx, query, ts); err == nil {
 			t.Fatalf("expected %q got nil", err)
 		}
 	}
@@ -106,7 +106,7 @@ func TestVMInstantQuery(t *testing.T) {
 	expErr("unknown status")                // 4
 	expErr("non-vector resultType error")   // 5

-	m, err := pq.Query(ctx, query, ts) // 6 - vector
+	m, _, err := pq.Query(ctx, query, ts) // 6 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -129,10 +129,13 @@ func TestVMInstantQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m, expected)
 	}

-	m, err = pq.Query(ctx, query, ts) // 7 - scalar
+	m, req, err := pq.Query(ctx, query, ts) // 7 - scalar
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
+	if req == nil {
+		t.Fatalf("expected request to be non-nil")
+	}
 	if len(m) != 1 {
 		t.Fatalf("expected 1 metrics got %d in %+v", len(m), m)
 	}
@@ -146,10 +149,9 @@ func TestVMInstantQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m, expected)
 	}

-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})

-	m, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
+	m, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -206,14 +208,13 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()

-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
 	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())

-	p := NewPrometheusType()
-	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})

 	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
@@ -239,15 +240,14 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}

-	g := NewGraphiteType()
-	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})

 	_, err = gq.QueryRange(ctx, queryRender, start, end)
 	expectError(t, err, "is not supported")
 }

 func TestRequestParams(t *testing.T) {
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
@@ -263,7 +263,7 @@ func TestRequestParams(t *testing.T) {
 			"prometheus path",
 			false,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
 				checkEqualString(t, "/api/v1/query", r.URL.Path)
@@ -273,7 +273,7 @@ func TestRequestParams(t *testing.T) {
 			"prometheus prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
@@ -284,7 +284,7 @@ func TestRequestParams(t *testing.T) {
 			"prometheus range path",
 			true,
 			&VMStorage{
-				dataSourceType: NewPrometheusType(),
+				dataSourceType: datasourcePrometheus,
 			},
 			func(t *testing.T, r *http.Request) {
 				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
@@ -294,7 +294,7 @@ func TestRequestParams(t *testing.T) {
 			"prometheus range prefix",
 			true,
 			&VMStorage{
-				dataSourceType:   NewPrometheusType(),
+				dataSourceType:   datasourcePrometheus,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
@@ -305,7 +305,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite path",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 			},
 			func(t *testing.T, r *http.Request) {
 				checkEqualString(t, graphitePath, r.URL.Path)
@@ -315,7 +315,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite prefix",
 			false,
 			&VMStorage{
-				dataSourceType:   NewGraphiteType(),
+				dataSourceType:   datasourceGraphite,
 				appendTypePrefix: true,
 			},
 			func(t *testing.T, r *http.Request) {
@@ -453,7 +453,7 @@ func TestRequestParams(t *testing.T) {
 			"graphite extra params",
 			false,
 			&VMStorage{
-				dataSourceType: NewGraphiteType(),
+				dataSourceType: datasourceGraphite,
 				extraParams: url.Values{
 					"nocache":      {"1"},
 					"max_lookback": {"1h"},
@@ -472,14 +472,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.String() {
-			case "prometheus":
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case "graphite":
+			case datasourceGraphite:
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -487,7 +487,7 @@ func TestRequestParams(t *testing.T) {
 	}
 }

-func TestAuthConfig(t *testing.T) {
+func TestHeaders(t *testing.T) {
 	var testCases = []struct {
 		name    string
 		vmFn    func() *VMStorage
@@ -527,6 +527,40 @@ func TestAuthConfig(t *testing.T) {
 				checkEqualString(t, "foo", token)
 			},
 		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
 	}
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"

+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -18,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )

 // Group is an entity for grouping rules
@@ -27,14 +28,16 @@ type Group struct {
 	Name           string
 	File           string
 	Rules          []Rule
-	Type           datasource.Type
+	Type           config.Type
 	Interval       time.Duration
+	Limit          int
 	Concurrency    int
 	Checksum       string
 	LastEvaluation time.Time

-	Labels map[string]string
-	Params url.Values
+	Labels  map[string]string
+	Params  url.Values
+	Headers map[string]string

 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -90,9 +93,11 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:        cfg.Name,
 		File:        cfg.File,
 		Interval:    cfg.Interval.Duration(),
+		Limit:       cfg.Limit,
 		Concurrency: cfg.Concurrency,
 		Checksum:    cfg.Checksum,
 		Params:      cfg.Params,
+		Headers:     make(map[string]string),
 		Labels:      cfg.Labels,

 		doneCh:     make(chan struct{}),
@@ -105,6 +110,9 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
+	for _, h := range cfg.Headers {
+		g.Headers[h.Key] = h.Value
+	}
 	g.metrics = newGroupMetrics(g)
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
@@ -214,7 +222,9 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
 	g.Params = newGroup.Params
+	g.Headers = newGroup.Headers
 	g.Labels = newGroup.Labels
+	g.Limit = newGroup.Limit
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
@@ -282,7 +292,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}

 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
-		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
 		for err := range errs {
 			if err != nil {
 				logger.Errorf("group %q: %s", g.Name, err)
@@ -360,12 +370,12 @@ type executor struct {
 	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }

-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, ts, resolveDuration)
+			res <- e.exec(ctx, rule, ts, resolveDuration, limit)
 		}
 		close(res)
 		return res
@@ -378,7 +388,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.T
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, ts, resolveDuration)
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -399,10 +409,10 @@ var (
 	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )

-func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration, limit int) error {
 	execTotal.Inc()

-	tss, err := rule.Exec(ctx, ts)
+	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
@@ -434,11 +444,17 @@ func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDur
 		return nil
 	}

+	wg := sync.WaitGroup{}
 	for _, nt := range e.notifiers() {
-		if err := nt.Send(ctx, alerts); err != nil {
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
-		}
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
 	}
+	wg.Wait()
 	return errGr.Err()
 }

--- a/app/vmalert/group_test.go
+++ b/app/vmalert/group_test.go
@@ -148,7 +148,7 @@ func TestUpdateWith(t *testing.T) {
 					t.Fatalf("expected to have rule %q; got %q", want, got)
 				}
 				if err := compareRules(t, got, want); err != nil {
-					t.Fatalf("comparsion error: %s", err)
+					t.Fatalf("comparison error: %s", err)
 				}
 			}
 		})
@@ -157,7 +157,7 @@ func TestUpdateWith(t *testing.T) {

 func TestGroupStart(t *testing.T) {
 	// TODO: make parsing from string instead of file
-	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, true, true)
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
@@ -413,3 +413,42 @@ func TestPurgeStaleSeries(t *testing.T) {
 		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
 	)
 }
+
+func TestFaultyNotifier(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := newTestAlertingRule("instant", 0)
+	r.q = fq
+
+	fn := &fakeNotifier{}
+	e := &executor{
+		notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&faultyNotifier{},
+				fn,
+			}
+		},
+	}
+	delay := 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), delay)
+	defer cancel()
+
+	go func() {
+		_ = e.exec(ctx, r, time.Now(), 0, 10)
+	}()
+
+	tn := time.Now()
+	deadline := tn.Add(delay / 2)
+	for {
+		if fn.getCounter() > 0 {
+			return
+		}
+		if tn.After(deadline) {
+			break
+		}
+		tn = time.Now()
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Fatalf("alive notifier didn't receive notification by %v", deadline)
+}
--- a/app/vmalert/helpers_test.go
+++ b/app/vmalert/helpers_test.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"reflect"
 	"sort"
 	"sync"
@@ -44,18 +45,20 @@ func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Qu
 }

 func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
-	return fq.Query(ctx, q, time.Now())
+	req, _, err := fq.Query(ctx, q, time.Now())
+	return req, err
 }

-func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
-		return nil, fq.err
+		return nil, nil, fq.err
 	}
 	cp := make([]datasource.Metric, len(fq.metrics))
 	copy(cp, fq.metrics)
-	return cp, nil
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	return cp, req, nil
 }

 type fakeNotifier struct {
@@ -87,6 +90,18 @@ func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	return fn.alerts
 }

+type faultyNotifier struct {
+	fakeNotifier
+}
+
+func (fn *faultyNotifier) Send(ctx context.Context, _ []notifier.Alert) error {
+	d, ok := ctx.Deadline()
+	if ok {
+		time.Sleep(time.Until(d))
+	}
+	return fmt.Errorf("send failed")
+}
+
 func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
 	return metricWithValuesAndLabels(t, []float64{value}, labels...)
 }
@@ -152,7 +167,7 @@ func compareGroups(t *testing.T, a, b *Group) {
 			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
 		}
 		if err := compareRules(t, want, got); err != nil {
-			t.Fatalf("comparsion error: %s", err)
+			t.Fatalf("comparison error: %s", err)
 		}
 	}
 }
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -23,11 +23,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )

 var (
-	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules.
+	rulePath = flagutil.NewArrayString("rule", `Path to the file with alert rules.
 Supports patterns. Flag can be specified multiple times.
 Examples:
 -rule="/path/to/file". Path to a single file with alerting rules
@@ -35,7 +36,7 @@ Examples:
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)

-	ruleTemplatesPath = flagutil.NewArray("rule.templates", `Path or glob pattern to location with go template definitions
+	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
 	for rules annotations templating. Flag can be specified multiple times.
 Examples:
 -rule.templates="/path/to/file". Path to a single file with go templates
@@ -58,9 +59,12 @@ absolute path to all .tpl files in root.`)
 	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")

 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
-	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
-	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
+	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
+		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
+		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]' . `+
+		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used`)
+	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")

 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
@@ -69,7 +73,7 @@ eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{

 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")

-	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )

 var alertURLGeneratorFn notifier.AlertURLGenerator
@@ -79,15 +83,20 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
+
 	err := templates.Load(*ruleTemplatesPath, true)
 	if err != nil {
 		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
 	}

 	if *dryRun {
-		groups, err := config.Parse(*rulePath, true, true)
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -107,6 +116,11 @@ func main() {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
 	}

+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
@@ -115,7 +129,7 @@ func main() {
 		if rw == nil {
 			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
-		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
 		}
@@ -137,7 +151,7 @@ func main() {
 		logger.Fatalf("failed to init: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
-	groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 	if err != nil {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
@@ -236,8 +250,9 @@ func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL

 func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
 	if externalAlertSource == "" {
-		return func(alert notifier.Alert) string {
-			return fmt.Sprintf("%s/api/v1/%s/%s/status", externalURL, strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10))
+		return func(a notifier.Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
 		}, nil
 	}
 	if validateTemplate {
@@ -251,7 +266,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, nil, m)
+		templated, err := alert.ExecTemplate(nil, alert.Labels, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
@@ -281,6 +296,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		defer ticker.Stop()
 	}

+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
 	// init reload metrics with positive values to improve alerting conditions
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
@@ -310,7 +330,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
+		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
--- a/app/vmalert/main_test.go
+++ b/app/vmalert/main_test.go
@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"testing"
@@ -35,24 +34,25 @@ func TestGetExternalURL(t *testing.T) {
 }

 func TestGetAlertURLGenerator(t *testing.T) {
-	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4}
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/api/v1/42/2/status"; exp != fn(testAlert) {
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
+	if exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Errorf("expected tempalte validation error got nil")
+		t.Errorf("expected template validation error got nil")
 	}
-	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true)
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
-	if exp := "https://victoriametrics.com/path/foo?query=4"; exp != fn(testAlert) {
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
@@ -86,7 +86,7 @@ groups:
 `
 	)

-	f, err := ioutil.TempFile("", "")
+	f, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +153,7 @@ groups:

 func writeToFile(t *testing.T, file, b string) {
 	t.Helper()
-	err := ioutil.WriteFile(file, []byte(b), 0644)
+	err := os.WriteFile(file, []byte(b), 0644)
 	if err != nil {
 		t.Fatal(err)
 	}
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -30,6 +30,23 @@ type manager struct {
 	groups   map[uint64]*Group
 }

+// RuleAPI generates APIRule object from alert by its ID(hash)
+func (m *manager) RuleAPI(gID, rID uint64) (APIRule, error) {
+	m.groupsMu.RLock()
+	defer m.groupsMu.RUnlock()
+
+	g, ok := m.groups[gID]
+	if !ok {
+		return APIRule{}, fmt.Errorf("can't find group with id %d", gID)
+	}
+	for _, rule := range g.Rules {
+		if rule.ID() == rID {
+			return rule.ToAPI(), nil
+		}
+	}
+	return APIRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
+}
+
 // AlertAPI generates APIAlert object from alert by its ID(hash)
 func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 	m.groupsMu.RLock()
@@ -70,9 +87,9 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
 		if err != nil {
 			if !*remoteReadIgnoreRestoreErrors {
-				return fmt.Errorf("failed to restore state for group %q: %w", group.Name, err)
+				return fmt.Errorf("failed to restore ruleState for group %q: %w", group.Name, err)
 			}
-			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
+			logger.Errorf("error while restoring ruleState for group %q: %s", group.Name, err)
 		}
 	}

@@ -170,6 +187,7 @@ func (g *Group) toAPI() APIGroup {
 		LastEvaluation: g.LastEvaluation,
 		Concurrency:    g.Concurrency,
 		Params:         urlValuesToStrings(g.Params),
+		Headers:        headersToStrings(g.Headers),
 		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
@@ -198,3 +216,23 @@ func urlValuesToStrings(values url.Values) []string {
 	}
 	return res
 }
+
+func headersToStrings(headers map[string]string) []string {
+	if len(headers) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(headers))
+	for k := range headers {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		v := headers[k]
+		res = append(res, fmt.Sprintf("%s: %s", k, v))
+	}
+
+	return res
+}
--- a/app/vmalert/manager_test.go
+++ b/app/vmalert/manager_test.go
@@ -10,7 +10,6 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
@@ -30,7 +29,7 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
 	cfg := loadCfg(t, []string{"foo/bar"}, true, true)
 	if err := m.update(context.Background(), cfg, false); err != nil {
-		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
+		t.Fatalf("expected to load successfully with empty rules dir; got err instead: %v", err)
 	}
 }

@@ -69,7 +68,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			defer wg.Done()
 			for i := 0; i < iterations; i++ {
 				rnd := rand.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, true, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -132,7 +131,7 @@ func TestManagerUpdate(t *testing.T) {
 				{
 					File:     "config/testdata/dir/rules1-good.rules",
 					Name:     "duplicatedGroupDiffFiles",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules: []Rule{
 						&AlertingRule{
@@ -157,14 +156,14 @@ func TestManagerUpdate(t *testing.T) {
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules:    []Rule{VMRows},
 					Interval: defaultEvalInterval,
 				},
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -179,7 +178,7 @@ func TestManagerUpdate(t *testing.T) {
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
@@ -187,7 +186,7 @@ func TestManagerUpdate(t *testing.T) {
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Name:     "TestGroup",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -202,14 +201,14 @@ func TestManagerUpdate(t *testing.T) {
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
 					Rules:    []Rule{VMRows},
 				},
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
-					Type:     datasource.NewPrometheusType(),
+					Type:     config.NewPrometheusType(),
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
@@ -232,7 +231,7 @@ func TestManagerUpdate(t *testing.T) {
 				t.Fatalf("failed to complete initial rules update: %s", err)
 			}

-			cfgUpdate, err := config.Parse([]string{tc.updatePath}, true, true)
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
 			if err == nil { // update can fail and that's expected
 				_ = m.update(ctx, cfgUpdate, false)
 			}
@@ -330,7 +329,11 @@ func TestManagerUpdateNegative(t *testing.T) {

 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
-	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)
+	var validateTplFn config.ValidateTplFn
+	if validateAnnotations {
+		validateTplFn = notifier.ValidateTemplates
+	}
+	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {
 		t.Fatal(err)
 	}
--- a/app/vmalert/multiarch/Dockerfile
+++ b/app/vmalert/multiarch/Dockerfile
@@ -9,4 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
 ARG TARGETARCH
-COPY vmalert-${TARGETARCH}-prod ./vmalert-prod
+COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -74,9 +74,12 @@ func (as AlertState) String() string {

 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Labels map[string]string
-	Value  float64
-	Expr   string
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
 }

 var tplHeaders = []string{
@@ -85,6 +88,9 @@ var tplHeaders = []string{
 	"{{ $expr := .Expr }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
+	"{{ $alertID := .AlertID }}",
+	"{{ $groupID := .GroupID }}",
+	"{{ $activeAt := .ActiveAt }}",
 }

 // ExecTemplate executes the Alert template for given
@@ -92,7 +98,7 @@ var tplHeaders = []string{
 // Every alert could have a different datasource, so function
 // requires a queryFunction as an argument.
 func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
+	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr, AlertID: a.ID, GroupID: a.GroupID, ActiveAt: a.ActiveAt}
 	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
 	if err != nil {
 		return nil, fmt.Errorf("error getting a template: %w", err)
@@ -163,9 +169,6 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
 	if !execute {
 		return nil
 	}
-	if !execute {
-		return nil
-	}
 	if err = tpl.Execute(dst, data); err != nil {
 		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}
--- a/app/vmalert/notifier/alert_test.go
+++ b/app/vmalert/notifier/alert_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -109,6 +110,65 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
 			},
 		},
+		{
+			name: "alert and group IDs",
+			alert: &Alert{
+				ID:      42,
+				GroupID: 24,
+			},
+			annotations: map[string]string{
+				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+			},
+			expTpl: map[string]string{
+				"url": "/api/v1/alert?alertID=42&groupID=24",
+			},
+		},
+		{
+			name: "ActiveAt time",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+			},
+			expTpl: map[string]string{
+				"diagram": "![](http://example.com?render=1660941298",
+			},
+		},
+		{
+			name:  "ActiveAt time is nil",
+			alert: &Alert{},
+			annotations: map[string]string{
+				"default_time": "{{$activeAt}}",
+			},
+			expTpl: map[string]string{
+				"default_time": "0001-01-01 00:00:00 +0000 UTC",
+			},
+		},
+		{
+			name: "ActiveAt custome format",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+			},
+			expTpl: map[string]string{
+				"fire_time": "2022/08/19 20:34:58",
+			},
+		},
+		{
+			name: "ActiveAt query range",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+			},
+			expTpl: map[string]string{
+				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+			},
+		},
 	}

 	qFn := func(q string) ([]datasource.Metric, error) {
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -79,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	req = req.WithContext(ctx)

 	if am.authCfg != nil {
-		if auth := am.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		am.authCfg.SetHeaders(req, true)
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -91,7 +89,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	defer func() { _ = resp.Body.Close() }()

 	if resp.StatusCode != http.StatusOK {
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
 		}
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@@ -4,8 +4,8 @@ import (
 	"crypto/md5"
 	"fmt"
 	"gopkg.in/yaml.v2"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -63,10 +63,13 @@ type Config struct {
 }

 // StaticConfig contains list of static targets in the following form:
-// 	targets:
-//  	[ - '<host>' ]
+//
+//	targets:
+//	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
+	// HTTPClientConfig contains HTTP configuration for the Targets
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 }

 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -103,7 +106,7 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }

 func parseConfig(path string) (*Config, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("error reading config file: %w", err)
 	}
--- a/app/vmalert/notifier/config_watcher.go
+++ b/app/vmalert/notifier/config_watcher.go
@@ -6,6 +6,7 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
@@ -161,12 +162,13 @@ func (cw *configWatcher) start() error {
 	if len(cw.cfg.StaticConfigs) > 0 {
 		var targets []Target
 		for _, cfg := range cw.cfg.StaticConfigs {
+			httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig)
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
@@ -252,3 +254,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
 	cw.targets[key] = targets
 	cw.targetsMu.Unlock()
 }
+
+// mergeHTTPClientConfigs merges fields between child and parent params
+// by populating child from parent params if they're missing.
+func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig {
+	if child.Authorization == nil {
+		child.Authorization = parent.Authorization
+	}
+	if child.BasicAuth == nil {
+		child.BasicAuth = parent.BasicAuth
+	}
+	if child.BearerToken == nil {
+		child.BearerToken = parent.BearerToken
+	}
+	if child.BearerTokenFile == "" {
+		child.BearerTokenFile = parent.BearerTokenFile
+	}
+	if child.OAuth2 == nil {
+		child.OAuth2 = parent.OAuth2
+	}
+	if child.TLSConfig == nil {
+		child.TLSConfig = parent.TLSConfig
+	}
+	if child.Headers == nil {
+		child.Headers = parent.Headers
+	}
+	return child
+}
--- a/app/vmalert/notifier/config_watcher_test.go
+++ b/app/vmalert/notifier/config_watcher_test.go
@@ -2,17 +2,18 @@ package notifier

 import (
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"sync"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )

 func TestConfigWatcherReload(t *testing.T) {
-	f, err := ioutil.TempFile("", "")
+	f, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -34,7 +35,7 @@ static_configs:
 		t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns)
 	}

-	f2, err := ioutil.TempFile("", "")
+	f2, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +62,7 @@ func TestConfigWatcherStart(t *testing.T) {
 	consulSDServer := newFakeConsulServer()
 	defer consulSDServer.Close()

-	consulSDFile, err := ioutil.TempFile("", "")
+	consulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -107,7 +108,7 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) {
 	consulSDServer2 := newFakeConsulServer()
 	defer consulSDServer2.Close()

-	consulSDFile, err := ioutil.TempFile("", "")
+	consulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -123,7 +124,7 @@ consul_sd_configs:
      - consul
 `, consulSDServer1.URL, consulSDServer2.URL))

-	staticAndConsulSDFile, err := ioutil.TempFile("", "")
+	staticAndConsulSDFile, err := os.CreateTemp("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -175,7 +176,7 @@ consul_sd_configs:

 func writeToFile(t *testing.T, file, b string) {
 	t.Helper()
-	checkErr(t, ioutil.WriteFile(file, []byte(b), 0644))
+	checkErr(t, os.WriteFile(file, []byte(b), 0644))
 }

 func checkErr(t *testing.T, err error) {
@@ -299,3 +300,20 @@ func newFakeConsulServer() *httptest.Server {

 	return httptest.NewServer(mux)
 }
+
+func TestMergeHTTPClientConfigs(t *testing.T) {
+	cfg1 := promauth.HTTPClientConfig{Headers: []string{"Header:Foo"}}
+	cfg2 := promauth.HTTPClientConfig{BasicAuth: &promauth.BasicAuthConfig{
+		Username: "foo",
+		Password: promauth.NewSecret("bar"),
+	}}
+
+	result := mergeHTTPClientConfigs(cfg1, cfg2)
+
+	if result.Headers == nil {
+		t.Fatalf("expected Headers to be inherited")
+	}
+	if result.BasicAuth == nil {
+		t.Fatalf("expected BasicAuth tp be present")
+	}
+}
--- a/app/vmalert/notifier/init.go
+++ b/app/vmalert/notifier/init.go
@@ -17,32 +17,32 @@ var (
 	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
 	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")

-	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	addrs = flagutil.NewArrayString("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")

-	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
-	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
+	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")

-	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
-	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
+	bearerToken     = flagutil.NewArrayString("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArrayString("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")

 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
-	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
-	tlsKeyFile            = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
-	tlsCAFile             = flagutil.NewArray("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
+	tlsCertFile           = flagutil.NewArrayString("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
+	tlsKeyFile            = flagutil.NewArrayString("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
+	tlsCAFile             = flagutil.NewArrayString("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
 		"By default system CA is used")
-	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
+	tlsServerName = flagutil.NewArrayString("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")

-	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+	oauth2ClientID = flagutil.NewArrayString("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+	oauth2ClientSecret = flagutil.NewArrayString("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+	oauth2ClientSecretFile = flagutil.NewArrayString("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+	oauth2TokenURL = flagutil.NewArrayString("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
-	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+	oauth2Scopes = flagutil.NewArrayString("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )

@@ -74,9 +74,10 @@ var (

 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
-//   * configuration via flags (for backward compatibility). Is always static
+//   - configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
-//   * configuration via file. Supports live reloads and service discovery.
+//   - configuration via file. Supports live reloads and service discovery.
+//
 // Init returns an error if both mods are used.
 func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
 	if externalLabels != nil || externalURL != "" {
@@ -145,7 +146,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}

 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
 		if err != nil {
 			return nil, err
 		}
--- a/app/vmalert/notifier/testdata/static.good.yaml
+++ b/app/vmalert/notifier/testdata/static.good.yaml
@@ -1,7 +1,21 @@
+headers:
+  - 'CustomHeader: foo'
+
 static_configs:
  - targets:
      - localhost:9093
      - localhost:9095
+    basic_auth:
+      username: foo
+      password: bar
+
+  - targets:
+      - localhost:9096
+      - localhost:9097
+    basic_auth:
+      username: foo
+      password: baz
+
 alert_relabel_configs:
  - target_label: "foo"
-    replacement: "aaa"
+    replacement: "aaa"
--- a/app/vmalert/recording.go
+++ b/app/vmalert/recording.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -18,7 +17,7 @@ import (
 // to evaluate configured Expression and
 // return TimeSeries as result.
 type RecordingRule struct {
-	Type    datasource.Type
+	Type    config.Type
 	RuleID  uint64
 	Name    string
 	Expr    string
@@ -27,19 +26,9 @@ type RecordingRule struct {

 	q datasource.Querier

-	// guard status fields
-	mu sync.RWMutex
-	// stores last moment of time Exec was called
-	lastExecTime time.Time
-	// stores the duration of the last Exec call
-	lastExecDuration time.Duration
-	// stores last error that happened in Exec func
-	// resets on every successful Exec
-	// may be used as Health state
-	lastExecError error
-	// stores the number of samples returned during
-	// the last evaluation
-	lastExecSamples int
+	// state stores recent state changes
+	// during evaluations
+	state *ruleState

 	metrics *recordingRuleMetrics
 }
@@ -69,28 +58,28 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		Labels:  cfg.Labels,
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
+		state:   newRuleState(),
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &group.Type,
+			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
 			QueryParams:        group.Params,
+			Headers:            group.Headers,
 		}),
 	}

 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
 	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
-			rr.mu.RLock()
-			defer rr.mu.RUnlock()
-			if rr.lastExecError == nil {
+			e := rr.state.getLast()
+			if e.err == nil {
 				return 0
 			}
 			return 1
 		})
 	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
-			rr.mu.RLock()
-			defer rr.mu.RUnlock()
-			return float64(rr.lastExecSamples)
+			e := rr.state.getLast()
+			return float64(e.samples)
 		})
 	return rr
 }
@@ -124,17 +113,30 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }

 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
-	rr.mu.Lock()
-	defer rr.mu.Unlock()
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, req, err := rr.q.Query(ctx, rr.Expr, ts)
+	curState := ruleStateEntry{
+		time:     start,
+		at:       ts,
+		duration: time.Since(start),
+		samples:  len(qMetrics),
+		req:      req,
+	}
+
+	defer func() {
+		rr.state.add(curState)
+	}()

-	rr.lastExecTime = ts
-	rr.lastExecDuration = time.Since(ts)
-	rr.lastExecError = err
-	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
-		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+		curState.err = fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+		return nil, curState.err
+	}
+
+	numSeries := len(qMetrics)
+	if limit > 0 && numSeries > limit {
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d series", limit, numSeries)
+		return nil, curState.err
 	}

 	duplicates := make(map[string]struct{}, len(qMetrics))
@@ -143,8 +145,8 @@ func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarsha
 		ts := rr.toTimeSeries(r)
 		key := stringifyLabels(ts)
 		if _, ok := duplicates[key]; ok {
-			rr.lastExecError = errDuplicate
-			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
+			curState.err = fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
+			return nil, curState.err
 		}
 		duplicates[key] = struct{}{}
 		tss = append(tss, ts)
@@ -199,23 +201,25 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 // ToAPI returns Rule's representation in form
 // of APIRule
 func (rr *RecordingRule) ToAPI() APIRule {
+	lastState := rr.state.getLast()
 	r := APIRule{
 		Type:           "recording",
 		DatasourceType: rr.Type.String(),
 		Name:           rr.Name,
 		Query:          rr.Expr,
 		Labels:         rr.Labels,
-		LastEvaluation: rr.lastExecTime,
-		EvaluationTime: rr.lastExecDuration.Seconds(),
+		LastEvaluation: lastState.time,
+		EvaluationTime: lastState.duration.Seconds(),
 		Health:         "ok",
-		LastSamples:    rr.lastExecSamples,
+		LastSamples:    lastState.samples,
+		Updates:        rr.state.getAll(),
+
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", rr.ID()),
 		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
-
-	if rr.lastExecError != nil {
-		r.LastError = rr.lastExecError.Error()
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
 		r.Health = "err"
 	}
 	return r
--- a/app/vmalert/recording_test.go
+++ b/app/vmalert/recording_test.go
@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )

-func TestRecoridngRule_Exec(t *testing.T) {
+func TestRecordingRule_Exec(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -19,7 +19,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 		expTS   []prompbmarshal.TimeSeries
 	}{
 		{
-			&RecordingRule{Name: "foo"},
+			&RecordingRule{Name: "foo", state: newRuleState()},
 			[]datasource.Metric{metricWithValueAndLabels(t, 10,
 				"__name__", "bar",
 			)},
@@ -30,7 +30,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			&RecordingRule{Name: "foobarbaz"},
+			&RecordingRule{Name: "foobarbaz", state: newRuleState()},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
@@ -52,9 +52,12 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			&RecordingRule{Name: "job:foo", Labels: map[string]string{
-				"source": "test",
-			}},
+			&RecordingRule{
+				Name:  "job:foo",
+				state: newRuleState(),
+				Labels: map[string]string{
+					"source": "test",
+				}},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
@@ -77,7 +80,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO(), time.Now())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now(), 0)
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -88,7 +91,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 	}
 }

-func TestRecoridngRule_ExecRange(t *testing.T) {
+func TestRecordingRule_ExecRange(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -169,16 +172,61 @@ func TestRecoridngRule_ExecRange(t *testing.T) {
 	}
 }

-func TestRecoridngRule_ExecNegative(t *testing.T) {
-	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
-		"job": "test",
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
+		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+	}
+	rule := &RecordingRule{Name: "job:foo", state: newRuleState(), Labels: map[string]string{
+		"source": "test_limit",
 	}}
+	var err error
+	for _, testCase := range testCases {
+		fq := &fakeQuerier{}
+		fq.add(testMetrics...)
+		rule.q = fq
+		_, err = rule.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRecordingRule_ExecNegative(t *testing.T) {
+	rr := &RecordingRule{
+		Name:  "job:foo",
+		state: newRuleState(),
+		Labels: map[string]string{
+			"job": "test",
+		},
+	}

 	fq := &fakeQuerier{}
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO(), time.Now())
+	_, err := rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +241,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))

-	_, err = rr.Exec(context.TODO(), time.Now())
+	_, err = rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
--- a/app/vmalert/remoteread/init.go
+++ b/app/vmalert/remoteread/init.go
@@ -7,12 +7,21 @@ import (

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )

 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
-		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+		"Remote read is used to restore alerts state."+
+		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+		"E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
+
+	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("remoteRead.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteRead.url. "+
+		"For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. "+
+		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")

 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
@@ -36,6 +45,13 @@ var (
 	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )

+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteReadURL {
+		flagutil.RegisterSecretFlag("remoteRead.url")
+	}
+}
+
 // Init creates a Querier from provided flag values.
 // Returns nil if addr flag wasn't set.
 func Init() (datasource.QuerierBuilder, error) {
@@ -50,7 +66,8 @@ func Init() (datasource.QuerierBuilder, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
--- a/app/vmalert/remotewrite/init.go
+++ b/app/vmalert/remotewrite/init.go
@@ -7,12 +7,19 @@ import (
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )

 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
-		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("remoteWrite.headers", "", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")

 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
@@ -41,6 +48,13 @@ var (
 	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )

+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init creates Client object from given flags.
 // Returns nil if addr flag wasn't set.
 func Init(ctx context.Context) (*Client, error) {
@@ -56,7 +70,8 @@ func Init(ctx context.Context) (*Client, error) {
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
--- a/app/vmalert/remotewrite/remotewrite.go
+++ b/app/vmalert/remotewrite/remotewrite.go
@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"strings"
@@ -236,10 +236,16 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
+
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+
 	if c.authCfg != nil {
-		if auth := c.authCfg.GetAuthHeader(); auth != "" {
-			req.Header.Set("Authorization", auth)
-		}
+		c.authCfg.SetHeaders(req, true)
 	}
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")
@@ -251,7 +257,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
 			resp.StatusCode, req.URL.Redacted(), body)
 	}
--- a/app/vmalert/remotewrite/remotewrite_test.go
+++ b/app/vmalert/remotewrite/remotewrite_test.go
@@ -3,7 +3,7 @@ package remotewrite
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
@@ -80,7 +80,23 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("bad method %q", r.Method))
 		return
 	}
-	data, err := ioutil.ReadAll(r.Body)
+
+	h := r.Header.Get("Content-Encoding")
+	if h != "snappy" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	}
+
+	h = r.Header.Get("Content-Type")
+	if h != "application/x-protobuf" {
+		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
+	}
+
+	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
+	if h != "0.1.0" {
+		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	}
+
+	data, err := io.ReadAll(r.Body)
 	if err != nil {
 		rw.err(w, fmt.Errorf("body read err: %w", err))
 		return
--- a/app/vmalert/replay.go
+++ b/app/vmalert/replay.go
@@ -7,12 +7,13 @@ import (
 	"strings"
 	"time"

+	"github.com/cheggaaa/pb/v3"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/dmitryk-dk/pb/v3"
 )

 var (
@@ -87,6 +88,10 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 		"\nrequests to make: \t%d"+
 		"\nmax range per request: \t%v\n",
 		g.Name, g.Interval, iterations, step)
+	if g.Limit > 0 {
+		fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
+			g.Limit)
+	}
 	for _, rule := range g.Rules {
 		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
 		var bar *pb.ProgressBar
--- a/app/vmalert/rule.go
+++ b/app/vmalert/rule.go
@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"errors"
+	"net/http"
+	"sync"
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -15,9 +17,10 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context at the given timestamp
-	Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error)
-	// ExecRange executes the rule on the given time range
+	// Exec executes the rule with given context at the given timestamp and limit.
+	// returns an err if number of resulting time series exceeds the limit.
+	Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error)
+	// ExecRange executes the rule on the given time range.
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
@@ -30,3 +33,74 @@ type Rule interface {
 }

 var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
+
+type ruleState struct {
+	sync.RWMutex
+	entries []ruleStateEntry
+	cur     int
+}
+
+type ruleStateEntry struct {
+	// stores last moment of time rule.Exec was called
+	time time.Time
+	// stores the timesteamp with which rule.Exec was called
+	at time.Time
+	// stores the duration of the last rule.Exec call
+	duration time.Duration
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health ruleState
+	err error
+	// stores the number of samples returned during
+	// the last evaluation
+	samples int
+	// stores the HTTP request used by datasource during rule.Exec
+	req *http.Request
+}
+
+const defaultStateEntriesLimit = 20
+
+func newRuleState() *ruleState {
+	return &ruleState{
+		entries: make([]ruleStateEntry, defaultStateEntriesLimit),
+	}
+}
+
+func (s *ruleState) getLast() ruleStateEntry {
+	s.RLock()
+	defer s.RUnlock()
+	return s.entries[s.cur]
+}
+
+func (s *ruleState) getAll() []ruleStateEntry {
+	entries := make([]ruleStateEntry, 0)
+
+	s.RLock()
+	defer s.RUnlock()
+
+	cur := s.cur
+	for {
+		e := s.entries[cur]
+		if !e.time.IsZero() || !e.at.IsZero() {
+			entries = append(entries, e)
+		}
+		cur--
+		if cur < 0 {
+			cur = cap(s.entries) - 1
+		}
+		if cur == s.cur {
+			return entries
+		}
+	}
+}
+
+func (s *ruleState) add(e ruleStateEntry) {
+	s.Lock()
+	defer s.Unlock()
+
+	s.cur++
+	if s.cur > cap(s.entries)-1 {
+		s.cur = 0
+	}
+	s.entries[s.cur] = e
+}
--- a/app/vmalert/rule_test.go
+++ b/app/vmalert/rule_test.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestRule_state(t *testing.T) {
+	state := newRuleState()
+	e := state.getLast()
+	if !e.at.IsZero() {
+		t.Fatalf("expected entry to be zero")
+	}
+
+	now := time.Now()
+	state.add(ruleStateEntry{at: now})
+
+	e = state.getLast()
+	if e.at != now {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, now)
+	}
+
+	time.Sleep(time.Millisecond)
+	now2 := time.Now()
+	state.add(ruleStateEntry{at: now2})
+
+	e = state.getLast()
+	if e.at != now2 {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, now2)
+	}
+
+	if len(state.getAll()) != 2 {
+		t.Fatalf("expected for state to have 2 entries only; got %d",
+			len(state.getAll()),
+		)
+	}
+
+	var last time.Time
+	for i := 0; i < defaultStateEntriesLimit*2; i++ {
+		last = time.Now()
+		state.add(ruleStateEntry{at: last})
+	}
+
+	e = state.getLast()
+	if e.at != last {
+		t.Fatalf("expected entry at %v to be equal to %v",
+			e.at, last)
+	}
+
+	if len(state.getAll()) != defaultStateEntriesLimit {
+		t.Fatalf("expected for state to have %d entries only; got %d",
+			defaultStateEntriesLimit, len(state.getAll()),
+		)
+	}
+}
+
+// TestRule_stateConcurrent supposed to test concurrent
+// execution of state updates.
+// Should be executed with -race flag
+func TestRule_stateConcurrent(t *testing.T) {
+	state := newRuleState()
+
+	const workers = 50
+	const iterations = 100
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				state.add(ruleStateEntry{at: time.Now()})
+				state.getAll()
+				state.getLast()
+			}
+		}()
+	}
+	wg.Wait()
+}
--- a/app/vmalert/static/css/bootstrap.min.css
+++ b/app/vmalert/static/css/bootstrap.min.css
--- a/app/vmalert/static/js/bootstrap.bundle.min.js
+++ b/app/vmalert/static/js/bootstrap.bundle.min.js
--- a/app/vmalert/static/js/jquery-3.6.0.min.js
+++ b/app/vmalert/static/js/jquery-3.6.0.min.js
--- a/app/vmalert/templates/template.go
+++ b/app/vmalert/templates/template.go
@@ -17,7 +17,7 @@ import (
 	"errors"
 	"fmt"
 	htmlTpl "html/template"
-	"io/ioutil"
+	"io"
 	"math"
 	"net"
 	"net/url"
@@ -71,7 +71,7 @@ func Load(pathPatterns []string, overwrite bool) error {
 		}
 	}
 	if len(tmpl.Templates()) > 0 {
-		err := tmpl.Execute(ioutil.Discard, nil)
+		err := tmpl.Execute(io.Discard, nil)
 		if err != nil {
 			return fmt.Errorf("failed to execute template: %w", err)
 		}
@@ -255,6 +255,15 @@ func templateFuncs() textTpl.FuncMap {
 			return d.Seconds(), nil
 		},

+		// same with parseDuration but returns a time.Duration
+		"parseDurationTime": func(s string) (time.Duration, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d, nil
+		},
+
 		/* Numbers */

 		// humanize converts given number to a human readable format
@@ -373,10 +382,23 @@ func templateFuncs() textTpl.FuncMap {
 			if math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v), nil
 			}
-			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
+			t := timeFromUnixTimestamp(v).Time().UTC()
 			return fmt.Sprint(t), nil
 		},

+		// toTime converts given timestamp to a time.Time.
+		"toTime": func(i interface{}) (time.Time, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return time.Time{}, err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return t, nil
+		},
+
 		/* URLs */

 		// externalURL returns value of `external.url` flag
@@ -492,10 +514,9 @@ func templateFuncs() textTpl.FuncMap {
 // (1970-01-01 00:00 UTC) excluding leap seconds.
 type Time int64

-// TimeFromUnixNano returns the Time equivalent to the Unix Time
-// t provided in nanoseconds.
-func TimeFromUnixNano(t int64) Time {
-	return Time(t / nanosPerTick)
+// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
+func timeFromUnixTimestamp(t float64) Time {
+	return Time(t * 1e3)
 }

 // The number of nanoseconds per minimum tick.
--- a/app/vmalert/templates/template_test.go
+++ b/app/vmalert/templates/template_test.go
@@ -185,7 +185,7 @@ func TestTemplates_Load(t *testing.T) {
 			}
 			if tc.expErr != "" && err == nil {
 				t.Error("%+w", err)
-				t.Error("expected error that didn't happend")
+				t.Error("expected error that didn't happened")
 			}
 			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
 				t.Error("%+w", err)
--- a/app/vmalert/tpl/footer.qtpl
+++ b/app/vmalert/tpl/footer.qtpl
@@ -1,7 +1,15 @@
-{% func Footer() %}
+{% import (
+    "net/http"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+) %}
+
+
+{% func Footer(r *http.Request) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
        </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="{%s prefix %}static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="{%s prefix %}static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
        <script type="text/javascript">
            function expandAll() {
               $('.collapse').addClass('show');
@@ -18,14 +26,14 @@

              $(".group-heading").click(function(e) {
                 let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                  new bootstrap.Collapse(el, {
                    toggle: true
                  });
              });

              var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
              if (group.length > 0) {
                group.click();
              }
--- a/app/vmalert/tpl/footer.qtpl.go
+++ b/app/vmalert/tpl/footer.qtpl.go
@@ -5,25 +5,46 @@
 package tpl

 //line app/vmalert/tpl/footer.qtpl:1
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+//line app/vmalert/tpl/footer.qtpl:8
 import (
 	qtio422016 "io"

 	qt422016 "github.com/valyala/quicktemplate"
 )

-//line app/vmalert/tpl/footer.qtpl:1
+//line app/vmalert/tpl/footer.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )

-//line app/vmalert/tpl/footer.qtpl:1
-func StreamFooter(qw422016 *qt422016.Writer) {
-//line app/vmalert/tpl/footer.qtpl:1
+//line app/vmalert/tpl/footer.qtpl:8
+func StreamFooter(qw422016 *qt422016.Writer, r *http.Request) {
+//line app/vmalert/tpl/footer.qtpl:8
+	qw422016.N().S(`
+    `)
+//line app/vmalert/tpl/footer.qtpl:9
+	prefix := utils.Prefix(r.URL.Path)
+
+//line app/vmalert/tpl/footer.qtpl:9
 	qw422016.N().S(`
        </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="`)
+//line app/vmalert/tpl/footer.qtpl:11
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/footer.qtpl:11
+	qw422016.N().S(`static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="`)
+//line app/vmalert/tpl/footer.qtpl:12
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/footer.qtpl:12
+	qw422016.N().S(`static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
        <script type="text/javascript">
            function expandAll() {
               $('.collapse').addClass('show');
@@ -40,14 +61,14 @@ func StreamFooter(qw422016 *qt422016.Writer) {

              $(".group-heading").click(function(e) {
                 let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                  new bootstrap.Collapse(el, {
                    toggle: true
                  });
              });

              var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
              if (group.length > 0) {
                group.click();
              }
@@ -56,31 +77,31 @@ func StreamFooter(qw422016 *qt422016.Writer) {
    </body>
 </html>
 `)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }

-//line app/vmalert/tpl/footer.qtpl:36
-func WriteFooter(qq422016 qtio422016.Writer) {
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+func WriteFooter(qq422016 qtio422016.Writer, r *http.Request) {
+//line app/vmalert/tpl/footer.qtpl:44
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/footer.qtpl:36
-	StreamFooter(qw422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+	StreamFooter(qw422016, r)
+//line app/vmalert/tpl/footer.qtpl:44
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }

-//line app/vmalert/tpl/footer.qtpl:36
-func Footer() string {
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+func Footer(r *http.Request) string {
+//line app/vmalert/tpl/footer.qtpl:44
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/footer.qtpl:36
-	WriteFooter(qb422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
+	WriteFooter(qb422016, r)
+//line app/vmalert/tpl/footer.qtpl:44
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 	return qs422016
-//line app/vmalert/tpl/footer.qtpl:36
+//line app/vmalert/tpl/footer.qtpl:44
 }
--- a/app/vmalert/tpl/header.qtpl
+++ b/app/vmalert/tpl/header.qtpl
@@ -1,9 +1,19 @@
-{% func Header(title string, pages []NavItem) %}
+{% import (
+    "strings"
+    "net/http"
+    "path"
+    "net/url"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+) %}
+
+{% func Header(r *http.Request, navItems []NavItem, title string) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
    <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="{%s prefix %}static/css/bootstrap.min.css" rel="stylesheet" />
    <style>
        body{
          min-height: 75rem;
@@ -49,9 +59,53 @@
          background-color: rgba(0,0,0,.5);
          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
        }
+        textarea.curl-area{
+            width: 100%;
+            line-height: 1;
+            font-size: 12px;
+            border: none;
+            margin: 0;
+            padding: 0;
+            overflow: scroll;
+        }
    </style>
 </head>
 <body>
-    {%= PrintNavItems(title, pages) %}
+    {%= printNavItems(r, title, navItems) %}
    <main class="px-2">
 {% endfunc %}
+
+
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func printNavItems(r *http.Request, current string, items []NavItem) %}
+{%code
+    prefix := "/vmalert/"
+    if strings.HasPrefix(r.URL.Path, prefix) {
+        prefix = ""
+    }
+%}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    {% code
+                    u, _ := url.Parse(item.Url)
+                    %}
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}"
+                       href="{% if u.IsAbs() %}{%s item.Url %}{% else %}{%s path.Join(prefix,item.Url) %}{% endif %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}
--- a/app/vmalert/tpl/header.qtpl.go
+++ b/app/vmalert/tpl/header.qtpl.go
@@ -5,37 +5,57 @@
 package tpl

 //line app/vmalert/tpl/header.qtpl:1
+import (
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+//line app/vmalert/tpl/header.qtpl:10
 import (
 	qtio422016 "io"

 	qt422016 "github.com/valyala/quicktemplate"
 )

-//line app/vmalert/tpl/header.qtpl:1
+//line app/vmalert/tpl/header.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )

-//line app/vmalert/tpl/header.qtpl:1
-func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
-//line app/vmalert/tpl/header.qtpl:1
+//line app/vmalert/tpl/header.qtpl:10
+func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string) {
+//line app/vmalert/tpl/header.qtpl:10
+	qw422016.N().S(`
+    `)
+//line app/vmalert/tpl/header.qtpl:11
+	prefix := utils.Prefix(r.URL.Path)
+
+//line app/vmalert/tpl/header.qtpl:11
 	qw422016.N().S(`
 <!DOCTYPE html>
 <html lang="en">
 <head>
    <title>vmalert`)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	if title != "" {
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 		qw422016.N().S(` - `)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 		qw422016.E().S(title)
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	}
-//line app/vmalert/tpl/header.qtpl:5
+//line app/vmalert/tpl/header.qtpl:15
 	qw422016.N().S(`</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="`)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.N().S(`static/css/bootstrap.min.css" rel="stylesheet" />
    <style>
        body{
          min-height: 75rem;
@@ -81,41 +101,152 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
          background-color: rgba(0,0,0,.5);
          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
        }
+        textarea.curl-area{
+            width: 100%;
+            line-height: 1;
+            font-size: 12px;
+            border: none;
+            margin: 0;
+            padding: 0;
+            overflow: scroll;
+        }
    </style>
 </head>
 <body>
    `)
-//line app/vmalert/tpl/header.qtpl:55
-	StreamPrintNavItems(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:55
+//line app/vmalert/tpl/header.qtpl:74
+	streamprintNavItems(qw422016, r, title, navItems)
+//line app/vmalert/tpl/header.qtpl:74
 	qw422016.N().S(`
    <main class="px-2">
 `)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 }

-//line app/vmalert/tpl/header.qtpl:57
-func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string) {
+//line app/vmalert/tpl/header.qtpl:76
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:57
-	StreamHeader(qw422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+	StreamHeader(qw422016, r, navItems, title)
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 }

-//line app/vmalert/tpl/header.qtpl:57
-func Header(title string, pages []NavItem) string {
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+func Header(r *http.Request, navItems []NavItem, title string) string {
+//line app/vmalert/tpl/header.qtpl:76
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:57
-	WriteHeader(qb422016, title, pages)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+	WriteHeader(qb422016, r, navItems, title)
+//line app/vmalert/tpl/header.qtpl:76
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:57
+//line app/vmalert/tpl/header.qtpl:76
+}
+
+//line app/vmalert/tpl/header.qtpl:80
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/header.qtpl:86
+func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem) {
+//line app/vmalert/tpl/header.qtpl:86
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:88
+	prefix := "/vmalert/"
+	if strings.HasPrefix(r.URL.Path, prefix) {
+		prefix = ""
+	}
+
+//line app/vmalert/tpl/header.qtpl:92
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/header.qtpl:97
+	for _, item := range items {
+//line app/vmalert/tpl/header.qtpl:97
+		qw422016.N().S(`
+                <li class="nav-item">
+                    `)
+//line app/vmalert/tpl/header.qtpl:100
+		u, _ := url.Parse(item.Url)
+
+//line app/vmalert/tpl/header.qtpl:101
+		qw422016.N().S(`
+                    <a class="nav-link`)
+//line app/vmalert/tpl/header.qtpl:102
+		if current == item.Name {
+//line app/vmalert/tpl/header.qtpl:102
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/header.qtpl:102
+		}
+//line app/vmalert/tpl/header.qtpl:102
+		qw422016.N().S(`"
+                       href="`)
+//line app/vmalert/tpl/header.qtpl:103
+		if u.IsAbs() {
+//line app/vmalert/tpl/header.qtpl:103
+			qw422016.E().S(item.Url)
+//line app/vmalert/tpl/header.qtpl:103
+		} else {
+//line app/vmalert/tpl/header.qtpl:103
+			qw422016.E().S(path.Join(prefix, item.Url))
+//line app/vmalert/tpl/header.qtpl:103
+		}
+//line app/vmalert/tpl/header.qtpl:103
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/header.qtpl:104
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/header.qtpl:104
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/header.qtpl:107
+	}
+//line app/vmalert/tpl/header.qtpl:107
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/header.qtpl:111
+}
+
+//line app/vmalert/tpl/header.qtpl:111
+func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem) {
+//line app/vmalert/tpl/header.qtpl:111
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:111
+	streamprintNavItems(qw422016, r, current, items)
+//line app/vmalert/tpl/header.qtpl:111
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:111
+}
+
+//line app/vmalert/tpl/header.qtpl:111
+func printNavItems(r *http.Request, current string, items []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:111
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:111
+	writeprintNavItems(qb422016, r, current, items)
+//line app/vmalert/tpl/header.qtpl:111
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:111
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:111
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:111
 }
--- a/app/vmalert/tpl/nav.qtpl
+++ b/app/vmalert/tpl/nav.qtpl
@@ -1,25 +0,0 @@
-{% code
-type NavItem struct {
-	Name string
-	Url  string
-}
-%}
-
-{% func PrintNavItems(current string, items []NavItem) %}
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
-  <div class="container-fluid">
-    <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
-            {% for _, item := range items  %}
-                <li class="nav-item">
-                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
-                        {%s item.Name %}
-                    </a>
-                </li>
-            {% endfor %}
-        </ul>
-  </div>
-</nav>
-{% endfunc %}
-
-
--- a/app/vmalert/tpl/nav.qtpl.go
+++ b/app/vmalert/tpl/nav.qtpl.go
@@ -1,96 +0,0 @@
-// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vmalert/tpl/nav.qtpl:1
-package tpl
-
-//line app/vmalert/tpl/nav.qtpl:1
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vmalert/tpl/nav.qtpl:1
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vmalert/tpl/nav.qtpl:2
-type NavItem struct {
-	Name string
-	Url  string
-}
-
-//line app/vmalert/tpl/nav.qtpl:8
-func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
-//line app/vmalert/tpl/nav.qtpl:8
-	qw422016.N().S(`
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
-  <div class="container-fluid">
-    <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
-            `)
-//line app/vmalert/tpl/nav.qtpl:13
-	for _, item := range items {
-//line app/vmalert/tpl/nav.qtpl:13
-		qw422016.N().S(`
-                <li class="nav-item">
-                    <a class="nav-link`)
-//line app/vmalert/tpl/nav.qtpl:15
-		if current == item.Name {
-//line app/vmalert/tpl/nav.qtpl:15
-			qw422016.N().S(` active`)
-//line app/vmalert/tpl/nav.qtpl:15
-		}
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.N().S(`" href="`)
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.E().S(item.Url)
-//line app/vmalert/tpl/nav.qtpl:15
-		qw422016.N().S(`">
-                        `)
-//line app/vmalert/tpl/nav.qtpl:16
-		qw422016.E().S(item.Name)
-//line app/vmalert/tpl/nav.qtpl:16
-		qw422016.N().S(`
-                    </a>
-                </li>
-            `)
-//line app/vmalert/tpl/nav.qtpl:19
-	}
-//line app/vmalert/tpl/nav.qtpl:19
-	qw422016.N().S(`
-        </ul>
-  </div>
-</nav>
-`)
-//line app/vmalert/tpl/nav.qtpl:23
-}
-
-//line app/vmalert/tpl/nav.qtpl:23
-func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
-//line app/vmalert/tpl/nav.qtpl:23
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/nav.qtpl:23
-	StreamPrintNavItems(qw422016, current, items)
-//line app/vmalert/tpl/nav.qtpl:23
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/nav.qtpl:23
-}
-
-//line app/vmalert/tpl/nav.qtpl:23
-func PrintNavItems(current string, items []NavItem) string {
-//line app/vmalert/tpl/nav.qtpl:23
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/nav.qtpl:23
-	WritePrintNavItems(qb422016, current, items)
-//line app/vmalert/tpl/nav.qtpl:23
-	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/nav.qtpl:23
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/nav.qtpl:23
-	return qs422016
-//line app/vmalert/tpl/nav.qtpl:23
-}
--- a/app/vmalert/utils.go
+++ b/app/vmalert/utils.go
@@ -1,7 +1,10 @@
 package main

 import (
+	"fmt"
+	"net/http"
 	"sort"
+	"strings"
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -47,3 +50,62 @@ func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarsha
 	ts.Labels = labels
 	return ts
 }
+
+type curlWriter struct {
+	b strings.Builder
+}
+
+func (cw *curlWriter) string() string {
+	res := "curl " + cw.b.String()
+	cw.b.Reset()
+	return strings.TrimSpace(res)
+}
+
+func (cw *curlWriter) addWithEsc(str string) {
+	escStr := `'` + strings.Replace(str, `'`, `'\''`, -1) + `'`
+	cw.add(escStr)
+}
+
+func (cw *curlWriter) add(str string) {
+	cw.b.WriteString(str)
+	cw.b.WriteString(" ")
+}
+
+func requestToCurl(req *http.Request) string {
+	if req.URL == nil {
+		return ""
+	}
+
+	cw := &curlWriter{}
+
+	schema := req.URL.Scheme
+	requestURL := req.URL.String()
+	if schema == "" {
+		schema = "http"
+		if req.TLS != nil {
+			schema = "https"
+		}
+		requestURL = schema + "://" + req.Host + requestURL
+	}
+
+	if schema == "https" {
+		cw.add("-k")
+	}
+
+	cw.add("-X")
+	cw.add(req.Method)
+
+	var keys []string
+	for k := range req.Header {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		cw.add("-H")
+		cw.addWithEsc(fmt.Sprintf("%s: %s", k, strings.Join(req.Header[k], " ")))
+	}
+
+	cw.addWithEsc(requestURL)
+	return cw.string()
+}
--- a/app/vmalert/utils/auth.go
+++ b/app/vmalert/utils/auth.go
@@ -58,3 +58,12 @@ func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string
 		}
 	}
 }
+
+// WithHeaders returns AuthConfigOptions and set Headers based on the given params
+func WithHeaders(headers string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if headers != "" {
+			config.Headers = strings.Split(headers, "^^")
+		}
+	}
+}
--- a/app/vmalert/utils/err_group.go
+++ b/app/vmalert/utils/err_group.go
@@ -3,24 +3,34 @@ package utils
 import (
 	"fmt"
 	"strings"
+	"sync"
 )

 // ErrGroup accumulates multiple errors
 // and produces single error message.
 type ErrGroup struct {
+	mu   sync.Mutex
 	errs []error
 }

 // Add adds a new error to group.
-// Isn't thread-safe.
+// Is thread-safe.
 func (eg *ErrGroup) Add(err error) {
+	eg.mu.Lock()
 	eg.errs = append(eg.errs, err)
+	eg.mu.Unlock()
 }

 // Err checks if group contains at least
 // one error.
 func (eg *ErrGroup) Err() error {
-	if eg == nil || len(eg.errs) == 0 {
+	if eg == nil {
+		return nil
+	}
+
+	eg.mu.Lock()
+	defer eg.mu.Unlock()
+	if len(eg.errs) == 0 {
 		return nil
 	}
 	return eg
@@ -28,6 +38,9 @@ func (eg *ErrGroup) Err() error {

 // Error satisfies Error interface
 func (eg *ErrGroup) Error() string {
+	eg.mu.Lock()
+	defer eg.mu.Unlock()
+
 	if len(eg.errs) == 0 {
 		return ""
 	}
--- a/app/vmalert/utils/err_group_test.go
+++ b/app/vmalert/utils/err_group_test.go
@@ -2,6 +2,7 @@ package utils

 import (
 	"errors"
+	"fmt"
 	"testing"
 )

@@ -36,3 +37,29 @@ func TestErrGroup(t *testing.T) {
 		}
 	}
 }
+
+// TestErrGroupConcurrent supposed to test concurrent
+// use of error group.
+// Should be executed with -race flag
+func TestErrGroupConcurrent(t *testing.T) {
+	eg := new(ErrGroup)
+
+	const writersN = 4
+	payload := make(chan error, writersN)
+	for i := 0; i < writersN; i++ {
+		go func() {
+			for err := range payload {
+				eg.Add(err)
+			}
+		}()
+	}
+
+	const iterations = 500
+	for i := 0; i < iterations; i++ {
+		payload <- fmt.Errorf("error %d", i)
+		if i%10 == 0 {
+			_ = eg.Err()
+		}
+	}
+	close(payload)
+}
--- a/app/vmalert/utils/links.go
+++ b/app/vmalert/utils/links.go
@@ -0,0 +1,13 @@
+package utils
+
+import "strings"
+
+const prefix = "/vmalert/"
+
+// Prefix returns "/vmalert/" prefix if it is missing in the path.
+func Prefix(path string) string {
+	if strings.HasPrefix(path, prefix) {
+		return ""
+	}
+	return prefix
+}
--- a/app/vmalert/utils/tls.go
+++ b/app/vmalert/utils/tls.go
@@ -4,8 +4,8 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"strings"
 )

@@ -38,7 +38,7 @@ func TLSConfig(certFile, keyFile, CAFile, serverName string, insecureSkipVerify

 	var rootCAs *x509.CertPool
 	if CAFile != "" {
-		pem, err := ioutil.ReadFile(CAFile)
+		pem, err := os.ReadFile(CAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot read `ca_file` %q: %w", CAFile, err)
 		}
--- a/app/vmalert/utils_test.go
+++ b/app/vmalert/utils_test.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestRequestToCurl(t *testing.T) {
+	f := func(req *http.Request, exp string) {
+		got := requestToCurl(req)
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q instead", exp, got)
+		}
+	}
+
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	f(req, "curl -X POST 'http://foo.com'")
+
+	req, _ = http.NewRequest(http.MethodGet, "https://foo.com", nil)
+	f(req, "curl -k -X GET 'https://foo.com'")
+
+	req, _ = http.NewRequest(http.MethodPost, "foo.com", nil)
+	req.Header.Set("foo", "bar")
+	req.Header.Set("baz", "qux")
+	f(req, "curl -X POST -H 'Baz: qux' -H 'Foo: bar' 'http://foo.com'")
+
+	req, _ = http.NewRequest(http.MethodPost, "foo.com", nil)
+	params := req.URL.Query()
+	params.Add("query", "up")
+	params.Add("step", "10")
+	req.URL.RawQuery = params.Encode()
+	f(req, "curl -X POST 'http://foo.com?query=up&step=10'")
+
+	req, _ = http.NewRequest(http.MethodPost, "http://foo.com", nil)
+	params = req.URL.Query()
+	params.Add("query", "up")
+	params.Add("step", "10")
+	req.URL.RawQuery = params.Encode()
+	f(req, "curl -X POST 'http://foo.com?query=up&step=10'")
+
+	req, _ = http.NewRequest(http.MethodPost, "https://foo.com", nil)
+	params = req.URL.Query()
+	params.Add("query", "up")
+	params.Add("step", "10")
+	req.URL.RawQuery = params.Encode()
+	f(req, "curl -k -X POST 'https://foo.com?query=up&step=10'")
+}
--- a/app/vmalert/vmalert_multiple_rw.excalidraw
+++ b/app/vmalert/vmalert_multiple_rw.excalidraw
@@ -0,0 +1,711 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 797,
+      "versionNonce": 1977657992,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "miEbzHxOPXe4PEYvXiJp5"
+        },
+        {
+          "type": "arrow",
+          "id": "rcmiQfIWtfbTTlwxqr1sl"
+        },
+        {
+          "type": "arrow",
+          "id": "P-dpWlSTtnsux-zr5oqgF"
+        },
+        {
+          "type": "arrow",
+          "id": "oAToSPttH7aWoD_AqXGFX"
+        },
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "sxEhnxlbT7ldlSsmHDUHp"
+        },
+        {
+          "type": "arrow",
+          "id": "pD9DcILMxa6GaR1U5YyMO"
+        },
+        {
+          "type": "arrow",
+          "id": "HPEwr85wL4IedW0AgdArp"
+        },
+        {
+          "type": "arrow",
+          "id": "EyecK0YM9Cc8T6ju-nTOc"
+        },
+        {
+          "id": "xpdAlCCGgIMAgSaqQ4K65",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372487772,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 671,
+      "versionNonce": 1438327288,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 83,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372487772,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmagent",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": null,
+      "originalText": "vmagent"
+    },
+    {
+      "type": "rectangle",
+      "version": 1247,
+      "versionNonce": 1809504904,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 542.2673645019531,
+      "y": 308.46409606933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 219.1235961914062,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "he-SpFjCxEQEWpWny2kKP"
+        },
+        {
+          "type": "arrow",
+          "id": "-pjrKo16rOsasM8viZPJ-"
+        },
+        {
+          "id": "HPEwr85wL4IedW0AgdArp",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372610014,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1149,
+      "versionNonce": 1939391880,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 629.1559448242188,
+      "y": 318.8975372314453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 48,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372621140,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vm-1",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "vm-1"
+    },
+    {
+      "type": "arrow",
+      "version": 337,
+      "versionNonce": 1739475336,
+      "isDeleted": false,
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 423.70701599121094,
+      "y": 431.0309448437124,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 107.82342529296875,
+      "height": 100.61778190120276,
+      "seed": 389863732,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1655372610015,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": 0.9042666945544442,
+        "gap": 10.736923217773438
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          107.82342529296875,
+          -100.61778190120276
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 429,
+      "versionNonce": 252631288,
+      "isDeleted": false,
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 424.7585906982422,
+      "y": 432.4328003132737,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 119.94342041015625,
+      "height": 83.58206327156176,
+      "seed": 981082124,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1655372623571,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "lXpACjXQqK7SZF_vrACjJ",
+        "focus": -0.8650156795513397,
+        "gap": 3.6341629028320312
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          119.94342041015625,
+          83.58206327156176
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 979,
+      "versionNonce": 896077192,
+      "isDeleted": false,
+      "id": "X08ptHmEm7tCgoFbQntAR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": -4.634010314941406,
+      "y": 402.69072341918945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1000953848,
+      "groupIds": [
+        "IAd7y_6yDxq13U11FuJvH"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "miEbzHxOPXe4PEYvXiJp5"
+        },
+        {
+          "type": "arrow",
+          "id": "rcmiQfIWtfbTTlwxqr1sl"
+        },
+        {
+          "type": "arrow",
+          "id": "P-dpWlSTtnsux-zr5oqgF"
+        },
+        {
+          "type": "arrow",
+          "id": "oAToSPttH7aWoD_AqXGFX"
+        },
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "sxEhnxlbT7ldlSsmHDUHp"
+        },
+        {
+          "type": "arrow",
+          "id": "pD9DcILMxa6GaR1U5YyMO"
+        },
+        {
+          "type": "arrow",
+          "id": "HPEwr85wL4IedW0AgdArp"
+        },
+        {
+          "type": "arrow",
+          "id": "EyecK0YM9Cc8T6ju-nTOc"
+        },
+        {
+          "id": "xpdAlCCGgIMAgSaqQ4K65",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 844,
+      "versionNonce": 2073980664,
+      "isDeleted": false,
+      "id": "4lz3UmUePrjYOJGyMEsNo",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 16.912559509277344,
+      "y": 423.7749671936035,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 808600456,
+      "groupIds": [
+        "IAd7y_6yDxq13U11FuJvH"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": null,
+      "originalText": "vmalert"
+    },
+    {
+      "id": "xpdAlCCGgIMAgSaqQ4K65",
+      "type": "arrow",
+      "x": 127.58199310302739,
+      "y": 437.3415815729096,
+      "width": 154.43469238281244,
+      "height": 0.2578931190849971,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1769759112,
+      "version": 140,
+      "versionNonce": 1727929480,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          154.43469238281244,
+          0.2578931190849971
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "X08ptHmEm7tCgoFbQntAR",
+        "focus": -0.042373209435744755,
+        "gap": 8.45587158203125
+      },
+      "endBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.062483627408895646,
+        "gap": 7.663612365722656
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 896,
+      "versionNonce": 619040760,
+      "isDeleted": false,
+      "id": "d_hJkkcPArQGdFiPDbjtp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 129.2102279663086,
+      "y": 404.1378517150879,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 2108447992,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        }
+      ],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "persist results"
+    },
+    {
+      "id": "P35cFQroIm2nrmm3Jlqgp",
+      "type": "text",
+      "x": -7.461128234863281,
+      "y": 483.3255958557129,
+      "width": 301,
+      "height": 20,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1314060792,
+      "version": 179,
+      "versionNonce": 139280376,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372636346,
+      "link": null,
+      "locked": false,
+      "text": " -remoteWrite.url=http://vmagent",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 16,
+      "containerId": null,
+      "originalText": " -remoteWrite.url=http://vmagent"
+    },
+    {
+      "type": "rectangle",
+      "version": 1339,
+      "versionNonce": 812947448,
+      "isDeleted": false,
+      "id": "lXpACjXQqK7SZF_vrACjJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 548.3361740112305,
+      "y": 487.1258888244629,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 219.1235961914062,
+      "height": 44.74725341796875,
+      "seed": 333549960,
+      "groupIds": [
+        "vuLTnxw8A0DXtmDYT1F4r"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "he-SpFjCxEQEWpWny2kKP"
+        },
+        {
+          "type": "arrow",
+          "id": "-pjrKo16rOsasM8viZPJ-"
+        },
+        {
+          "id": "EyecK0YM9Cc8T6ju-nTOc",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372623571,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1244,
+      "versionNonce": 666803448,
+      "isDeleted": false,
+      "id": "v9qzZSsHdJ_ETRlP4Msn5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 635.2247543334961,
+      "y": 497.55932998657227,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 48,
+      "height": 24,
+      "seed": 1105210104,
+      "groupIds": [
+        "vuLTnxw8A0DXtmDYT1F4r"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372625794,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vm-2",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "vm-2"
+    },
+    {
+      "id": "yb3B2pFN0OZOd4yLmSU2m",
+      "type": "text",
+      "x": 449.79036712646484,
+      "y": 406.616886138916,
+      "width": 442,
+      "height": 20,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1374332808,
+      "version": 196,
+      "versionNonce": 526480264,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372596999,
+      "link": null,
+      "locked": false,
+      "text": "-remoteWrite.url=https://vm-1:8428/api/v1/write",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 16,
+      "containerId": null,
+      "originalText": "-remoteWrite.url=https://vm-1:8428/api/v1/write"
+    },
+    {
+      "type": "text",
+      "version": 242,
+      "versionNonce": 1304477832,
+      "isDeleted": false,
+      "id": "8CXNdrDePIAAwgJB2b8YT",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 450.0511703491211,
+      "y": 432.6653480529785,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "width": 442,
+      "height": 20,
+      "seed": 1349606392,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372600292,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "-remoteWrite.url=https://vm-2:8428/api/v1/write",
+      "baseline": 16,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "-remoteWrite.url=https://vm-2:8428/api/v1/write"
+    },
+    {
+      "type": "text",
+      "version": 1195,
+      "versionNonce": 1912405496,
+      "isDeleted": false,
+      "id": "Ev-VujoFglVNIh5GIhsba",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 357.2894821166992,
+      "y": 370.6587562561035,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 114,
+      "height": 20,
+      "seed": 1289300104,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        }
+      ],
+      "updated": 1655372703770,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "fan-out data",
+      "baseline": 16,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "fan-out data"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}
--- a/app/vmalert/vmalert_multiple_rw.png
+++ b/app/vmalert/vmalert_multiple_rw.png
--- a/app/vmalert/vmalert_state.png
+++ b/app/vmalert/vmalert_state.png
--- a/app/vmalert/vmalert_ts_data_delay.gif
+++ b/app/vmalert/vmalert_ts_data_delay.gif
--- a/app/vmalert/vmalert_ts_normal.gif
+++ b/app/vmalert/vmalert_ts_normal.gif
--- a/app/vmalert/web.go
+++ b/app/vmalert/web.go
@@ -1,10 +1,10 @@
 package main

 import (
+	"embed"
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -24,23 +24,23 @@ var (
 )

 func initLinks() {
-	pathPrefix := httpserver.GetPathPrefix()
-	if pathPrefix == "" {
-		pathPrefix = "/"
-	}
 	apiLinks = [][2]string{
-		{path.Join(pathPrefix, "api/v1/rules"), "list all loaded groups and rules"},
-		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
-		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
-		{path.Join(pathPrefix, "flags"), "command-line flags"},
-		{path.Join(pathPrefix, "metrics"), "list of application metrics"},
-		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
+		// api links are relative since they can be used by external clients,
+		// such as Grafana, and proxied via vmselect.
+		{"api/v1/rules", "list all loaded groups and rules"},
+		{"api/v1/alerts", "list all active alerts"},
+		{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
+
+		// system links
+		{"/flags", "command-line flags"},
+		{"/metrics", "list of application metrics"},
+		{"/-/reload", "reload configuration"},
 	}
 	navItems = []tpl.NavItem{
-		{Name: "vmalert", Url: path.Join(pathPrefix, "/")},
-		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
-		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
-		{Name: "Notifiers", Url: path.Join(pathPrefix, "notifiers")},
+		{Name: "vmalert", Url: "."},
+		{Name: "Groups", Url: "groups"},
+		{Name: "Alerts", Url: "alerts"},
+		{Name: "Notifiers", Url: "notifiers"},
 		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
 	}
 }
@@ -49,33 +49,67 @@ type requestHandler struct {
 	m *manager
 }

+var (
+	//go:embed static
+	staticFiles   embed.FS
+	staticHandler = http.FileServer(http.FS(staticFiles))
+	staticServer  = http.StripPrefix("/vmalert", staticHandler)
+)
+
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	once.Do(func() {
 		initLinks()
 	})

-	pathPrefix := httpserver.GetPathPrefix()
-	if pathPrefix == "" {
-		pathPrefix = "/"
+	if strings.HasPrefix(r.URL.Path, "/vmalert/static") {
+		staticServer.ServeHTTP(w, r)
+		return true
 	}

 	switch r.URL.Path {
-	case "/":
+	case "/", "/vmalert", "/vmalert/":
 		if r.Method != "GET" {
+			httpserver.Errorf(w, r, "path %q supports only GET method", r.URL.Path)
 			return false
 		}
-		WriteWelcome(w)
+		WriteWelcome(w, r)
 		return true
-	case "/alerts":
-		WriteListAlerts(w, pathPrefix, rh.groupAlerts())
+	case "/vmalert/alerts":
+		WriteListAlerts(w, r, rh.groupAlerts())
 		return true
-	case "/groups", "/rules":
-		WriteListGroups(w, rh.groups())
+	case "/vmalert/alert":
+		alert, err := rh.getAlert(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		WriteAlert(w, r, alert)
 		return true
-	case "/notifiers":
-		WriteListTargets(w, notifier.GetTargets())
+	case "/vmalert/rule":
+		rule, err := rh.getRule(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		WriteRuleDetails(w, r, rule)
 		return true
-	case "/api/v1/rules":
+	case "/vmalert/groups":
+		WriteListGroups(w, r, rh.groups())
+		return true
+	case "/vmalert/notifiers":
+		WriteListTargets(w, r, notifier.GetTargets())
+		return true
+
+	// special cases for Grafana requests,
+	// served without `vmalert` prefix:
+	case "/rules":
+		// Grafana makes an extra request to `/rules`
+		// handler in addition to `/api/v1/rules` calls in alerts UI,
+		WriteListGroups(w, r, rh.groups())
+		return true
+
+	case "/vmalert/api/v1/rules", "/api/v1/rules":
+		// path used by Grafana for ng alerting
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
@@ -84,7 +118,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
-	case "/api/v1/alerts":
+	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
+		// path used by Grafana for ng alerting
 		data, err := rh.listAlerts()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
@@ -93,13 +128,34 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
+	case "/vmalert/api/v1/alert", "/api/v1/alert":
+		alert, err := rh.getAlert(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		data, err := json.Marshal(alert)
+		if err != nil {
+			httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(data)
+		return true
 	case "/-/reload":
 		logger.Infof("api config reload was called, sending sighup")
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+
 	default:
+		// Support of deprecated links:
+		// * /api/v1/<groupID>/<alertID>/status
+		// * <groupID>/<alertID>/status
+		// TODO: to remove in next versions
+
 		if !strings.HasSuffix(r.URL.Path, "/status") {
+			httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
 			return false
 		}
 		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
@@ -108,24 +164,53 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}

-		// /api/v1/<groupID>/<alertID>/status
+		redirectURL := alert.WebLink()
 		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
-			data, err := json.Marshal(alert)
-			if err != nil {
-				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
-				return true
-			}
-			w.Header().Set("Content-Type", "application/json")
-			w.Write(data)
-			return true
+			redirectURL = alert.APILink()
 		}
-
-		// <groupID>/<alertID>/status
-		WriteAlert(w, pathPrefix, alert)
+		httpserver.Redirect(w, "/"+redirectURL)
 		return true
 	}
 }

+const (
+	paramGroupID = "group_id"
+	paramAlertID = "alert_id"
+	paramRuleID  = "rule_id"
+)
+
+func (rh *requestHandler) getRule(r *http.Request) (APIRule, error) {
+	groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 0)
+	if err != nil {
+		return APIRule{}, fmt.Errorf("failed to read %q param: %s", paramGroupID, err)
+	}
+	ruleID, err := strconv.ParseUint(r.FormValue(paramRuleID), 10, 0)
+	if err != nil {
+		return APIRule{}, fmt.Errorf("failed to read %q param: %s", paramRuleID, err)
+	}
+	rule, err := rh.m.RuleAPI(groupID, ruleID)
+	if err != nil {
+		return APIRule{}, errResponse(err, http.StatusNotFound)
+	}
+	return rule, nil
+}
+
+func (rh *requestHandler) getAlert(r *http.Request) (*APIAlert, error) {
+	groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %q param: %s", paramGroupID, err)
+	}
+	alertID, err := strconv.ParseUint(r.FormValue(paramAlertID), 10, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %q param: %s", paramAlertID, err)
+	}
+	a, err := rh.m.AlertAPI(groupID, alertID)
+	if err != nil {
+		return nil, errResponse(err, http.StatusNotFound)
+	}
+	return a, nil
+}
+
 type listGroupsResponse struct {
 	Status string `json:"status"`
 	Data   struct {
@@ -191,6 +276,9 @@ func (rh *requestHandler) groupAlerts() []GroupAlerts {
 			})
 		}
 	}
+	sort.Slice(groupAlerts, func(i, j int) bool {
+		return groupAlerts[i].Group.Name < groupAlerts[j].Group.Name
+	})
 	return groupAlerts
 }

@@ -225,10 +313,10 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 }

 func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
-	rh.m.groupsMu.RLock()
-	defer rh.m.groupsMu.RUnlock()
-
-	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", 3)
+	if strings.HasPrefix(path, "/vmalert") {
+		path = strings.TrimLeft(path, "/vmalert")
+	}
+	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", -1)
 	if len(parts) != 3 {
 		return nil, &httpserver.ErrorWithStatusCode{
 			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
--- a/app/vmalert/web.qtpl
+++ b/app/vmalert/web.qtpl
@@ -3,15 +3,16 @@
 {% import (
    "time"
    "sort"
-    "path"
+    "net/http"

    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 ) %}


-{% func Welcome() %}
-    {%= tpl.Header("vmalert", navItems) %}
+{% func Welcome(r *http.Request) %}
+    {%= tpl.Header(r, navItems, "vmalert") %}
    <p>
        API:<br>
        {% for _, p := range apiLinks  %}
@@ -21,11 +22,12 @@
        	<a href="{%s p %}">{%s p %}</a> - {%s doc %}<br/>
        {% endfor %}
    </p>
-    {%= tpl.Footer() %}
+    {%= tpl.Footer(r) %}
 {% endfunc %}

-{% func ListGroups(groups []APIGroup) %}
-    {%= tpl.Header("Groups", navItems) %}
+{% func ListGroups(r *http.Request, groups []APIGroup) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
+    {%= tpl.Header(r, navItems, "Groups") %}
    {%  if len(groups) > 0 %}
        {%code
            rOk := make(map[string]int)
@@ -56,6 +58,13 @@
                    {% endfor %}
                    </div>
                {% endif %}
+                {% if len(g.Headers) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra headers
+                    {% for _, header := range g.Headers %}
+                            <span class="float-left badge bg-primary">{%s header %}</span>
+                    {% endfor %}
+                    </div>
+                {% endif %}
            </div>
            <div class="collapse" id="rules-{%s g.ID %}">
                <table class="table table-striped table-hover table-sm">
@@ -77,6 +86,7 @@
                                        {% else %}
                                        <b>record:</b> {%s r.Name %}
                                        {% endif %}
+                                        | <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                    </div>
                                    <div class="col-12">
                                        <code><pre>{%s r.Query %}</pre></code>
@@ -108,17 +118,18 @@

    {% else %}
        <div>
-            <p>No items...</p>
+            <p>No groups...</p>
        </div>
    {% endif %}

-    {%= tpl.Footer() %}
+    {%= tpl.Footer(r) %}

 {% endfunc %}


-{% func ListAlerts(pathPrefix string, groupAlerts []GroupAlerts) %}
-    {%= tpl.Header("Alerts", navItems) %}
+{% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
+    {%= tpl.Header(r, navItems, "Alerts") %}
    {% if len(groupAlerts) > 0 %}
         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@@ -182,7 +193,7 @@
                                </td>
                                <td>{%s ar.Value %}</td>
                                <td>
-                                    <a href="{%s path.Join(pathPrefix, g.ID, ar.ID, "status") %}">Details</a>
+                                    <a href="{%s prefix+ar.WebLink() %}">Details</a>
                                </td>
                            </tr>
                        {% endfor %}
@@ -195,16 +206,16 @@

    {% else %}
        <div>
-            <p>No items...</p>
+            <p>No active alerts...</p>
        </div>
    {% endif %}

-    {%= tpl.Footer() %}
+    {%= tpl.Footer(r) %}

 {% endfunc %}

-{% func ListTargets(targets map[notifier.TargetType][]notifier.Target) %}
-    {%= tpl.Header("Notifiers", navItems) %}
+{% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %}
+    {%= tpl.Header(r, navItems, "Notifiers") %}
    {% if len(targets) > 0 %}
         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@@ -251,16 +262,17 @@

    {% else %}
        <div>
-            <p>No items...</p>
+            <p>No targets...</p>
        </div>
    {% endif %}

-    {%= tpl.Footer() %}
+    {%= tpl.Footer(r) %}

 {% endfunc %}

-{% func Alert(pathPrefix string, alert *APIAlert) %}
-    {%= tpl.Header("", navItems) %}
+{% func Alert(r *http.Request, alert *APIAlert) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
+    {%= tpl.Header(r, navItems, "") %}
    {%code
        var labelKeys []string
        for k := range alert.Labels {
@@ -274,7 +286,7 @@
        }
        sort.Strings(annotationKeys)
    %}
-    <div class="display-6 pb-3 mb-3">{%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
+    <div class="display-6 pb-3 mb-3">Alert: {%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
    <div class="container border-bottom p-2">
      <div class="row">
        <div class="col-2">
@@ -326,7 +338,7 @@
          Group
        </div>
        <div class="col">
-           <a target="_blank" href="{%s path.Join(pathPrefix,"groups") %}#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
+           <a target="_blank" href="{%s prefix %}groups#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
        </div>
      </div>
    </div>
@@ -340,10 +352,129 @@
        </div>
      </div>
    </div>
-    {%= tpl.Footer() %}
+    {%= tpl.Footer(r) %}

 {% endfunc %}

+
+{% func RuleDetails(r *http.Request, rule APIRule) %}
+    {%code prefix := utils.Prefix(r.URL.Path) %}
+    {%= tpl.Header(r, navItems, "") %}
+    {%code
+        var labelKeys []string
+        for k := range rule.Labels {
+            labelKeys = append(labelKeys, k)
+        }
+        sort.Strings(labelKeys)
+
+        var annotationKeys []string
+        for k := range rule.Annotations {
+            annotationKeys = append(annotationKeys, k)
+        }
+        sort.Strings(annotationKeys)
+    %}
+    <div class="display-6 pb-3 mb-3">Rule: {%s rule.Name %}<span class="ms-2 badge {% if rule.Health!="ok" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s rule.Health %}</span></div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Expr
+        </div>
+        <div class="col">
+          <code><pre>{%s rule.Query %}</pre></code>
+        </div>
+      </div>
+    </div>
+    {% if rule.Type == "alerting" %}
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          For
+        </div>
+        <div class="col">
+         {%v rule.Duration %} seconds
+        </div>
+      </div>
+    </div>
+    {% endif %}
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Labels
+        </div>
+        <div class="col">
+          {% for _, k := range labelKeys %}
+                <span class="m-1 badge bg-primary">{%s k %}={%s rule.Labels[k] %}</span>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    {% if rule.Type == "alerting" %}
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Annotations
+        </div>
+        <div class="col">
+          {% for _, k := range annotationKeys %}
+                <b>{%s k %}:</b><br>
+                <p>{%s rule.Annotations[k] %}</p>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    {% endif %}
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Group
+        </div>
+        <div class="col">
+           <a target="_blank" href="{%s prefix %}groups#group-{%s rule.GroupID %}">{%s rule.GroupID %}</a>
+        </div>
+      </div>
+    </div>
+
+    <br>
+    <div class="display-6 pb-3">Last {%d len(rule.Updates) %} updates</span>:</div>
+        <table class="table table-striped table-hover table-sm">
+            <thead>
+                <tr>
+                    <th scope="col" title="The time when event was created">Updated at</th>
+                    <th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th>
+                    <th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
+                    <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
+                    <th scope="col" class="text-center" title="cURL command with request example">cURL</th>
+                </tr>
+            </thead>
+            <tbody>
+
+     {% for _, u := range rule.Updates %}
+             <tr{% if u.err != nil %} class="alert-danger"{% endif %}>
+                 <td>
+                    <span class="badge bg-primary rounded-pill me-3" title="Updated at">{%s u.time.Format(time.RFC3339) %}</span>
+                 </td>
+                 <td class="text-center" wi>{%d u.samples %}</td>
+                 <td class="text-center">{%f.3 u.duration.Seconds() %}s</td>
+                 <td class="text-center">{%s u.at.Format(time.RFC3339) %}</td>
+                 <td>
+                    <textarea class="curl-area" rows="1" onclick="this.focus();this.select()">{%s requestToCurl(u.req) %}</textarea>
+                </td>
+             </tr>
+          </li>
+          {% if u.err != nil %}
+             <tr{% if u.err != nil %} class="alert-danger"{% endif %}>
+               <td colspan="5">
+                   <span class="alert-danger">{%v u.err %}</span>
+               </td>
+             </tr>
+          {% endif %}
+     {% endfor %}
+
+    {%= tpl.Footer(r) %}
+{% endfunc %}
+
+
+
 {% func badgeState(state string) %}
 {%code
    badgeClass := "bg-warning text-dark"
--- a/app/vmalert/web.qtpl.go
+++ b/app/vmalert/web.qtpl.go
--- a/app/vmalert/web_test.go
+++ b/app/vmalert/web_test.go
@@ -2,6 +2,7 @@ package main

 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -16,6 +17,7 @@ func TestHandler(t *testing.T) {
 		alerts: map[uint64]*notifier.Alert{
 			0: {State: notifier.StateFiring},
 		},
+		state: newRuleState(),
 	}
 	g := &Group{
 		Name:  "group",
@@ -29,7 +31,7 @@ func TestHandler(t *testing.T) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
-			t.Errorf("unexpected err %s", err)
+			t.Fatalf("unexpected err %s", err)
 		}
 		if code != resp.StatusCode {
 			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
@@ -47,20 +49,87 @@ func TestHandler(t *testing.T) {
 	}
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rh.handler(w, r) }))
 	defer ts.Close()
+
+	t.Run("/", func(t *testing.T) {
+		getResp(ts.URL, nil, 200)
+		getResp(ts.URL+"/vmalert", nil, 200)
+		getResp(ts.URL+"/vmalert/alerts", nil, 200)
+		getResp(ts.URL+"/vmalert/groups", nil, 200)
+		getResp(ts.URL+"/vmalert/notifiers", nil, 200)
+		getResp(ts.URL+"/rules", nil, 200)
+	})
+
+	t.Run("/vmalert/rule", func(t *testing.T) {
+		a := ar.ToAPI()
+		getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+	})
+	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
+		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)
+		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
+
+		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramRuleID)
+		getResp(ts.URL+"/vmalert/rule"+params, nil, 404)
+	})
+
 	t.Run("/api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
 			t.Errorf("expected 1 alert got %d", length)
 		}
+
+		lr = listAlertsResponse{}
+		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		if length := len(lr.Data.Alerts); length != 1 {
+			t.Errorf("expected 1 alert got %d", length)
+		}
 	})
+	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
+		expAlert := ar.newAlertAPI(*ar.alerts[0])
+		alert := &APIAlert{}
+		getResp(ts.URL+"/"+expAlert.APILink(), alert, 200)
+		if !reflect.DeepEqual(alert, expAlert) {
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
+		}
+
+		alert = &APIAlert{}
+		getResp(ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
+		if !reflect.DeepEqual(alert, expAlert) {
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
+		}
+	})
+
+	t.Run("/api/v1/alert?badParams", func(t *testing.T) {
+		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramAlertID)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+
+		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramAlertID)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 404)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
+
+		// bad request, alertID is missing
+		params = fmt.Sprintf("?%s=1", paramGroupID)
+		getResp(ts.URL+"/api/v1/alert"+params, nil, 400)
+		getResp(ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
+	})
+
 	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
 			t.Errorf("expected 1 group got %d", length)
 		}
+
+		lr = listGroupsResponse{}
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		if length := len(lr.Data.Groups); length != 1 {
+			t.Errorf("expected 1 group got %d", length)
+		}
 	})
+
+	// check deprecated links support
+	// TODO: remove as soon as deprecated links removed
 	t.Run("/api/v1/0/0/status", func(t *testing.T) {
 		alert := &APIAlert{}
 		getResp(ts.URL+"/api/v1/0/0/status", alert, 200)
@@ -75,7 +144,5 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/1/0/status", func(t *testing.T) {
 		getResp(ts.URL+"/api/v1/1/0/status", nil, 404)
 	})
-	t.Run("/", func(t *testing.T) {
-		getResp(ts.URL, nil, 200)
-	})
+
 }
--- a/app/vmalert/web_types.go
+++ b/app/vmalert/web_types.go
@@ -1,14 +1,15 @@
 package main

 import (
+	"fmt"
 	"time"
 )

-// APIAlert represents a notifier.AlertingRule state
+// APIAlert represents a notifier.AlertingRule ruleState
 // for WEB view
 // https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIAlert struct {
-	State       string            `json:"state"`
+	State       string            `json:"ruleState"`
 	Name        string            `json:"name"`
 	Value       string            `json:"value"`
 	Labels      map[string]string `json:"labels,omitempty"`
@@ -29,10 +30,22 @@ type APIAlert struct {
 	// SourceLink contains a link to a system which should show
 	// why Alert was generated
 	SourceLink string `json:"source"`
-	// Restored shows whether Alert's state was restored on restart
+	// Restored shows whether Alert's ruleState was restored on restart
 	Restored bool `json:"restored"`
 }

+// WebLink returns a link to the alert which can be used in UI.
+func (aa *APIAlert) WebLink() string {
+	return fmt.Sprintf("alert?%s=%s&%s=%s",
+		paramGroupID, aa.GroupID, paramAlertID, aa.ID)
+}
+
+// APILink returns a link to the alert's JSON representation.
+func (aa *APIAlert) APILink() string {
+	return fmt.Sprintf("api/v1/alert?%s=%s&%s=%s",
+		paramGroupID, aa.GroupID, paramAlertID, aa.ID)
+}
+
 // APIGroup represents Group for WEB view
 // https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIGroup struct {
@@ -57,6 +70,8 @@ type APIGroup struct {
 	Concurrency int `json:"concurrency"`
 	// Params contains HTTP URL parameters added to each Rule's request
 	Params []string `json:"params,omitempty"`
+	// Headers contains HTTP headers added to each Rule's request
+	Headers []string `json:"headers,omitempty"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	Labels map[string]string `json:"labels,omitempty"`
 }
@@ -71,10 +86,10 @@ type GroupAlerts struct {
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIRule struct {
 	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
-	State string `json:"state"`
+	//  "pending": at least 1 alert in the rule in pending ruleState and no other alert in firing ruleState.
+	//  "firing": at least 1 alert in the rule in firing ruleState.
+	//  "inactive": no alert in the rule in firing or pending ruleState.
+	State string `json:"ruleState"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
 	Query string `json:"query"`
@@ -101,8 +116,17 @@ type APIRule struct {
 	// Type of the rule: recording or alerting
 	DatasourceType string `json:"datasourceType"`
 	LastSamples    int    `json:"lastSamples"`
-	// ID is an unique Alert's ID within a group
+	// ID is a unique Alert's ID within a group
 	ID string `json:"id"`
 	// GroupID is an unique Group's ID
 	GroupID string `json:"group_id"`
+
+	// TODO:
+	Updates []ruleStateEntry `json:"updates"`
+}
+
+// WebLink returns a link to the alert which can be used in UI.
+func (ar APIRule) WebLink() string {
+	return fmt.Sprintf("rule?%s=%s&%s=%s",
+		paramGroupID, ar.GroupID, paramRuleID, ar.ID)
 }
--- a/app/vmauth/Makefile
+++ b/app/vmauth/Makefile
@@ -12,20 +12,20 @@ vmauth-prod:
 vmauth-pure-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-pure

-vmauth-amd64-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-amd64
+vmauth-linux-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-amd64

-vmauth-arm-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-arm
+vmauth-linux-arm-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-arm

-vmauth-arm64-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-arm64
+vmauth-linux-arm64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-arm64

-vmauth-ppc64le-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-ppc64le
+vmauth-linux-ppc64le-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-ppc64le

-vmauth-386-prod:
-	APP_NAME=vmauth $(MAKE) app-via-docker-386
+vmauth-linux-386-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-386

 vmauth-darwin-amd64-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-amd64
@@ -33,6 +33,12 @@ vmauth-darwin-amd64-prod:
 vmauth-darwin-arm64-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-arm64

+vmauth-freebsd-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-freebsd-amd64
+
+vmauth-openbsd-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-openbsd-amd64
+
 vmauth-windows-amd64-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-windows-amd64

@@ -66,26 +72,35 @@ run-vmauth:
 	ARGS='-auth.config=app/vmauth/example_config.yml' \
 	$(MAKE) run-via-docker

-vmauth-amd64:
-	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmauth-local-with-goarch
+vmauth-linux-amd64:
+	APP_NAME=vmauth CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

-vmauth-arm:
-	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmauth-local-with-goarch
+vmauth-linux-arm:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch

-vmauth-arm64:
-	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmauth-local-with-goarch
+vmauth-linux-arm64:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch

-vmauth-ppc64le:
-	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmauth-local-with-goarch
+vmauth-linux-ppc64le:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch

-vmauth-386:
-	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmauth-local-with-goarch
+vmauth-linux-386:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch

-vmauth-local-with-goarch:
-	APP_NAME=vmauth $(MAKE) app-local-with-goarch
+vmauth-darwin-amd64:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmauth-darwin-arm64:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmauth-freebsd-amd64:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmauth-openbsd-amd64:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmauth-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmauth $(MAKE) app-local-windows-goarch

 vmauth-pure:
 	APP_NAME=vmauth $(MAKE) app-local-pure
-
-vmauth-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmauth $(MAKE) app-local-windows-with-goarch
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -10,7 +10,7 @@ The `-auth.config` can point to either local file or to http url.
 Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and pass the following flag to `vmauth` binary in order to start authorizing and routing requests:

-```bash
+```console
 /path/to/vmauth -auth.config=/path/to/auth/config.yml
 ```

@@ -36,7 +36,8 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl

 ```yml
 # Arbitrary number of usernames may be put here.
-# Username and bearer_token values must be unique.
+# It is possible to set multiple identical usernames with different passwords.
+# Such usernames can be differentiated by `name` option.

 users:
  # Requests with the 'Authorization: Bearer XXXX' and 'Authorization: Token XXXX'
@@ -129,7 +130,7 @@ It is expected that all the backend services protected by `vmauth` are located i

 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:

-```bash
+```console
  -tls
     Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
  -tlsCertFile string
@@ -140,7 +141,11 @@ Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable

 Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.

-It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
+It is recommended protecting  following endpoints with authKeys:
+* `/-/reload` with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
+* `/flags` with `-flagsAuthkey` command-line flag, so unauthorized users couldn't get application command-line flags.
+* `/metrics` with `metricsAuthkey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
+* `/debug/pprof` with `pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).

 ## Monitoring

@@ -162,7 +167,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic

 ### Development build

-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.1.
 2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
   It builds `vmauth` binary and puts it into the `bin` folder.

@@ -181,7 +186,7 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:

-```bash
+```console
 ROOT_IMAGE=scratch make package-vmauth
 ```

@@ -193,7 +198,7 @@ ROOT_IMAGE=scratch make package-vmauth

 <div class="with-copy" markdown="1">

-```bash
+```console
 curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof
 ```

@@ -203,7 +208,7 @@ curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof

 <div class="with-copy" markdown="1">

-```bash
+```console
 curl http://0.0.0.0:8427/debug/pprof/profile > cpu.pprof
 ```

@@ -212,12 +217,13 @@ curl http://0.0.0.0:8427/debug/pprof/profile > cpu.pprof
 The command for collecting CPU profile waits for 30 seconds before returning.

 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+It is safe sharing the collected profiles from security point of view, since they do not contain sensitive information.

 ## Advanced usage

 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:

-```bash
+```console
 ./vmauth -help

 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
@@ -233,7 +239,9 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
  -envflag.prefix string
     Prefix for environment variables if -envflag.enable is set
  -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
+     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in enterprise version of VictoriaMetrics
+  -flagsAuthKey string
+     Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -fs.disableMmap
     Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
  -http.connTimeout duration
@@ -278,9 +286,17 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
  -memory.allowedPercent float
     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
  -metricsAuthKey string
-     Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
  -pprofAuthKey string
-     Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -pushmetrics.extraLabel array
+     Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url
+     Supports an array of values separated by comma or specified via multiple flags.
+  -pushmetrics.interval duration
+     Interval for pushing metrics to -pushmetrics.url (default 10s)
+  -pushmetrics.url array
+     Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default metrics exposed at /metrics page aren't pushed to any remote storage
+     Supports an array of values separated by comma or specified via multiple flags.
  -reloadAuthKey string
     Auth key for /-/reload http endpoint. It must be passed as authKey=...
  -tls
@@ -292,6 +308,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
     Supports an array of values separated by comma or specified via multiple flags.
  -tlsKeyFile string
     Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
+  -tlsMinVersion string
+     Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
  -version
     Show VictoriaMetrics version
 ```
--- a/Show More
+++ b/Show More