Merge branch 'master' into streaming-aggregation-ui

* docs: vmanomaly update vmanomaly + vmalert guide

Signed-off-by: Artem Navoiev <tenmozes@gmail.com>

* docs: vmanomaly update vmanomaly + vmalert guide. Update docker compose and monitoring section

Signed-off-by: Artem Navoiev <tenmozes@gmail.com>

* typos and fixes

Signed-off-by: Artem Navoiev <tenmozes@gmail.com>

---------

Signed-off-by: Artem Navoiev <tenmozes@gmail.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,7 +6,7 @@ body:
     attributes:
       value: |
         Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
         It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
   - type: textarea
@@ -60,7 +60,7 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/)
           * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
         
         See how to setup monitoring here:

.github/workflows/check-licenses.yml

@@ -14,13 +14,25 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Go
-        uses: actions/setup-go@main
-        with:
-          go-version: 1.21.1
-        id: go
       - name: Code checkout
         uses: actions/checkout@master
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
+
       - name: Check License
-        run: |
-          make check-licenses
+        run: make check-licenses

.github/workflows/codeql-analysis.yml

@@ -55,11 +55,22 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        id: go
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.1
-          check-latest: true
-          cache: true
+          go-version: stable
+          cache: false
+        if: ${{ matrix.language == 'go' }}
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
         if: ${{ matrix.language == 'go' }}
 
       # Initializes the CodeQL tools for scanning.

.github/workflows/main.yml

@@ -7,6 +7,8 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
   pull_request:
     branches:
       - master
@@ -14,6 +16,8 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
 permissions:
   contents: read
 
@@ -30,18 +34,55 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        id: go
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.1
-          check-latest: true
-          cache: true
+          go-version: stable
+          cache: false
 
-      - name: Dependencies
+      - name: Cache Go artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-check-all-
+
+      - name: Run check-all
         run: |
-          make install-golangci-lint
           make check-all
           git diff --exit-code
 
+  build:
+    needs: lint
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+
+      - name: Build
+        run: make crossbuild
+
   test:
     needs: lint
     strategy:
@@ -54,43 +95,26 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        id: go
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.1
-          check-latest: true
-          cache: true
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
       - name: run tests
-        run: |
-          make ${{ matrix.scenario}}
+        run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
         uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
-
-  build:
-    needs: test
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.1
-          check-latest: true
-          cache: true
-
-      - uses: actions/cache@v3
-        with:
-          path: gocache-for-docker
-          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
-
-      - name: Build
-        run: |
-          make victoria-metrics-crossbuild
-          make vmuitils-crossbuild

.github/workflows/sync-docs.yml

@@ -6,6 +6,9 @@ on:
     paths:
       - 'docs/**'
   workflow_dispatch: {}
+env:
+  PAGEFIND_VERSION: "1.0.4"
+  HUGO_VERSION: "latest"
 permissions:
   contents: read  # This is required for actions/checkout and to commit back image update
   deployments: write
@@ -24,7 +27,16 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
-
+      - uses: peaceiris/actions-hugo@v2
+        with:
+          hugo-version: ${{env.HUGO_VERSION}}
+          extended: true
+      - name: Install PageFind #install the static search engine for index build
+        uses: supplypike/setup-bin@v3
+        with:
+          uri: "https://github.com/CloudCannon/pagefind/releases/download/v${{env.PAGEFIND_VERSION}}/pagefind-v${{env.PAGEFIND_VERSION}}-x86_64-unknown-linux-musl.tar.gz"
+          name: "pagefind"
+          version: ${{env.PAGEFIND_VERSION}}
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v5
         with:
@@ -45,6 +57,7 @@ jobs:
           rm -rf content
           cp -r ../main/docs content
           make clean-after-copy
+          make build-search-index
           git config --global user.name "${{ steps.import-gpg.outputs.email }}"
           git config --global user.email "${{ steps.import-gpg.outputs.email }}"
           git add .

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2023 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,5 +1,7 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
+MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
+MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
@@ -15,6 +17,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
+include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include snap/local/Makefile
@@ -25,6 +28,7 @@ all: \
 	victoria-logs-prod \
 	vmagent-prod \
 	vmalert-prod \
+	vmalert-tool-prod \
 	vmauth-prod \
 	vmbackup-prod \
 	vmrestore-prod \
@@ -33,10 +37,11 @@ all: \
 clean:
 	rm -rf bin/*
 
-publish: package-base \
+publish: \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
+	publish-vmalert-tool \
 	publish-vmauth \
 	publish-vmbackup \
 	publish-vmrestore \
@@ -47,6 +52,7 @@ package: \
 	package-victoria-logs \
 	package-vmagent \
 	package-vmalert \
+	package-vmalert-tool \
 	package-vmauth \
 	package-vmbackup \
 	package-vmrestore \
@@ -55,6 +61,7 @@ package: \
 vmutils: \
 	vmagent \
 	vmalert \
+	vmalert-tool \
 	vmauth \
 	vmbackup \
 	vmrestore \
@@ -63,6 +70,7 @@ vmutils: \
 vmutils-pure: \
 	vmagent-pure \
 	vmalert-pure \
+	vmalert-tool-pure \
 	vmauth-pure \
 	vmbackup-pure \
 	vmrestore-pure \
@@ -71,6 +79,7 @@ vmutils-pure: \
 vmutils-linux-amd64: \
 	vmagent-linux-amd64 \
 	vmalert-linux-amd64 \
+	vmalert-tool-linux-amd64 \
 	vmauth-linux-amd64 \
 	vmbackup-linux-amd64 \
 	vmrestore-linux-amd64 \
@@ -79,6 +88,7 @@ vmutils-linux-amd64: \
 vmutils-linux-arm64: \
 	vmagent-linux-arm64 \
 	vmalert-linux-arm64 \
+	vmalert-tool-linux-arm64 \
 	vmauth-linux-arm64 \
 	vmbackup-linux-arm64 \
 	vmrestore-linux-arm64 \
@@ -87,6 +97,7 @@ vmutils-linux-arm64: \
 vmutils-linux-arm: \
 	vmagent-linux-arm \
 	vmalert-linux-arm \
+	vmalert-tool-linux-arm \
 	vmauth-linux-arm \
 	vmbackup-linux-arm \
 	vmrestore-linux-arm \
@@ -95,6 +106,7 @@ vmutils-linux-arm: \
 vmutils-linux-386: \
 	vmagent-linux-386 \
 	vmalert-linux-386 \
+	vmalert-tool-linux-386 \
 	vmauth-linux-386 \
 	vmbackup-linux-386 \
 	vmrestore-linux-386 \
@@ -103,6 +115,7 @@ vmutils-linux-386: \
 vmutils-linux-ppc64le: \
 	vmagent-linux-ppc64le \
 	vmalert-linux-ppc64le \
+	vmalert-tool-linux-ppc64le \
 	vmauth-linux-ppc64le \
 	vmbackup-linux-ppc64le \
 	vmrestore-linux-ppc64le \
@@ -111,6 +124,7 @@ vmutils-linux-ppc64le: \
 vmutils-darwin-amd64: \
 	vmagent-darwin-amd64 \
 	vmalert-darwin-amd64 \
+	vmalert-tool-darwin-amd64 \
 	vmauth-darwin-amd64 \
 	vmbackup-darwin-amd64 \
 	vmrestore-darwin-amd64 \
@@ -119,6 +133,7 @@ vmutils-darwin-amd64: \
 vmutils-darwin-arm64: \
 	vmagent-darwin-arm64 \
 	vmalert-darwin-arm64 \
+	vmalert-tool-darwin-arm64 \
 	vmauth-darwin-arm64 \
 	vmbackup-darwin-arm64 \
 	vmrestore-darwin-arm64 \
@@ -127,6 +142,7 @@ vmutils-darwin-arm64: \
 vmutils-freebsd-amd64: \
 	vmagent-freebsd-amd64 \
 	vmalert-freebsd-amd64 \
+	vmalert-tool-freebsd-amd64 \
 	vmauth-freebsd-amd64 \
 	vmbackup-freebsd-amd64 \
 	vmrestore-freebsd-amd64 \
@@ -135,6 +151,7 @@ vmutils-freebsd-amd64: \
 vmutils-openbsd-amd64: \
 	vmagent-openbsd-amd64 \
 	vmalert-openbsd-amd64 \
+	vmalert-tool-openbsd-amd64 \
 	vmauth-openbsd-amd64 \
 	vmbackup-openbsd-amd64 \
 	vmrestore-openbsd-amd64 \
@@ -143,17 +160,20 @@ vmutils-openbsd-amd64: \
 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
 	vmalert-windows-amd64 \
+	vmalert-tool-windows-amd64 \
 	vmauth-windows-amd64 \
 	vmbackup-windows-amd64 \
 	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
+crossbuild:
+	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
+
 victoria-metrics-crossbuild: \
 	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
 	victoria-metrics-linux-arm64 \
 	victoria-metrics-linux-arm \
-	victoria-metrics-linux-386 \
 	victoria-metrics-linux-ppc64le \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
@@ -165,7 +185,6 @@ vmutils-crossbuild: \
 	vmutils-linux-amd64 \
 	vmutils-linux-arm64 \
 	vmutils-linux-arm \
-	vmutils-linux-386 \
 	vmutils-linux-ppc64le \
 	vmutils-darwin-amd64 \
 	vmutils-darwin-arm64 \
@@ -175,14 +194,15 @@ vmutils-crossbuild: \
 
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
-		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
-		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
-		git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
+	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
 
-release: \
-	release-victoria-metrics \
-	release-vmutils
+release:
+	$(MAKE_PARALLEL) \
+		release-victoria-metrics \
+		release-vmutils
 
 release-victoria-metrics: \
 	release-victoria-metrics-linux-386 \
@@ -241,16 +261,16 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
-release-victoria-logs: \
-	release-victoria-logs-linux-386 \
-	release-victoria-logs-linux-amd64 \
-	release-victoria-logs-linux-arm \
-	release-victoria-logs-linux-arm64 \
-	release-victoria-logs-darwin-amd64 \
-	release-victoria-logs-darwin-arm64 \
-	release-victoria-logs-freebsd-amd64 \
-	release-victoria-logs-openbsd-amd64 \
-	release-victoria-logs-windows-amd64
+release-victoria-logs:
+	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
+		release-victoria-logs-linux-amd64 \
+		release-victoria-logs-linux-arm \
+		release-victoria-logs-linux-arm64 \
+		release-victoria-logs-darwin-amd64 \
+		release-victoria-logs-darwin-arm64 \
+		release-victoria-logs-freebsd-amd64 \
+		release-victoria-logs-openbsd-amd64 \
+		release-victoria-logs-windows-amd64
 
 release-victoria-logs-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
@@ -339,6 +359,7 @@ release-vmutils-windows-amd64:
 release-vmutils-goos-goarch: \
 	vmagent-$(GOOS)-$(GOARCH)-prod \
 	vmalert-$(GOOS)-$(GOARCH)-prod \
+	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 	vmauth-$(GOOS)-$(GOARCH)-prod \
 	vmbackup-$(GOOS)-$(GOARCH)-prod \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -347,6 +368,7 @@ release-vmutils-goos-goarch: \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -354,6 +376,7 @@ release-vmutils-goos-goarch: \
 		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
+			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -362,6 +385,7 @@ release-vmutils-goos-goarch: \
 	cd bin && rm -rf \
 		vmagent-$(GOOS)-$(GOARCH)-prod \
 		vmalert-$(GOOS)-$(GOARCH)-prod \
+		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 		vmauth-$(GOOS)-$(GOARCH)-prod \
 		vmbackup-$(GOOS)-$(GOARCH)-prod \
 		vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -370,6 +394,7 @@ release-vmutils-goos-goarch: \
 release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
+	vmalert-tool-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
 	vmbackup-windows-$(GOARCH)-prod \
 	vmrestore-windows-$(GOARCH)-prod \
@@ -378,6 +403,7 @@ release-vmutils-windows-goarch: \
 		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
+			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmbackup-windows-$(GOARCH)-prod.exe \
 			vmrestore-windows-$(GOARCH)-prod.exe \
@@ -385,6 +411,7 @@ release-vmutils-windows-goarch: \
 		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
+			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmbackup-windows-$(GOARCH)-prod.exe \
 			vmrestore-windows-$(GOARCH)-prod.exe \
@@ -393,6 +420,7 @@ release-vmutils-windows-goarch: \
 	cd bin && rm -rf \
 		vmagent-windows-$(GOARCH)-prod.exe \
 		vmalert-windows-$(GOARCH)-prod.exe \
+		vmalert-tool-windows-$(GOARCH)-prod.exe \
 		vmauth-windows-$(GOARCH)-prod.exe \
 		vmbackup-windows-$(GOARCH)-prod.exe \
 		vmrestore-windows-$(GOARCH)-prod.exe \
@@ -463,7 +491,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.54.2
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.55.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -506,11 +534,3 @@ copy-docs:
 docs-sync:
 	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
 	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs
-	SRC=app/vmagent/README.md DST=docs/vmagent.md OLD_URL='/vmagent.html' ORDER=3 TITLE=vmagent $(MAKE) copy-docs
-	SRC=app/vmalert/README.md DST=docs/vmalert.md OLD_URL='/vmalert.html' ORDER=4 TITLE=vmalert $(MAKE) copy-docs
-	SRC=app/vmauth/README.md DST=docs/vmauth.md OLD_URL='/vmauth.html' ORDER=5 TITLE=vmauth $(MAKE) copy-docs
-	SRC=app/vmbackup/README.md DST=docs/vmbackup.md OLD_URL='/vmbackup.html' ORDER=6 TITLE=vmbackup $(MAKE) copy-docs
-	SRC=app/vmrestore/README.md DST=docs/vmrestore.md OLD_URL='/vmrestore.html' ORDER=7 TITLE=vmrestore $(MAKE) copy-docs
-	SRC=app/vmctl/README.md DST=docs/vmctl.md OLD_URL='/vmctl.html' ORDER=8 TITLE=vmctl $(MAKE) copy-docs
-	SRC=app/vmgateway/README.md DST=docs/vmgateway.md OLD_URL='/vmgateway.html' ORDER=9 TITLE=vmgateway $(MAKE) copy-docs
-	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md OLD_URL='/vmbackupmanager.html' ORDER=10 TITLE=vmbackupmanager $(MAKE) copy-docs

app/victoria-logs/main.go

@@ -37,7 +37,6 @@ func main() {
 	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
 	startTime := time.Now()
@@ -49,8 +48,10 @@ func main() {
 	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
 	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()

app/victoria-metrics/main.go

@@ -26,7 +26,7 @@ import (
 )
 
 var (
-	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -48,7 +48,6 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -74,13 +73,16 @@ func main() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
+
 	startSelfScraper()
 
 	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
 	stopSelfScraper()
 
@@ -89,8 +91,8 @@ func main() {
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
-	vminsert.Stop()
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	vminsert.Stop()
 
 	vmstorage.Stop()
 	vmselect.Stop()
@@ -117,6 +119,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"expand-with-exprs", "WITH expressions' tutorial"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"api/v1/status/tsdb", "tsdb status page"},

app/victoria-metrics/main_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -54,15 +55,14 @@ var (
 )
 
 type test struct {
-	Name             string     `json:"name"`
-	Data             []string   `json:"data"`
-	InsertQuery      string     `json:"insert_query"`
-	Query            []string   `json:"query"`
-	ResultMetrics    []Metric   `json:"result_metrics"`
-	ResultSeries     Series     `json:"result_series"`
-	ResultQuery      Query      `json:"result_query"`
-	ResultQueryRange QueryRange `json:"result_query_range"`
-	Issue            string     `json:"issue"`
+	Name          string   `json:"name"`
+	Data          []string `json:"data"`
+	InsertQuery   string   `json:"insert_query"`
+	Query         []string `json:"query"`
+	ResultMetrics []Metric `json:"result_metrics"`
+	ResultSeries  Series   `json:"result_series"`
+	ResultQuery   Query    `json:"result_query"`
+	Issue         string   `json:"issue"`
 }
 
 type Metric struct {
@@ -80,42 +80,90 @@ type Series struct {
 	Status string              `json:"status"`
 	Data   []map[string]string `json:"data"`
 }
+
 type Query struct {
-	Status string    `json:"status"`
-	Data   QueryData `json:"data"`
-}
-type QueryData struct {
-	ResultType string            `json:"resultType"`
-	Result     []QueryDataResult `json:"result"`
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string          `json:"resultType"`
+		Result     json.RawMessage `json:"result"`
+	} `json:"data"`
 }
 
-type QueryDataResult struct {
-	Metric map[string]string `json:"metric"`
-	Value  []interface{}     `json:"value"`
+const rtVector, rtMatrix = "vector", "matrix"
+
+func (q *Query) metrics() ([]Metric, error) {
+	switch q.Data.ResultType {
+	case rtVector:
+		var r QueryInstant
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	case rtMatrix:
+		var r QueryRange
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	default:
+		return nil, fmt.Errorf("unknown result type %q", q.Data.ResultType)
+	}
 }
 
-func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
-	type plain QueryDataResult
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+type QueryInstant struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TV     [2]interface{}    `json:"value"`
+	} `json:"result"`
+}
+
+func (q QueryInstant) metrics() ([]Metric, error) {
+	result := make([]Metric, len(q.Result))
+	for i, res := range q.Result {
+		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
+		}
+		var m Metric
+		m.Metric = res.Labels
+		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
+		m.Values = append(m.Values, f)
+		result[i] = m
+	}
+	return result, nil
 }
 
 type QueryRange struct {
-	Status string         `json:"status"`
-	Data   QueryRangeData `json:"data"`
-}
-type QueryRangeData struct {
-	ResultType string                 `json:"resultType"`
-	Result     []QueryRangeDataResult `json:"result"`
+	Result []struct {
+		Metric map[string]string `json:"metric"`
+		Values [][]interface{}   `json:"values"`
+	} `json:"result"`
 }
 
-type QueryRangeDataResult struct {
-	Metric map[string]string `json:"metric"`
-	Values [][]interface{}   `json:"values"`
+func (q QueryRange) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range q.Result {
+		var m Metric
+		for _, tv := range res.Values {
+			f, err := strconv.ParseFloat(tv[1].(string), 64)
+			if err != nil {
+				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
+			}
+			m.Values = append(m.Values, f)
+			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
+		}
+		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
+			return nil, fmt.Errorf("metric %v contains no values", res)
+		}
+		m.Metric = q.Result[i].Metric
+		result = append(result, m)
+	}
+	return result, nil
 }
 
-func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
-	type plain QueryRangeDataResult
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+func (q *Query) UnmarshalJSON(b []byte) error {
+	type plain Query
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(q))
 }
 
 func TestMain(m *testing.M) {
@@ -197,6 +245,9 @@ func TestWriteRead(t *testing.T) {
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
 		for _, test := range readIn("prometheus", t, insertionTime) {
+			if test.Data == nil {
+				continue
+			}
 			s := newSuite(t)
 			r := testutil.WriteRequest{}
 			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
@@ -272,17 +323,19 @@ func testRead(t *testing.T) {
 							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
 								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
 							}
-						case strings.HasPrefix(q, "/api/v1/query_range"):
-							queryResult := QueryRange{}
-							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
-								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
-							}
 						case strings.HasPrefix(q, "/api/v1/query"):
 							queryResult := Query{}
 							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
-								t.Fatalf("Query. %s fails with error: %s.%s", q, err, test.Issue)
+							gotMetrics, err := queryResult.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse query response: %s", err)
+							}
+							expMetrics, err := test.ResultQuery.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse expected response: %s", err)
+							}
+							if err := checkMetricsResult(gotMetrics, expMetrics); err != nil {
+								t.Fatalf("%q fails with error %s.%s", q, err, test.Issue)
 							}
 						default:
 							t.Fatalf("unsupported read query %s", q)
@@ -417,60 +470,6 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
-func checkQueryResult(got, want Query) error {
-	if got.Status != want.Status {
-		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
-	}
-	if got.Data.ResultType != want.Data.ResultType {
-		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
-	}
-	wantData := append([]QueryDataResult(nil), want.Data.Result...)
-	for _, r := range got.Data.Result {
-		wantData = removeIfFoundQueryData(r, wantData)
-	}
-	if len(wantData) > 0 {
-		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
-	}
-	return nil
-}
-
-func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
-	for i, item := range contains {
-		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
-			contains[i] = contains[len(contains)-1]
-			return contains[:len(contains)-1]
-		}
-	}
-	return contains
-}
-
-func checkQueryRangeResult(got, want QueryRange) error {
-	if got.Status != want.Status {
-		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
-	}
-	if got.Data.ResultType != want.Data.ResultType {
-		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
-	}
-	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
-	for _, r := range got.Data.Result {
-		wantData = removeIfFoundQueryRangeData(r, wantData)
-	}
-	if len(wantData) > 0 {
-		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
-	}
-	return nil
-}
-
-func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
-	for i, item := range contains {
-		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
-			contains[i] = contains[len(contains)-1]
-			return contains[:len(contains)-1]
-		}
-	}
-	return contains
-}
-
 type suite struct{ t *testing.T }
 
 func newSuite(t *testing.T) *suite { return &suite{t: t} }

app/victoria-metrics/self_scraper.go

@@ -98,7 +98,7 @@ func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
-	lb.Name = bytesutil.ToUnsafeBytes(key)
-	lb.Value = bytesutil.ToUnsafeBytes(value)
+	lb.Name = key
+	lb.Value = value
 	return dst
 }

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -7,7 +7,7 @@
     "not_nan_not_inf;item=y 3 {TIME_S-1m}",
     "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
   "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -6,7 +6,7 @@
     "empty_label_match;foo=bar 2 {TIME_S-1m}",
     "empty_label_match;foo=baz 3 {TIME_S-1m}"],
   "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -8,7 +8,7 @@
     "max_lookback_set 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_set"},"values":[

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -8,7 +8,7 @@
     "max_lookback_unset 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -8,7 +8,7 @@
     "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
   ],
   "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
-  "result_query_range": {
+  "result_query": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/prometheus/instant-matrix.json

@@ -0,0 +1,12 @@
+{
+  "name": "instant query with look-behind window",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo[5m]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/instant-scalar.json

@@ -0,0 +1,11 @@
+{
+  "name": "instant scalar query",
+  "query": ["/api/v1/query?query=42&time={TIME_S}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"vector",
+        "result":[{"metric":{},"value":["{TIME_S}", "42"]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/issue-5553-too-big-lookback.json

@@ -0,0 +1,13 @@
+{
+  "name": "too big look-behind window",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"},{\"name\":\"issue\",\"value\":\"5553\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo{issue=\"5553\"}[100y]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo", "issue": "5553"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/query-range.json

@@ -0,0 +1,18 @@
+{
+  "name": "query range",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"bar\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}, {\"value\":2,\"timestamp\":\"{TIME_MS-120s}\"}, {\"value\":1,\"timestamp\":\"{TIME_MS-180s}\"}]}]"],
+  "query": ["/api/v1/query_range?query=bar&step=30s&start={TIME_MS-180s}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[
+          {
+            "metric":{"__name__":"bar"},
+            "values":[["{TIME_S-180s}", "1"],["{TIME_S-150s}", "1"],["{TIME_S-120s}", "2"],["{TIME_S-90s}", "2"], ["{TIME_S-60s}", "1"], ["{TIME_S-30s}", "1"], ["{TIME_S}", "1"]]
+          }
+        ]
+      }
+    }
+}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -105,7 +105,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		vlstorage.MustAddRows(lr)
 		logstorage.PutLogRows(lr)
 		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s", n, err)
+			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
 			return true
 		}
 

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -120,10 +120,10 @@ func compressData(s string) string {
 	var bb bytes.Buffer
 	zw := gzip.NewWriter(&bb)
 	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %s", err))
+		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
 	}
 	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %s", err))
+		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
 	}
 	return bb.String()
 }

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -43,7 +43,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 			r.Reset(dataBytes)
 			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %s", err))
+				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
 	})

app/vlinsert/loki/loki_json_timing_test.go

@@ -29,7 +29,7 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 		for pb.Next() {
 			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %s", err))
+				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
 	})

app/vlinsert/loki/loki_protobuf.go

@@ -84,7 +84,7 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 
 	err = req.Unmarshal(bb.B)
 	if err != nil {
-		return 0, fmt.Errorf("cannot parse request body: %s", err)
+		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
 	var commonFields []logstorage.Field
@@ -97,7 +97,7 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 		// Labels are same for all entries in the stream.
 		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
 		if err != nil {
-			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %s", stream.Labels, err)
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
 		fields := commonFields
 

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -31,7 +31,7 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 		for pb.Next() {
 			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %s", err))
+				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
 	})

app/vlselect/main.go

@@ -88,6 +88,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if strings.HasPrefix(path, "/vmui/") {
+		if strings.HasPrefix(path, "/vmui/static/") {
+			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
+			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
+			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
+			w.Header().Set("Cache-Control", "max-age=31536000")
+		}
 		r.URL.Path = path
 		vmuiFileServer.ServeHTTP(w, r)
 		return true

app/vlselect/vmui/asset-manifest.json

@@ -1,14 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.3258b6c4.css",
-    "main.js": "./static/js/main.41b816cc.js",
-    "static/js/522.b5ae4365.chunk.js": "./static/js/522.b5ae4365.chunk.js",
-    "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
-    "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
+    "main.css": "./static/css/main.d1313636.css",
+    "main.js": "./static/js/main.1919fefe.js",
+    "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.3258b6c4.css",
-    "static/js/main.41b816cc.js"
+    "static/css/main.d1313636.css",
+    "static/js/main.1919fefe.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.41b816cc.js"></script><link href="./static/css/main.3258b6c4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.1919fefe.js.LICENSE.txt

@@ -7,7 +7,7 @@
 /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
 
 /**
- * @remix-run/router v1.7.2
+ * @remix-run/router v1.10.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -18,7 +18,7 @@
  */
 
 /**
- * React Router DOM v6.14.2
+ * React Router DOM v6.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -29,7 +29,7 @@
  */
 
 /**
- * React Router v6.14.2
+ * React Router v6.17.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vmagent/csvimport/request_handler.go

@@ -65,7 +65,9 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadogv1/request_handler.go

@@ -1,4 +1,4 @@
-package datadog
+package datadogv1
 
 import (
 	"net/http"
@@ -8,33 +8,32 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
 )
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
+	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -63,7 +62,7 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := parser.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -88,7 +87,9 @@ func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmars
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/datadogv2/request_handler.go

@@ -0,0 +1,102 @@
+package datadogv2
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ct := req.Header.Get("Content-Type")
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
+		return insertRows(at, series, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		for _, rs := range ss.Resources {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  rs.Type,
+				Value: rs.Name,
+			})
+		}
+		if ss.SourceTypeName != "" {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "source_type_name",
+				Value: ss.SourceTypeName,
+			})
+		}
+		for _, tag := range ss.Tags {
+			name, value := datadogutils.SplitTag(tag)
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp * 1000,
+				Value:     pt.Value,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/graphite/request_handler.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
@@ -20,10 +21,12 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, insertRows)
+	return stream.Parse(r, false, func(rows []parser.Row) error {
+		return insertRows(nil, rows)
+	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -56,7 +59,9 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(nil, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -36,9 +36,9 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
 	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-		return insertRows(nil, db, rows, nil)
+		return insertRows(at, db, rows, nil)
 	})
 }
 
@@ -130,7 +130,9 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.Push(at, &ctx.ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -11,13 +11,13 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
@@ -41,12 +41,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -95,7 +97,6 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -124,7 +125,7 @@ func main() {
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(r, false)
+			return influx.InsertHandlerForReader(nil, r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
@@ -139,15 +140,17 @@ func main() {
 		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
 	}
 
-	promscrape.Init(remotewrite.Push)
+	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
 	if len(*httpListenAddr) > 0 {
 		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
 
 	startTime = time.Now()
 	if len(*httpListenAddr) > 0 {
@@ -227,6 +230,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"stream-agg", "streaming aggregation status"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"-/reload", "reload configuration"},
@@ -319,10 +323,44 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/newrelic":
+		newrelicCheckRequest.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/newrelic/inventory/deltas":
+		newrelicInventoryRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
+		return true
+	case "/newrelic/infra/v2/metrics/events/bulk":
+		newrelicWriteRequests.Inc()
+		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
+			newrelicWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
 	case "/datadog/api/v1/series":
-		datadogWriteRequests.Inc()
-		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogWriteErrors.Inc()
+		datadogv1WriteRequests.Inc()
+		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogv1WriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v2/series":
+		datadogv2WriteRequests.Inc()
+		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogv2WriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -408,6 +446,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/stream-agg":
+		streamaggr.WriteHumanReadableState(w, r, remotewrite.GetAggregators())
+		return true
 	case "/ready":
 		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
@@ -519,10 +560,43 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "newrelic":
+		newrelicCheckRequest.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "newrelic/inventory/deltas":
+		newrelicInventoryRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
+		return true
+	case "newrelic/infra/v2/metrics/events/bulk":
+		newrelicWriteRequests.Inc()
+		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
+			newrelicWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
 	case "datadog/api/v1/series":
-		datadogWriteRequests.Inc()
-		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
-			datadogWriteErrors.Inc()
+		datadogv1WriteRequests.Inc()
+		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
+			datadogv1WriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v2/series":
+		datadogv2WriteRequests.Inc()
+		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
+			datadogv2WriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -580,8 +654,11 @@ var (
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
-	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
+	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
@@ -591,6 +668,12 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
 
+	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
+	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
+
+	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
+	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
+
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
 

app/vmagent/native/request_handler.go

@@ -84,6 +84,8 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	return nil
 }

app/vmagent/newrelic/request_handler.go

@@ -0,0 +1,88 @@
+package newrelic
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	isGzip := ce == "gzip"
+	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	samplesCount := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		tags := r.Tags
+		srcSamples := r.Samples
+		for j := range srcSamples {
+			s := &srcSamples[j]
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: bytesutil.ToUnsafeString(s.Name),
+			})
+			for k := range tags {
+				t := &tags[k]
+				labels = append(labels, prompbmarshal.Label{
+					Name:  bytesutil.ToUnsafeString(t.Key),
+					Value: bytesutil.ToUnsafeString(t.Value),
+				})
+			}
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     s.Value,
+				Timestamp: r.Timestamp,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+			labels = append(labels, extraLabels...)
+		}
+		samplesCount += len(srcSamples)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(samplesCount)
+	}
+	rowsPerInsert.Update(float64(samplesCount))
+	return nil
+}

app/vmagent/opentelemetry/request_handler.go

@@ -59,7 +59,9 @@ func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/opentsdb/request_handler.go

@@ -56,7 +56,9 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(nil, &ctx.WriteRequest)
+	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -64,7 +64,9 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -73,7 +73,9 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/promremotewrite/request_handler.go

@@ -6,7 +6,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -48,8 +47,8 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
 			labels = append(labels, prompbmarshal.Label{
-				Name:  bytesutil.ToUnsafeString(label.Name),
-				Value: bytesutil.ToUnsafeString(label.Value),
+				Name:  label.Name,
+				Value: label.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
@@ -69,7 +68,9 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -58,8 +58,10 @@ var (
 	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
@@ -106,12 +108,15 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	tlsCfg, err := authCfg.NewTLSConfig()
+	if err != nil {
+		logger.Fatalf("cannot initialize tls config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
-	tlsCfg := authCfg.NewTLSConfig()
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
 		DialContext:         statDial,
@@ -231,10 +236,16 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
 	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
 	if clientSecretFile != "" || clientSecret != "" {
+		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
+		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
+		}
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
 			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
+			EndpointParams:   endpointParams,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
 		}
@@ -302,7 +313,7 @@ func (c *client) runWorker() {
 				continue
 			}
 			// Return unsent block to the queue.
-			c.fq.MustWriteBlock(block)
+			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait for a while in the hope the block will be sent.
@@ -311,11 +322,11 @@ func (c *client) runWorker() {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
-					c.fq.MustWriteBlock(block)
+					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 				}
 			case <-time.After(graceDuration):
 				// Return unsent block to the queue.
-				c.fq.MustWriteBlock(block)
+				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			}
 			return
 		}
@@ -323,26 +334,42 @@ func (c *client) runWorker() {
 }
 
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req := c.newRequest(url, body)
-	resp, err := c.hc.Do(req)
-	if err != nil && errors.Is(err, io.EOF) {
-		// it is likely connection became stale.
-		// So we do one more attempt in hope request will succeed.
-		// If not, the error should be handled by the caller as usual.
-		// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-		req = c.newRequest(url, body)
-		resp, err = c.hc.Do(req)
+	req, err := c.newRequest(url, body)
+	if err != nil {
+		return nil, err
 	}
-	return resp, err
+	resp, err := c.hc.Do(req)
+	if err == nil {
+		return resp, nil
+	}
+	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		return nil, err
+	}
+	// It is likely connection became stale or timed out during the first request.
+	// Make another attempt in hope request will succeed.
+	// If not, the error should be handled by the caller as usual.
+	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
+	req, err = c.newRequest(url, body)
+	if err != nil {
+		return nil, fmt.Errorf("second attempt: %w", err)
+	}
+	resp, err = c.hc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("second attempt: %w", err)
+	}
+	return resp, nil
 }
 
-func (c *client) newRequest(url string, body []byte) *http.Request {
+func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	reqBody := bytes.NewBuffer(body)
 	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
 	}
-	c.authCfg.SetHeaders(req, true)
+	err = c.authCfg.SetHeaders(req, true)
+	if err != nil {
+		return nil, err
+	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
@@ -356,11 +383,10 @@ func (c *client) newRequest(url string, body []byte) *http.Request {
 	if c.awsCfg != nil {
 		sigv4Hash := awsapi.HashHex(body)
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
-			// there is no need in retry, request will be rejected by client.Do and retried by code below
-			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
+			return nil, fmt.Errorf("cannot sign remoteWrite request with AWS sigv4: %w", err)
 		}
 	}
-	return req
+	return req, nil
 }
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.

app/vmagent/remotewrite/pendingseries.go

@@ -37,9 +37,9 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(pushBlock func(block []byte), isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
-	ps.wr.pushBlock = pushBlock
+	ps.wr.fq = fq
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
@@ -57,10 +57,11 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+func (ps *pendingSeries) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	ps.mu.Lock()
-	ps.wr.push(tss)
+	ok := ps.wr.tryPush(tss)
 	ps.mu.Unlock()
+	return ok
 }
 
 func (ps *pendingSeries) periodicFlusher() {
@@ -70,18 +71,20 @@ func (ps *pendingSeries) periodicFlusher() {
 	}
 	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
-	mustStop := false
-	for !mustStop {
+	for {
 		select {
 		case <-ps.stopCh:
-			mustStop = true
+			ps.mu.Lock()
+			ps.wr.mustFlushOnStop()
+			ps.mu.Unlock()
+			return
 		case <-ticker.C:
 			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
 		ps.mu.Lock()
-		ps.wr.flush()
+		_ = ps.wr.tryFlush()
 		ps.mu.Unlock()
 	}
 }
@@ -90,16 +93,16 @@ type writeRequest struct {
 	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
 	lastFlushTime uint64
 
-	// pushBlock is called when whe write request is ready to be sent.
-	pushBlock func(block []byte)
+	// The queue to send blocks to.
+	fq *persistentqueue.FastQueue
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
 	isVMRemoteWrite bool
 
-	// How many significant figures must be left before sending the writeRequest to pushBlock.
+	// How many significant figures must be left before sending the writeRequest to fq.
 	significantFigures int
 
-	// How many decimal digits after point must be left before sending the writeRequest to pushBlock.
+	// How many decimal digits after point must be left before sending the writeRequest to fq.
 	roundDigits int
 
 	wr prompbmarshal.WriteRequest
@@ -112,7 +115,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, pushBlock, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -130,23 +133,40 @@ func (wr *writeRequest) reset() {
 	wr.buf = wr.buf[:0]
 }
 
-func (wr *writeRequest) flush() {
+// mustFlushOnStop force pushes wr data into wr.fq
+//
+// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
+func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	wr.adjustSampleValues()
-	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
-	pushWriteRequest(&wr.wr, wr.pushBlock, wr.isVMRemoteWrite)
+	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
+		logger.Panicf("BUG: final flush must always return true")
+	}
 	wr.reset()
 }
 
-func (wr *writeRequest) adjustSampleValues() {
-	samples := wr.samples
-	if n := wr.significantFigures; n > 0 {
+func (wr *writeRequest) mustWriteBlock(block []byte) bool {
+	wr.fq.MustWriteBlockIgnoreDisabledPQ(block)
+	return true
+}
+
+func (wr *writeRequest) tryFlush() bool {
+	wr.wr.Timeseries = wr.tss
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
+		return false
+	}
+	wr.reset()
+	return true
+}
+
+func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roundDigits int) {
+	if n := significantFigures; n > 0 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
 		}
 	}
-	if n := wr.roundDigits; n < 100 {
+	if n := roundDigits; n < 100 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
@@ -154,21 +174,27 @@ func (wr *writeRequest) adjustSampleValues() {
 	}
 }
 
-func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
 		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
-			wr.flush()
+			if !wr.tryFlush() {
+				return false
+			}
 			tssDst = wr.tss
 		}
+		tsSrc := &src[i]
+		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
+
 	wr.tss = tssDst
+	return true
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
@@ -196,13 +222,13 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte), isVMRemoteWrite bool) {
+func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
-		return
+		return true
 	}
 	bb := writeRequestBufPool.Get()
-	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
 		zb := snappyBufPool.Get()
 		if isVMRemoteWrite {
@@ -212,11 +238,13 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		}
 		writeRequestBufPool.Put(bb)
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			pushBlock(zb.B)
+			if !tryPushBlock(zb.B) {
+				return false
+			}
 			blockSizeRows.Update(float64(len(wr.Timeseries)))
 			blockSizeBytes.Update(float64(len(zb.B)))
 			snappyBufPool.Put(zb)
-			return
+			return true
 		}
 		snappyBufPool.Put(zb)
 	} else {
@@ -229,23 +257,36 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		samples := wr.Timeseries[0].Samples
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return
+			return true
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Timeseries[0].Samples = samples
+			return false
+		}
 		wr.Timeseries[0].Samples = samples[n:]
-		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Timeseries[0].Samples = samples
+			return false
+		}
 		wr.Timeseries[0].Samples = samples
-		return
+		return true
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+		wr.Timeseries = timeseries
+		return false
+	}
 	wr.Timeseries = timeseries[n:]
-	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+		wr.Timeseries = timeseries
+		return false
+	}
 	wr.Timeseries = timeseries
+	return true
 }
 
 var (

app/vmagent/remotewrite/pendingseries_test.go

@@ -26,13 +26,16 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 		t.Helper()
 		wr := newTestWriteRequest(rowsCount, 20)
 		pushBlockLen := 0
-		pushBlock := func(block []byte) {
+		pushBlock := func(block []byte) bool {
 			if pushBlockLen > 0 {
 				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
 			}
 			pushBlockLen = len(block)
+			return true
+		}
+		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
+			t.Fatalf("cannot push data to to remote storage")
 		}
-		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
 			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
 				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
@@ -40,7 +43,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 	}
 
 	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 0)
+	f(false, expectedBlockLenProm, 3)
 
 	// Check VictoriaMetrics remote write
 	f(true, expectedBlockLenVM, 15)

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/golang/snappy"
 	"github.com/klauspost/compress/s2"
 )
@@ -22,7 +21,7 @@ func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []by
 	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
 		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
 			wr := newTestWriteRequest(rowsCount, 10)
-			data := prompbmarshal.MarshalWriteRequest(nil, wr)
+			data := wr.MarshalProtobuf(nil)
 			b.ReportAllocs()
 			b.SetBytes(int64(rowsCount))
 			b.RunParallel(func(pb *testing.PB) {

app/vmagent/remotewrite/relabel.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -92,6 +93,7 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 		// Nothing to change.
 		return tss
 	}
+	rctx.reset()
 	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
@@ -120,6 +122,7 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 	if len(extraLabels) == 0 {
 		return
 	}
+	rctx.reset()
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
@@ -139,6 +142,34 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 	rctx.labels = labels
 }
 
+func (rctx *relabelCtx) tenantToLabels(tss []prompbmarshal.TimeSeries, accountID, projectID uint32) {
+	rctx.reset()
+	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
+	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		for _, label := range ts.Labels {
+			labelName := label.Name
+			if labelName == "vm_account_id" || labelName == "vm_project_id" {
+				continue
+			}
+			labels = append(labels, label)
+		}
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "vm_account_id",
+			Value: accountIDStr,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "vm_project_id",
+			Value: projectIDStr,
+		})
+		ts.Labels = labels[labelsLen:]
+	}
+	rctx.labels = labels
+}
+
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
@@ -160,7 +191,7 @@ func getRelabelCtx() *relabelCtx {
 }
 
 func putRelabelCtx(rctx *relabelCtx) {
-	rctx.labels = rctx.labels[:0]
+	rctx.reset()
 	relabelCtxPool.Put(rctx)
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"net/url"
 	"path/filepath"
 	"strconv"
@@ -10,6 +11,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -23,6 +26,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -33,15 +37,22 @@ var (
 	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
 		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
-		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set. "+
-		"See also -remoteWrite.multitenantURL")
+		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
 	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
-		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. "+
+		"This flag is deprecated in favor of -enableMultitenantHandlers . See https://docs.victoriametrics.com/vmagent.html#multitenancy")
+	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
+		"https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format . By default incoming data is processed via single-node insert handlers "+
+		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details")
 	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
 		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
-	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
-		"See also -remoteWrite.maxDiskUsagePerURL")
+	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
+		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
+		"even distribution of series over the specified -remoteWrite.url systems")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
+		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
@@ -80,6 +91,11 @@ var (
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
 		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
+	disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
+		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence ."+
+		"See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence")
 )
 
 var (
@@ -92,11 +108,19 @@ var (
 
 	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
 	defaultAuthToken = &auth.Token{}
+
+	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
+	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
+		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
+			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
+			"see https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence"),
+		StatusCode: http.StatusTooManyRequests,
+	}
 )
 
-// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
+// MultitenancyEnabled returns true if -enableMultitenantHandlers or -remoteWrite.multitenantURL is specified.
 func MultitenancyEnabled() bool {
-	return len(*remoteWriteMultitenantURLs) > 0
+	return *enableMultitenantHandlers || len(*remoteWriteMultitenantURLs) > 0
 }
 
 // Contains the current relabelConfigs.
@@ -116,6 +140,8 @@ func InitSecretFlags() {
 	}
 }
 
+var shardByURLLabelsMap map[string]struct{}
+
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -152,6 +178,13 @@ func Init() {
 	if *queues <= 0 {
 		*queues = 1
 	}
+	if len(*shardByURLLabels) > 0 {
+		m := make(map[string]struct{}, len(*shardByURLLabels))
+		for _, label := range *shardByURLLabels {
+			m[label] = struct{}{}
+		}
+		shardByURLLabelsMap = m
+	}
 	initLabelsGlobal()
 
 	// Register SIGHUP handler for config reload before loadRelabelConfigs.
@@ -170,6 +203,7 @@ func Init() {
 	if len(*remoteWriteURLs) > 0 {
 		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
+	dropDanglingQueues()
 
 	// Start config reloader.
 	configReloaderWG.Add(1)
@@ -187,6 +221,42 @@ func Init() {
 	}()
 }
 
+func dropDanglingQueues() {
+	if *keepDanglingQueues {
+		return
+	}
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		// Do not drop dangling queues for *remoteWriteMultitenantURLs, since it is impossible to determine
+		// unused queues for multitenant urls - they are created on demand when new sample for the given
+		// tenant is pushed to remote storage.
+		return
+	}
+	// Remove dangling persistent queues, if any.
+	// This is required for the case when the number of queues has been changed or URL have been changed.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
+	//
+	existingQueues := make(map[string]struct{}, len(rwctxsDefault))
+	for _, rwctx := range rwctxsDefault {
+		existingQueues[rwctx.fq.Dirname()] = struct{}{}
+	}
+
+	queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
+	files := fs.MustReadDir(queuesDir)
+	removed := 0
+	for _, f := range files {
+		dirname := f.Name()
+		if _, ok := existingQueues[dirname]; !ok {
+			logger.Infof("removing dangling queue %q", dirname)
+			fullPath := filepath.Join(queuesDir, dirname)
+			fs.MustRemoveAll(fullPath)
+			removed++
+		}
+	}
+	if removed > 0 {
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsDefault))
+	}
+}
+
 func reloadRelabelConfigs() {
 	relabelConfigReloads.Inc()
 	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
@@ -206,7 +276,7 @@ func reloadRelabelConfigs() {
 var (
 	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
 	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
+	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -260,33 +330,6 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
-
-	if !*keepDanglingQueues {
-		// Remove dangling queues, if any.
-		// This is required for the case when the number of queues has been changed or URL have been changed.
-		// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
-		existingQueues := make(map[string]struct{}, len(rwctxs))
-		for _, rwctx := range rwctxs {
-			existingQueues[rwctx.fq.Dirname()] = struct{}{}
-		}
-
-		queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
-		files := fs.MustReadDir(queuesDir)
-		removed := 0
-		for _, f := range files {
-			dirname := f.Name()
-			if _, ok := existingQueues[dirname]; !ok {
-				logger.Infof("removing dangling queue %q", dirname)
-				fullPath := filepath.Join(queuesDir, dirname)
-				fs.MustRemoveAll(fullPath)
-				removed++
-			}
-		}
-		if removed > 0 {
-			logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
-		}
-	}
-
 	return rwctxs
 }
 
@@ -295,7 +338,7 @@ var configReloaderWG sync.WaitGroup
 
 // Stop stops remotewrite.
 //
-// It is expected that nobody calls Push during and after the call to this func.
+// It is expected that nobody calls TryPush during and after the call to this func.
 func Stop() {
 	close(configReloaderStopCh)
 	configReloaderWG.Wait()
@@ -305,7 +348,7 @@ func Stop() {
 	}
 	rwctxsDefault = nil
 
-	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
+	// There is no need in locking rwctxsMapLock here, since nobody should call TryPush during the Stop call.
 	for _, rwctxs := range rwctxsMap {
 		for _, rwctx := range rwctxs {
 			rwctx.MustStop()
@@ -321,24 +364,47 @@ func Stop() {
 	}
 }
 
-// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+// PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
 //
-// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
-// If at isn't nil, the data is pushed to the configured `-remoteWrite.multitenantURL`.
+// If at is nil, then the data is pushed to the configured -remoteWrite.url.
+// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
 //
-// Note that wr may be modified by Push because of relabeling and rounding.
-func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
-	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
-		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
+// PushDropSamplesOnFailure can modify wr contents.
+func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	_ = tryPush(at, wr, true)
+}
+
+// TryPush tries sending wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+//
+// If at is nil, then the data is pushed to the configured -remoteWrite.url.
+// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
+//
+// TryPush can modify wr contents, so the caller must re-initialize wr before calling TryPush() after unsuccessful attempt.
+// TryPush may send partial data from wr on unsuccessful attempt, so repeated call for the same wr may send the data multiple times.
+//
+// The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
+func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
+	return tryPush(at, wr, *dropSamplesOnOverload)
+}
+
+func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailure bool) bool {
+	tss := wr.Timeseries
+
+	if at == nil && MultitenancyEnabled() {
+		// Write data to default tenant if at isn't set when multitenancy is enabled.
 		at = defaultAuthToken
 	}
+
+	var tenantRctx *relabelCtx
 	var rwctxs []*remoteWriteCtx
 	if at == nil {
 		rwctxs = rwctxsDefault
+	} else if len(*remoteWriteMultitenantURLs) == 0 {
+		// Convert at to (vm_account_id, vm_project_id) labels.
+		tenantRctx = getRelabelCtx()
+		defer putRelabelCtx(tenantRctx)
+		rwctxs = rwctxsDefault
 	} else {
-		if len(*remoteWriteMultitenantURLs) == 0 {
-			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
-		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
 			AccountID: at.AccountID,
@@ -352,18 +418,37 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rwctxsMapLock.Unlock()
 	}
 
+	rowsCount := getRowsCount(tss)
+
+	if *disableOnDiskQueue {
+		// Quick check whether writes to configured remote storage systems are blocked.
+		// This allows saving CPU time spent on relabeling and block compression
+		// if some of remote storage systems cannot keep up with the data ingestion rate.
+		for _, rwctx := range rwctxs {
+			if rwctx.fq.IsWriteBlocked() {
+				pushFailures.Inc()
+				if dropSamplesOnFailure {
+					// Just drop samples
+					samplesDropped.Add(rowsCount)
+					return true
+				}
+				return false
+			}
+		}
+	}
+
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
 	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
+		defer putRelabelCtx(rctx)
 	}
-	tss := wr.Timeseries
-	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
+
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -371,7 +456,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		i := 0
 		for i < len(tss) {
 			samplesCount += len(tss[i].Samples)
-			labelsCount += len(tss[i].Labels)
+			labelsCount += len(tss[i].Samples) * len(tss[i].Labels)
 			i++
 			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
@@ -384,6 +469,9 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		} else {
 			tss = nil
 		}
+		if tenantRctx != nil {
+			tenantRctx.tenantToLabels(tssBlock, at.AccountID, at.ProjectID)
+		}
 		if rctx != nil {
 			rowsCountBeforeRelabel := getRowsCount(tssBlock)
 			tssBlock = rctx.applyRelabeling(tssBlock, pcsGlobal)
@@ -392,25 +480,35 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		pushBlockToRemoteStorages(rwctxs, tssBlock)
-		if rctx != nil {
-			rctx.reset()
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock) {
+			if !*disableOnDiskQueue {
+				logger.Panicf("BUG: tryPushBlockToRemoteStorages must return true if -remoteWrite.disableOnDiskQueue isn't set")
+			}
+			pushFailures.Inc()
+			if dropSamplesOnFailure {
+				samplesDropped.Add(rowsCount)
+				return true
+			}
+			return false
 		}
 	}
-	if rctx != nil {
-		putRelabelCtx(rctx)
-	}
+	return true
 }
 
-func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+var (
+	samplesDropped = metrics.NewCounter(`vmagent_remotewrite_samples_dropped_total`)
+	pushFailures   = metrics.NewCounter(`vmagent_remotewrite_push_failures_total`)
+)
+
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
-		return
+		return true
 	}
+
 	if len(rwctxs) == 1 {
 		// Fast path - just push data to the configured single remote storage
-		rwctxs[0].Push(tssBlock)
-		return
+		return rwctxs[0].TryPush(tssBlock)
 	}
 
 	// We need to push tssBlock to multiple remote storages.
@@ -418,15 +516,28 @@ func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarsha
 	if *shardByURL {
 		// Shard the data among rwctxs
 		tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs))
+		tmpLabels := promutils.GetLabels()
 		for _, ts := range tssBlock {
-			h := getLabelsHash(ts.Labels)
+			hashLabels := ts.Labels
+			if len(shardByURLLabelsMap) > 0 {
+				hashLabels = tmpLabels.Labels[:0]
+				for _, label := range ts.Labels {
+					if _, ok := shardByURLLabelsMap[label.Name]; ok {
+						hashLabels = append(hashLabels, label)
+					}
+				}
+			}
+			h := getLabelsHash(hashLabels)
 			idx := h % uint64(len(tssByURL))
 			tssByURL[idx] = append(tssByURL[idx], ts)
 		}
+		promutils.PutLabels(tmpLabels)
+
 		// Push sharded data to remote storages in parallel in order to reduce
 		// the time needed for sending the data to multiple remote storage systems.
 		var wg sync.WaitGroup
 		wg.Add(len(rwctxs))
+		var anyPushFailed uint64
 		for i, rwctx := range rwctxs {
 			tssShard := tssByURL[i]
 			if len(tssShard) == 0 {
@@ -434,11 +545,13 @@ func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarsha
 			}
 			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
 				defer wg.Done()
-				rwctx.Push(tss)
+				if !rwctx.TryPush(tss) {
+					atomic.StoreUint64(&anyPushFailed, 1)
+				}
 			}(rwctx, tssShard)
 		}
 		wg.Wait()
-		return
+		return atomic.LoadUint64(&anyPushFailed) == 0
 	}
 
 	// Replicate data among rwctxs.
@@ -446,13 +559,17 @@ func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarsha
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
+	var anyPushFailed uint64
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			rwctx.Push(tssBlock)
+			if !rwctx.TryPush(tssBlock) {
+				atomic.StoreUint64(&anyPushFailed, 1)
+			}
 		}(rwctx)
 	}
 	wg.Wait()
+	return atomic.LoadUint64(&anyPushFailed) == 0
 }
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
@@ -572,13 +689,19 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, *disableOnDiskQueue)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
+		if fq.IsWriteBlocked() {
+			return 1
+		}
+		return 0
+	})
 
 	var c *client
 	switch remoteWriteURL.Scheme {
@@ -600,7 +723,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq.MustWriteBlock, c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{
@@ -617,7 +740,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
 	if sasFile != "" {
 		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
@@ -653,7 +776,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
@@ -691,7 +814,9 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		}
 		matchIdxsPool.Put(matchIdxs)
 	}
-	rwctx.pushInternal(tss)
+
+	// Try pushing the data to remote storage
+	ok := rwctx.tryPushInternal(tss)
 
 	// Return back relabeling contexts to the pool
 	if rctx != nil {
@@ -699,6 +824,8 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}
+
+	return ok
 }
 
 var matchIdxsPool bytesutil.ByteBufferPool
@@ -718,7 +845,21 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 	return dst
 }
 
-func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSeries) {
+	if rwctx.tryPushInternal(tss) {
+		return
+	}
+	if !*disableOnDiskQueue {
+		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
+	}
+	pushFailures.Inc()
+	if *dropSamplesOnOverload {
+		rowsCount := getRowsCount(tss)
+		samplesDropped.Add(rowsCount)
+	}
+}
+
+func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
 	if len(labelsGlobal) > 0 {
@@ -732,33 +873,36 @@ func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
-	pss[idx].Push(tss)
+
+	ok := pss[idx].TryPush(tss)
 
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}
+
+	return ok
 }
 
 func (rwctx *remoteWriteCtx) reinitStreamAggr() {
-	sas := rwctx.sas.Load()
-	if sas == nil {
+	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	if sasFile == "" {
 		// There is no stream aggregation for rwctx
 		return
 	}
 
-	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
 	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
 		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
 		return
 	}
+	sas := rwctx.sas.Load()
 	if !sasNew.Equal(sas) {
 		sasOld := rwctx.sas.Swap(sasNew)
 		sasOld.MustStop()
@@ -802,3 +946,24 @@ func CheckStreamAggrConfigs() error {
 	}
 	return nil
 }
+
+// GetAggregators returns aggregators for all the configured remote writes.
+func GetAggregators() map[string]*streamaggr.Aggregators {
+	var result = map[string]*streamaggr.Aggregators{}
+
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for tenant, rwctxs := range rwctxsMap {
+			for rwNum, rw := range rwctxs {
+				result[fmt.Sprintf("rw %d for tenant %v:%v", rwNum, tenant.AccountID, tenant.ProjectID)] = rw.sas.Load()
+			}
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		for rwNum, rw := range rwctxsDefault {
+			result[fmt.Sprintf("remote write %d", rwNum)] = rw.sas.Load()
+		}
+	}
+
+	return result
+}

app/vmagent/vmimport/request_handler.go

@@ -76,7 +76,9 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(at, &ctx.WriteRequest)
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmalert-tool/Makefile

@@ -0,0 +1,103 @@
+# All these commands must run from repository root.
+
+vmalert-tool:
+	APP_NAME=vmalert-tool $(MAKE) app-local
+
+vmalert-tool-race:
+	APP_NAME=vmalert-tool RACE=-race $(MAKE) app-local
+
+vmalert-tool-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker
+
+vmalert-tool-pure-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-pure
+
+vmalert-tool-linux-amd64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-amd64
+
+vmalert-tool-linux-arm-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm
+
+vmalert-tool-linux-arm64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm64
+
+vmalert-tool-linux-ppc64le-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-ppc64le
+
+vmalert-tool-linux-386-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386
+
+vmalert-tool-darwin-amd64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64
+
+vmalert-tool-darwin-arm64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-arm64
+
+vmalert-tool-freebsd-amd64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-freebsd-amd64
+
+vmalert-tool-openbsd-amd64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-openbsd-amd64
+
+vmalert-tool-windows-amd64-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-windows-amd64
+
+package-vmalert-tool:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker
+
+package-vmalert-tool-pure:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-pure
+
+package-vmalert-tool-amd64:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-amd64
+
+package-vmalert-tool-arm:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm
+
+package-vmalert-tool-arm64:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm64
+
+package-vmalert-tool-ppc64le:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-ppc64le
+
+package-vmalert-tool-386:
+	APP_NAME=vmalert-tool $(MAKE) package-via-docker-386
+
+publish-vmalert-tool:
+	APP_NAME=vmalert-tool $(MAKE) publish-via-docker
+
+vmalert-tool-linux-amd64:
+	APP_NAME=vmalert-tool CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-linux-arm:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+
+vmalert-tool-linux-arm64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-linux-ppc64le:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+
+vmalert-tool-linux-s390x:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
+vmalert-tool-linux-386:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-darwin-amd64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-darwin-arm64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-freebsd-amd64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-openbsd-amd64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-tool-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmalert-tool $(MAKE) app-local-windows-goarch
+
+vmalert-tool-pure:
+	APP_NAME=vmalert-tool $(MAKE) app-local-pure

app/vmalert-tool/README.md

@@ -0,0 +1,3 @@
+See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
+
+vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).

app/vmalert-tool/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert-tool/unittest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+)
+
+func main() {
+	start := time.Now()
+	app := &cli.App{
+		Name:      "vmalert-tool",
+		Usage:     "VMAlert command-line tool",
+		UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html",
+		Version:   buildinfo.Version,
+		Commands: []*cli.Command{
+			{
+				Name:      "unittest",
+				Usage:     "Run unittest for alerting and recording rules.",
+				UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
+				Flags: []cli.Flag{
+					&cli.StringSliceFlag{
+						Name:     "files",
+						Usage:    "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
+						Required: true,
+					},
+					&cli.BoolFlag{
+						Name:     "disableAlertgroupLabel",
+						Usage:    "disable adding group's Name as label to generated alerts and time series.",
+						Required: false,
+					},
+				},
+				Action: func(c *cli.Context) error {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
+						return fmt.Errorf("unittest failed")
+					}
+					return nil
+				},
+			},
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Printf("Total time: %v", time.Since(start))
+}

app/vmalert-tool/multiarch/Dockerfile

@@ -0,0 +1,12 @@
+# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
+ARG certs_image
+ARG root_image
+FROM $certs_image as certs
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+
+FROM $root_image
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+EXPOSE 8429
+ENTRYPOINT ["/vmalert-tool-prod"]
+ARG TARGETARCH
+COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod

app/vmalert-tool/unittest/alerting.go

@@ -0,0 +1,19 @@
+package unittest
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// alertTestCase holds alert_rule_test cases defined in test file
+type alertTestCase struct {
+	EvalTime  *promutils.Duration `yaml:"eval_time"`
+	GroupName string              `yaml:"groupname"`
+	Alertname string              `yaml:"alertname"`
+	ExpAlerts []expAlert          `yaml:"exp_alerts"`
+}
+
+// expAlert holds exp_alerts defined in test file
+type expAlert struct {
+	ExpLabels      map[string]string `yaml:"exp_labels"`
+	ExpAnnotations map[string]string `yaml:"exp_annotations"`
+}

app/vmalert-tool/unittest/input.go

@@ -0,0 +1,182 @@
+package unittest
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// series holds input_series defined in the test file
+type series struct {
+	Series string `yaml:"series"`
+	Values string `yaml:"values"`
+}
+
+// sequenceValue is an omittable value in a sequence of time series values.
+type sequenceValue struct {
+	Value   float64
+	Omitted bool
+}
+
+func httpWrite(address string, r io.Reader) {
+	resp, err := http.Post(address, "", r)
+	if err != nil {
+		logger.Fatalf("failed to send to storage: %v", err)
+	}
+	resp.Body.Close()
+}
+
+// writeInputSeries send input series to vmstorage and flush them
+func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
+	r := testutil.WriteRequest{}
+	for _, data := range input {
+		expr, err := metricsql.Parse(data.Series)
+		if err != nil {
+			return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
+		}
+		promvals, err := parseInputValue(data.Values, true)
+		if err != nil {
+			return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
+		}
+		metricExpr, ok := expr.(*metricsql.MetricExpr)
+		if !ok {
+			return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
+		}
+		samples := make([]testutil.Sample, 0, len(promvals))
+		ts := startStamp
+		for _, v := range promvals {
+			if !v.Omitted {
+				samples = append(samples, testutil.Sample{
+					Timestamp: ts.UnixMilli(),
+					Value:     v.Value,
+				})
+			}
+			ts = ts.Add(interval.Duration())
+		}
+		var ls []testutil.Label
+		for _, filter := range metricExpr.LabelFilterss[0] {
+			ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
+		}
+		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
+	}
+
+	data, err := testutil.Compress(r)
+	if err != nil {
+		return fmt.Errorf("failed to compress data: %v", err)
+	}
+	// write input series to vm
+	httpWrite(dst, bytes.NewBuffer(data))
+	vmstorage.Storage.DebugFlush()
+	return nil
+}
+
+// parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.
+func parseInputValue(input string, origin bool) ([]sequenceValue, error) {
+	var res []sequenceValue
+	items := strings.Split(input, " ")
+	reg := regexp.MustCompile(`\D?\d*\D?`)
+	for _, item := range items {
+		if item == "stale" {
+			res = append(res, sequenceValue{Value: decimal.StaleNaN})
+			continue
+		}
+		vals := reg.FindAllString(item, -1)
+		switch len(vals) {
+		case 1:
+			if vals[0] == "_" {
+				res = append(res, sequenceValue{Omitted: true})
+				continue
+			}
+			v, err := strconv.ParseFloat(vals[0], 64)
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, sequenceValue{Value: v})
+			continue
+		case 2:
+			p1 := vals[0][:len(vals[0])-1]
+			v2, err := strconv.ParseInt(vals[1], 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			option := vals[0][len(vals[0])-1]
+			switch option {
+			case '+':
+				v1, err := strconv.ParseFloat(p1, 64)
+				if err != nil {
+					return nil, err
+				}
+				res = append(res, sequenceValue{Value: v1 + float64(v2)})
+			case 'x':
+				for i := int64(0); i <= v2; i++ {
+					if p1 == "_" {
+						if i == 0 {
+							i = 1
+						}
+						res = append(res, sequenceValue{Omitted: true})
+						continue
+					}
+					v1, err := strconv.ParseFloat(p1, 64)
+					if err != nil {
+						return nil, err
+					}
+					if !origin || v1 == 0 {
+						res = append(res, sequenceValue{Value: v1 * float64(i)})
+						continue
+					}
+					newVal := fmt.Sprintf("%s+0x%s", p1, vals[1])
+					newRes, err := parseInputValue(newVal, false)
+					if err != nil {
+						return nil, err
+					}
+					res = append(res, newRes...)
+					break
+				}
+
+			default:
+				return nil, fmt.Errorf("got invalid operation %b", option)
+			}
+		case 3:
+			r1, err := parseInputValue(fmt.Sprintf("%s%s", vals[1], vals[2]), false)
+			if err != nil {
+				return nil, err
+			}
+			p1 := vals[0][:len(vals[0])-1]
+			v1, err := strconv.ParseFloat(p1, 64)
+			if err != nil {
+				return nil, err
+			}
+			option := vals[0][len(vals[0])-1]
+			var isAdd bool
+			if option == '+' {
+				isAdd = true
+			}
+			for _, r := range r1 {
+				if isAdd {
+					res = append(res, sequenceValue{
+						Value: r.Value + v1,
+					})
+				} else {
+					res = append(res, sequenceValue{
+						Value: v1 - r.Value,
+					})
+				}
+			}
+		default:
+			return nil, fmt.Errorf("unsupported input %s", input)
+		}
+	}
+	return res, nil
+}

app/vmalert-tool/unittest/input_test.go

@@ -0,0 +1,93 @@
+package unittest
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+)
+
+func TestParseInputValue(t *testing.T) {
+	testCases := []struct {
+		input  string
+		exp    []sequenceValue
+		failed bool
+	}{
+		{
+			"",
+			nil,
+			true,
+		},
+		{
+			"testfailed",
+			nil,
+			true,
+		},
+		// stale doesn't support operations
+		{
+			"stalex3",
+			nil,
+			true,
+		},
+		{
+			"-4",
+			[]sequenceValue{{Value: -4}},
+			false,
+		},
+		{
+			"_",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"stale",
+			[]sequenceValue{{Value: decimal.StaleNaN}},
+			false,
+		},
+		{
+			"-4x1",
+			[]sequenceValue{{Value: -4}, {Value: -4}},
+			false,
+		},
+		{
+			"_x1",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"1+1x4",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}},
+			false,
+		},
+		{
+			"2-1x4",
+			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
+			false,
+		},
+		{
+			"1+1x1 _ -4 stale 3+20x1",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		output, err := parseInputValue(tc.input, true)
+		if err != nil != tc.failed {
+			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
+		}
+		if len(tc.exp) != len(output) {
+			t.Fatalf("expect %v, got %v", tc.exp, output)
+		}
+		for i := 0; i < len(tc.exp); i++ {
+			if tc.exp[i].Omitted != output[i].Omitted {
+				t.Fatalf("expect %v, got %v", tc.exp, output)
+			}
+			if tc.exp[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+					continue
+				}
+				t.Fatalf("expect %v, got %v", tc.exp, output)
+			}
+		}
+	}
+}

app/vmalert-tool/unittest/recording.go

@@ -0,0 +1,100 @@
+package unittest
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"reflect"
+	"sort"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// metricsqlTestCase holds metricsql_expr_test cases defined in test file
+type metricsqlTestCase struct {
+	Expr       string              `yaml:"expr"`
+	EvalTime   *promutils.Duration `yaml:"eval_time"`
+	ExpSamples []expSample         `yaml:"exp_samples"`
+}
+
+type expSample struct {
+	Labels string  `yaml:"labels"`
+	Value  float64 `yaml:"value"`
+}
+
+// checkMetricsqlCase will check metricsql_expr_test cases
+func checkMetricsqlCase(cases []metricsqlTestCase, q datasource.QuerierBuilder) (checkErrs []error) {
+	queries := q.BuildWithParams(datasource.QuerierParams{QueryParams: url.Values{"nocache": {"1"}, "latency_offset": {"1ms"}}, DataSourceType: "prometheus"})
+Outer:
+	for _, mt := range cases {
+		result, _, err := queries.Query(context.Background(), mt.Expr, durationToTime(mt.EvalTime))
+		if err != nil {
+			checkErrs = append(checkErrs, fmt.Errorf("    expr: %q, time: %s, err: %w", mt.Expr,
+				mt.EvalTime.Duration().String(), err))
+			continue
+		}
+		var gotSamples []parsedSample
+		for _, s := range result.Data {
+			sort.Slice(s.Labels, func(i, j int) bool {
+				return s.Labels[i].Name < s.Labels[j].Name
+			})
+			gotSamples = append(gotSamples, parsedSample{
+				Labels: s.Labels,
+				Value:  s.Values[0],
+			})
+		}
+		var expSamples []parsedSample
+		for _, s := range mt.ExpSamples {
+			expLb := datasource.Labels{}
+			if s.Labels != "" {
+				metricsqlExpr, err := metricsql.Parse(s.Labels)
+				if err != nil {
+					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
+						mt.EvalTime.Duration().String(), fmt.Errorf("failed to parse labels %q: %w", s.Labels, err)))
+					continue Outer
+				}
+				metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
+				if !ok {
+					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
+						mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
+					continue Outer
+				}
+				for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
+					expLb = append(expLb, datasource.Label{
+						Name:  l.Label,
+						Value: l.Value,
+					})
+				}
+			}
+			sort.Slice(expLb, func(i, j int) bool {
+				return expLb[i].Name < expLb[j].Name
+			})
+			expSamples = append(expSamples, parsedSample{
+				Labels: expLb,
+				Value:  s.Value,
+			})
+		}
+		sort.Slice(expSamples, func(i, j int) bool {
+			return datasource.LabelCompare(expSamples[i].Labels, expSamples[j].Labels) <= 0
+		})
+		sort.Slice(gotSamples, func(i, j int) bool {
+			return datasource.LabelCompare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
+		})
+		if !reflect.DeepEqual(expSamples, gotSamples) {
+			checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s,\n        exp: %v\n        got: %v", mt.Expr,
+				mt.EvalTime.Duration().String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
+		}
+
+	}
+	return
+}
+
+func durationToTime(pd *promutils.Duration) time.Time {
+	if pd == nil {
+		return time.Time{}
+	}
+	return time.UnixMilli(pd.Duration().Milliseconds())
+}

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -0,0 +1,43 @@
+rule_files:
+  - rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'up{job="vmagent2", instance="localhost:9090"}'
+        values: "0+0x1440"
+
+    metricsql_expr_test:
+      - expr: suquery_interval_test
+        eval_time: 4m
+        exp_samples:
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+            value: 1
+
+    alert_rule_test:
+      - eval_time: 2h
+        alertname: InstanceDown
+        exp_alerts:
+          - exp_labels:
+              job: vmagent2
+              severity: page
+              instance: localhost:9090
+              datacenter: dc-123
+            exp_annotations:
+              summary: "Instance localhost:9090 down"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+
+      - eval_time: 0
+        alertname: AlwaysFiring
+        exp_alerts:
+          - exp_labels:
+              datacenter: dc-123
+
+      - eval_time: 0
+        alertname: InstanceDown
+        exp_alerts: []
+
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/failed-test.yaml

@@ -0,0 +1,49 @@
+rule_files:
+  - rules.yaml
+
+tests:
+  - interval: 1m
+    name: "Failing test"
+    input_series:
+      - series: test
+        values: "0"
+
+    metricsql_expr_test:
+      - expr: test
+        eval_time: 0m
+        exp_samples:
+          - value: 0
+            labels: test
+
+    # will failed cause there is no "Test" group and rule defined
+    alert_rule_test:
+      - eval_time: 0m
+        groupname: Test
+        alertname: Test
+        exp_alerts:
+          - exp_labels: {}
+
+  - interval: 1m
+    name: Failing alert test
+    input_series:
+      - series: 'up{job="test"}'
+        values: 0x10
+
+    alert_rule_test:
+      # will failed cause rule is firing
+      - eval_time: 5m
+        groupname: group1
+        alertname: InstanceDown
+        exp_alerts: []
+
+  - interval: 1m
+    name: Failing alert test with missing groupname
+    input_series:
+      - series: 'up{job="test"}'
+        values: 0x10
+
+    alert_rule_test:
+      # will failed cause missing groupname
+      - eval_time: 5m
+        alertname: AlwaysFiring
+        exp_alerts: []

app/vmalert-tool/unittest/testdata/long-period.yaml

@@ -0,0 +1,30 @@
+# can be executed successfully but will take more than 1 minute
+# not included in unit test now
+evaluation_interval: 100d
+
+rule_files:
+  - rules.yaml
+
+tests:
+  - interval: 1d
+    input_series:
+      - series: test
+        # Max time in time.Duration is 106751d from 1970 (2^63/10^9), i.e. 2262.
+        # But VictoriaMetrics supports maxTimestamp value +2 days from now. see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/827.
+        # We input series to 2024-01-01T00:00:00 here.
+        values: "0+1x19723"
+
+    metricsql_expr_test:
+      - expr: timestamp(test)
+        eval_time: 0m
+        exp_samples:
+          - value: 0
+      - expr: test
+        eval_time: 100d
+        exp_samples:
+          - labels: test
+            value: 100
+      - expr: timestamp(test)
+        eval_time: 19000d
+        exp_samples:
+          - value: 1641600000 # 19000d -> seconds.

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -0,0 +1,39 @@
+groups:
+  - name: group1
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      - alert: AlwaysFiring
+        expr: 1
+      - alert: SameAlertNameWithDifferentGroup
+        expr: absent(test)
+        for: 1m
+
+  - name: group2
+    rules:
+      - record: t1
+        expr: test
+      - record: job:test:count_over_time1m
+        expr: sum without(instance) (count_over_time(test[1m]))
+      - record: suquery_interval_test
+        expr: count_over_time(up[5m:])
+
+      - alert: SameAlertNameWithDifferentGroup
+        expr: absent(test)
+        for: 5m
+
+  - name: group3
+    rules:
+      - record: t2
+        expr: t1
+
+  - name: group4
+    rules:
+      - record: t3
+        expr: t1

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -0,0 +1,104 @@
+rule_files:
+  - rules.yaml
+
+evaluation_interval: 1m
+group_eval_order: ["group4", "group2", "group3"]
+
+tests:
+  - interval: 1m
+    name: "basic test"
+    input_series:
+      - series: "test"
+        values: "_x5 1x5 _ stale"
+
+    alert_rule_test:
+      - eval_time: 1m
+        groupname: group1
+        alertname: SameAlertNameWithDifferentGroup
+        exp_alerts:
+          - {}
+      - eval_time: 1m
+        groupname: group2
+        alertname: SameAlertNameWithDifferentGroup
+        exp_alerts: []
+      - eval_time: 150s
+        groupname: group1
+        alertname: SameAlertNameWithDifferentGroup
+        exp_alerts:
+          - {}
+      - eval_time: 6m
+        groupname: group1
+        alertname: SameAlertNameWithDifferentGroup
+        exp_alerts: []
+
+    metricsql_expr_test:
+      - expr: test
+        eval_time: 11m
+        exp_samples:
+          - labels: '{__name__="test"}'
+            value: 1
+      - expr: test
+        eval_time: 12m
+        exp_samples: []
+
+  - interval: 1m
+    name: "basic test2"
+    input_series:
+      - series: 'up{job="vmagent1", instance="localhost:9090"}'
+        values: "0+0x1440"
+      - series: "test"
+        values: "0+1x1440"
+
+    metricsql_expr_test:
+      - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
+        eval_time: 4m
+        exp_samples:
+          - labels: '{alertgroup="group1", alertname="AlwaysFiring", alertstate="firing"}'
+            value: 1
+          - labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}'
+            value: 1
+      - expr: t1
+        eval_time: 4m
+        exp_samples:
+          - value: 4
+            labels: '{__name__="t1", datacenter="dc-123"}'
+      - expr: t2
+        eval_time: 4m
+        exp_samples:
+          - value: 4
+            labels: '{__name__="t2", datacenter="dc-123"}'
+      - expr: t3
+        eval_time: 4m
+        exp_samples:
+          # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
+          - value: 3
+            labels: '{__name__="t3", datacenter="dc-123"}'
+
+    alert_rule_test:
+      - eval_time: 10m
+        groupname: group1
+        alertname: InstanceDown
+        exp_alerts:
+          - exp_labels:
+              job: vmagent1
+              severity: page
+              instance: localhost:9090
+              datacenter: dc-123
+            exp_annotations:
+              summary: "Instance localhost:9090 down"
+              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
+
+      - eval_time: 0
+        groupname: group1
+        alertname: AlwaysFiring
+        exp_alerts:
+          - exp_labels:
+              datacenter: dc-123
+
+      - eval_time: 0
+        groupname: alerts
+        alertname: InstanceDown
+        exp_alerts: []
+
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -0,0 +1,46 @@
+rule_files:
+  - rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'up{job="vmagent2", instance="localhost:9090"}'
+        values: "0+0x1440"
+
+    metricsql_expr_test:
+      - expr: suquery_interval_test
+        eval_time: 4m
+        exp_samples:
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+            value: 1
+
+    alert_rule_test:
+      - eval_time: 2h
+        groupname: group1
+        alertname: InstanceDown
+        exp_alerts:
+          - exp_labels:
+              job: vmagent2
+              severity: page
+              instance: localhost:9090
+              datacenter: dc-123
+            exp_annotations:
+              summary: "Instance localhost:9090 down"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+
+      - eval_time: 0
+        groupname: group1
+        alertname: AlwaysFiring
+        exp_alerts:
+          - exp_labels:
+              datacenter: dc-123
+
+      - eval_time: 0
+        groupname: group1
+        alertname: InstanceDown
+        exp_alerts: []
+
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/type.go

@@ -0,0 +1,83 @@
+package unittest
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+)
+
+// parsedSample is a sample with parsed Labels
+type parsedSample struct {
+	Labels datasource.Labels
+	Value  float64
+}
+
+func (ps *parsedSample) String() string {
+	return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64)
+}
+
+func parsedSamplesString(pss []parsedSample) string {
+	if len(pss) == 0 {
+		return "nil"
+	}
+	s := pss[0].String()
+	for _, ps := range pss[1:] {
+		s += ", " + ps.String()
+	}
+	return s
+}
+
+// labelAndAnnotation holds labels and annotations
+type labelAndAnnotation struct {
+	Labels      datasource.Labels
+	Annotations datasource.Labels
+}
+
+func (la *labelAndAnnotation) String() string {
+	return "Labels:" + la.Labels.String() + "\nAnnotations:" + la.Annotations.String()
+}
+
+// labelsAndAnnotations is collection of LabelAndAnnotation
+type labelsAndAnnotations []labelAndAnnotation
+
+func (la labelsAndAnnotations) Len() int { return len(la) }
+
+func (la labelsAndAnnotations) Swap(i, j int) { la[i], la[j] = la[j], la[i] }
+func (la labelsAndAnnotations) Less(i, j int) bool {
+	diff := datasource.LabelCompare(la[i].Labels, la[j].Labels)
+	if diff != 0 {
+		return diff < 0
+	}
+	return datasource.LabelCompare(la[i].Annotations, la[j].Annotations) < 0
+}
+
+func (la labelsAndAnnotations) String() string {
+	if len(la) == 0 {
+		return "[]"
+	}
+	s := "[\n0:" + indentLines("\n"+la[0].String(), "  ")
+	for i, l := range la[1:] {
+		s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), "  ")
+	}
+	s += "\n]"
+
+	return s
+}
+
+// indentLines prefixes each line in the supplied string with the given "indent" string.
+func indentLines(lines, indent string) string {
+	sb := strings.Builder{}
+	n := strings.Split(lines, "\n")
+	for i, l := range n {
+		if i > 0 {
+			sb.WriteString(indent)
+		}
+		sb.WriteString(l)
+		if i != len(n)-1 {
+			sb.WriteRune('\n')
+		}
+	}
+	return sb.String()
+}

app/vmalert-tool/unittest/unittest.go

@@ -0,0 +1,443 @@
+package unittest
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"time"
+
+	"gopkg.in/yaml.v2"
+
+	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	storagePath    string
+	httpListenAddr = ":8880"
+	// insert series from 1970-01-01T00:00:00
+	testStartTime = time.Unix(0, 0).UTC()
+
+	testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write"
+	testDataSourcePath    = "http://127.0.0.1" + httpListenAddr + "/prometheus"
+	testRemoteWritePath   = "http://127.0.0.1" + httpListenAddr
+	testHealthHTTPPath    = "http://127.0.0.1" + httpListenAddr + "/health"
+
+	disableAlertgroupLabel bool
+)
+
+const (
+	testStoragePath = "vmalert-unittest"
+	testLogLevel    = "ERROR"
+)
+
+// UnitTest runs unittest for files
+func UnitTest(files []string, disableGroupLabel bool) bool {
+	if err := templates.Load([]string{}, true); err != nil {
+		logger.Fatalf("failed to load template: %v", err)
+	}
+	storagePath = filepath.Join(os.TempDir(), testStoragePath)
+	processFlags()
+	vminsert.Init()
+	vmselect.Init()
+	// storagePath will be created again when closing vmselect, so remove it again.
+	defer fs.MustRemoveAll(storagePath)
+	defer vminsert.Stop()
+	defer vmselect.Stop()
+	disableAlertgroupLabel = disableGroupLabel
+	return rulesUnitTest(files)
+}
+
+func rulesUnitTest(files []string) bool {
+	var failed bool
+	for _, f := range files {
+		if err := ruleUnitTest(f); err != nil {
+			fmt.Println("  FAILED")
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
+			failed = true
+		} else {
+			fmt.Println("  SUCCESS")
+		}
+	}
+	return failed
+}
+
+func ruleUnitTest(filename string) []error {
+	fmt.Println("\nUnit Testing: ", filename)
+	b, err := os.ReadFile(filename)
+	if err != nil {
+		return []error{fmt.Errorf("failed to read file: %w", err)}
+	}
+
+	var unitTestInp unitTestFile
+	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
+	}
+	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
+		return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
+	}
+
+	if unitTestInp.EvaluationInterval.Duration() == 0 {
+		fmt.Println("evaluation_interval set to 1m by default")
+		unitTestInp.EvaluationInterval = &promutils.Duration{D: 1 * time.Minute}
+	}
+
+	groupOrderMap := make(map[string]int)
+	for i, gn := range unitTestInp.GroupEvalOrder {
+		if _, ok := groupOrderMap[gn]; ok {
+			return []error{fmt.Errorf("group name repeated in `group_eval_order`: %s", gn)}
+		}
+		groupOrderMap[gn] = i
+	}
+
+	testGroups, err := vmalertconfig.Parse(unitTestInp.RuleFiles, nil, true)
+	if err != nil {
+		return []error{fmt.Errorf("failed to parse `rule_files`: %w", err)}
+	}
+
+	var errs []error
+	for _, t := range unitTestInp.Tests {
+		if err := verifyTestGroup(t); err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
+		errs = append(errs, testErrs...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func verifyTestGroup(group testGroup) error {
+	var testGroupName string
+	if group.TestGroupName != "" {
+		testGroupName = fmt.Sprintf("testGroupName: %s\n", group.TestGroupName)
+	}
+	for _, at := range group.AlertRuleTests {
+		if at.Alertname == "" {
+			return fmt.Errorf("\n%s    missing required filed \"alertname\"", testGroupName)
+		}
+		if !disableAlertgroupLabel && at.GroupName == "" {
+			return fmt.Errorf("\n%s    missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName)
+		}
+		if disableAlertgroupLabel && at.GroupName != "" {
+			return fmt.Errorf("\n%s    shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName)
+		}
+		if at.EvalTime == nil {
+			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+		}
+	}
+	for _, et := range group.MetricsqlExprTests {
+		if et.Expr == "" {
+			return fmt.Errorf("\n%s    missing required filed \"expr\"", testGroupName)
+		}
+		if et.EvalTime == nil {
+			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+		}
+	}
+	return nil
+}
+
+func processFlags() {
+	flag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "loggerLevel", value: testLogLevel},
+		{flag: "search.disableCache", value: "true"},
+		// set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00.
+		{flag: "retentionPeriod", value: "100y"},
+		{flag: "datasource.url", value: testDataSourcePath},
+		{flag: "remoteWrite.url", value: testRemoteWritePath},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			logger.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func setUp() {
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
+		switch r.URL.Path {
+		case "/prometheus/api/v1/query":
+			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+			return true
+		case "/prometheus/api/v1/write", "/api/v1/write":
+			if err := promremotewrite.InsertHandler(r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+			return true
+		default:
+		}
+		return false
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	readyCheckFunc := func() bool {
+		resp, err := http.Get(testHealthHTTPPath)
+		if err != nil {
+			return false
+		}
+		_ = resp.Body.Close()
+		return resp.StatusCode == 200
+	}
+checkCheck:
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Fatalf("http server can't be ready in 30s")
+		default:
+			if readyCheckFunc() {
+				break checkCheck
+			}
+			time.Sleep(3 * time.Second)
+		}
+	}
+}
+
+func tearDown() {
+	if err := httpserver.Stop(httpListenAddr); err != nil {
+		logger.Errorf("cannot stop the webservice: %s", err)
+	}
+	vmstorage.Stop()
+	metrics.UnregisterAllMetrics()
+	fs.MustRemoveAll(storagePath)
+}
+
+// resolveAndGlobFilepaths joins all relative paths in a configuration
+// with a given base directory and replaces all globs with matching files.
+func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
+	for i, rf := range utf.RuleFiles {
+		if rf != "" && !filepath.IsAbs(rf) {
+			utf.RuleFiles[i] = filepath.Join(baseDir, rf)
+		}
+	}
+
+	var globbedFiles []string
+	for _, rf := range utf.RuleFiles {
+		m, err := filepath.Glob(rf)
+		if err != nil {
+			return err
+		}
+		if len(m) == 0 {
+			fmt.Fprintln(os.Stderr, "  WARNING: no file match pattern", rf)
+		}
+		globbedFiles = append(globbedFiles, m...)
+	}
+	utf.RuleFiles = globbedFiles
+	return nil
+}
+
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
+	// set up vmstorage and http server for ingest and read queries
+	setUp()
+	// tear down vmstorage and clean the data dir
+	defer tearDown()
+
+	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath)
+	if err != nil {
+		return []error{err}
+	}
+
+	q, err := datasource.Init(nil)
+	if err != nil {
+		return []error{fmt.Errorf("failed to init datasource: %v", err)}
+	}
+	rw, err := remotewrite.NewDebugClient()
+	if err != nil {
+		return []error{fmt.Errorf("failed to init wr: %v", err)}
+	}
+
+	alertEvalTimesMap := map[time.Duration]struct{}{}
+	alertExpResultMap := map[time.Duration]map[string]map[string][]expAlert{}
+	for _, at := range tg.AlertRuleTests {
+		et := at.EvalTime.Duration()
+		alertEvalTimesMap[et] = struct{}{}
+		if _, ok := alertExpResultMap[et]; !ok {
+			alertExpResultMap[et] = make(map[string]map[string][]expAlert)
+		}
+		if _, ok := alertExpResultMap[et][at.GroupName]; !ok {
+			alertExpResultMap[et][at.GroupName] = make(map[string][]expAlert)
+		}
+		alertExpResultMap[et][at.GroupName][at.Alertname] = at.ExpAlerts
+	}
+	alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
+	for k := range alertEvalTimesMap {
+		alertEvalTimes = append(alertEvalTimes, k)
+	}
+	sort.Slice(alertEvalTimes, func(i, j int) bool {
+		return alertEvalTimes[i] < alertEvalTimes[j]
+	})
+
+	// sort group eval order according to the given "group_eval_order".
+	sort.Slice(testGroups, func(i, j int) bool {
+		return groupOrderMap[testGroups[i].Name] < groupOrderMap[testGroups[j].Name]
+	})
+
+	// create groups with given rule
+	var groups []*rule.Group
+	for _, group := range testGroups {
+		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
+		groups = append(groups, ng)
+	}
+
+	evalIndex := 0
+	maxEvalTime := testStartTime.Add(tg.maxEvalTime())
+	for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) {
+		for _, g := range groups {
+			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
+			for err := range errs {
+				if err != nil {
+					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,
+						ts, err))
+				}
+			}
+			// flush series after each group evaluation
+			vmstorage.Storage.DebugFlush()
+		}
+
+		// check alert_rule_test case at every eval time
+		for evalIndex < len(alertEvalTimes) {
+			if ts.Sub(testStartTime) > alertEvalTimes[evalIndex] ||
+				alertEvalTimes[evalIndex] >= ts.Add(evalInterval).Sub(testStartTime) {
+				break
+			}
+			gotAlertsMap := map[string]map[string]labelsAndAnnotations{}
+			for _, g := range groups {
+				if disableAlertgroupLabel {
+					g.Name = ""
+				}
+				if _, ok := alertExpResultMap[alertEvalTimes[evalIndex]][g.Name]; !ok {
+					continue
+				}
+				if _, ok := gotAlertsMap[g.Name]; !ok {
+					gotAlertsMap[g.Name] = make(map[string]labelsAndAnnotations)
+				}
+				for _, r := range g.Rules {
+					ar, isAlertRule := r.(*rule.AlertingRule)
+					if !isAlertRule {
+						continue
+					}
+					if _, ok := alertExpResultMap[alertEvalTimes[evalIndex]][g.Name][ar.Name]; ok {
+						for _, got := range ar.GetAlerts() {
+							if got.State != notifier.StateFiring {
+								continue
+							}
+							if disableAlertgroupLabel {
+								delete(got.Labels, "alertgroup")
+							}
+							laa := labelAndAnnotation{
+								Labels:      datasource.ConvertToLabels(got.Labels),
+								Annotations: datasource.ConvertToLabels(got.Annotations),
+							}
+							gotAlertsMap[g.Name][ar.Name] = append(gotAlertsMap[g.Name][ar.Name], laa)
+						}
+					}
+
+				}
+			}
+			for groupname, gres := range alertExpResultMap[alertEvalTimes[evalIndex]] {
+				for alertname, res := range gres {
+					var expAlerts labelsAndAnnotations
+					for _, expAlert := range res {
+						if expAlert.ExpLabels == nil {
+							expAlert.ExpLabels = make(map[string]string)
+						}
+						// alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false
+						if !disableAlertgroupLabel {
+							expAlert.ExpLabels["alertgroup"] = groupname
+						}
+						// alertNameLabel is added as additional labels in vmalert.
+						expAlert.ExpLabels["alertname"] = alertname
+						expAlerts = append(expAlerts, labelAndAnnotation{
+							Labels:      datasource.ConvertToLabels(expAlert.ExpLabels),
+							Annotations: datasource.ConvertToLabels(expAlert.ExpAnnotations),
+						})
+					}
+					sort.Sort(expAlerts)
+
+					gotAlerts := gotAlertsMap[groupname][alertname]
+					sort.Sort(gotAlerts)
+					if !reflect.DeepEqual(expAlerts, gotAlerts) {
+						var testGroupName string
+						if tg.TestGroupName != "" {
+							testGroupName = fmt.Sprintf("testGroupName: %s,\n", tg.TestGroupName)
+						}
+						expString := indentLines(expAlerts.String(), "            ")
+						gotString := indentLines(gotAlerts.String(), "            ")
+						checkErrs = append(checkErrs, fmt.Errorf("\n%s    groupname: %s, alertname: %s, time: %s, \n        exp:%v, \n        got:%v ",
+							testGroupName, groupname, alertname, alertEvalTimes[evalIndex].String(), expString, gotString))
+					}
+				}
+			}
+			evalIndex++
+		}
+
+	}
+
+	checkErrs = append(checkErrs, checkMetricsqlCase(tg.MetricsqlExprTests, q)...)
+	return checkErrs
+}
+
+// unitTestFile holds the contents of a single unit test file
+type unitTestFile struct {
+	RuleFiles          []string            `yaml:"rule_files"`
+	EvaluationInterval *promutils.Duration `yaml:"evaluation_interval"`
+	GroupEvalOrder     []string            `yaml:"group_eval_order"`
+	Tests              []testGroup         `yaml:"tests"`
+}
+
+// testGroup is a group of input series and test cases associated with it
+type testGroup struct {
+	Interval           *promutils.Duration `yaml:"interval"`
+	InputSeries        []series            `yaml:"input_series"`
+	AlertRuleTests     []alertTestCase     `yaml:"alert_rule_test"`
+	MetricsqlExprTests []metricsqlTestCase `yaml:"metricsql_expr_test"`
+	ExternalLabels     map[string]string   `yaml:"external_labels"`
+	TestGroupName      string              `yaml:"name"`
+}
+
+// maxEvalTime returns the max eval time among all alert_rule_test and metricsql_expr_test
+func (tg *testGroup) maxEvalTime() time.Duration {
+	var maxd time.Duration
+	for _, alert := range tg.AlertRuleTests {
+		if alert.EvalTime.Duration() > maxd {
+			maxd = alert.EvalTime.Duration()
+		}
+	}
+	for _, met := range tg.MetricsqlExprTests {
+		if met.EvalTime.Duration() > maxd {
+			maxd = met.EvalTime.Duration()
+		}
+	}
+	return maxd
+}

app/vmalert-tool/unittest/unittest_test.go

@@ -0,0 +1,47 @@
+package unittest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+)
+
+func TestMain(m *testing.M) {
+	if err := templates.Load([]string{}, true); err != nil {
+		os.Exit(1)
+	}
+	os.Exit(m.Run())
+}
+
+func TestUnitRule(t *testing.T) {
+	testCases := []struct {
+		name              string
+		disableGroupLabel bool
+		files             []string
+		failed            bool
+	}{
+		{
+			name:   "run multi files",
+			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
+			failed: false,
+		},
+		{
+			name:              "disable group label",
+			disableGroupLabel: true,
+			files:             []string{"./testdata/disable-group-label.yaml"},
+			failed:            false,
+		},
+		{
+			name:   "failing test",
+			files:  []string{"./testdata/failed-test.yaml"},
+			failed: true,
+		},
+	}
+	for _, tc := range testCases {
+		fail := UnitTest(tc.files, tc.disableGroupLabel)
+		if fail != tc.failed {
+			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
+		}
+	}
+}

app/vmalert/config/config.go

@@ -19,11 +19,14 @@ import (
 // Group contains list of Rules grouped into
 // entity with one name and evaluation interval
 type Group struct {
-	Type        Type `yaml:"type,omitempty"`
-	File        string
-	Name        string              `yaml:"name"`
-	Interval    *promutils.Duration `yaml:"interval,omitempty"`
-	EvalOffset  *promutils.Duration `yaml:"eval_offset,omitempty"`
+	Type       Type `yaml:"type,omitempty"`
+	File       string
+	Name       string              `yaml:"name"`
+	Interval   *promutils.Duration `yaml:"interval,omitempty"`
+	EvalOffset *promutils.Duration `yaml:"eval_offset,omitempty"`
+	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
+	EvalDelay   *promutils.Duration `yaml:"eval_delay,omitempty"`
 	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
@@ -39,6 +42,8 @@ type Group struct {
 	Headers []Header `yaml:"headers,omitempty"`
 	// NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications
 	NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
+	// EvalAlignment will make the timestamp of group query requests be aligned with interval
+	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
 }
@@ -231,7 +236,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 	files, err := readFromFS(pathPatterns)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read from the config: %s", err)
+		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
 	return parse(files, validateTplFn, validateExpressions)
 }
@@ -240,11 +245,11 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	files, err := readFromFS(pathPatterns)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read from the config: %s", err)
+		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
 	groups, err := parse(files, validateTplFn, validateExpressions)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
+		return nil, fmt.Errorf("failed to parse %s: %w", pathPatterns, err)
 	}
 	if len(groups) < 1 {
 		cLogger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))

app/vmalert/config/config_test.go

@@ -106,7 +106,7 @@ func TestParseBad(t *testing.T) {
 		},
 		{
 			[]string{"http://unreachable-url"},
-			"failed to read",
+			"failed to",
 		},
 	}
 	for _, tc := range testCases {

app/vmalert/config/fsurl/url.go

@@ -49,7 +49,7 @@ func (fs *FS) Read(files []string) (map[string][]byte, error) {
 				path, resp.StatusCode, http.StatusOK, data)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("cannot read %q: %s", path, err)
+			return nil, fmt.Errorf("cannot read %q: %w", path, err)
 		}
 		result[path] = data
 	}

app/vmalert/config/testdata/rules/rules3-good.rules

@@ -15,6 +15,7 @@ groups:
     interval: 2s
     concurrency: 2
     type: prometheus
+    eval_delay: 30s
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by (instance) > 1

app/vmalert/datasource/datasource.go

@@ -44,7 +44,6 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     string
 	EvaluationInterval time.Duration
-	EvalOffset         *time.Duration
 	QueryParams        url.Values
 	Headers            map[string]string
 	Debug              bool

app/vmalert/datasource/faker.go

@@ -0,0 +1,131 @@
+package datasource
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+)
+
+// FakeQuerier is a mock querier that return predefined results and error message
+type FakeQuerier struct {
+	sync.Mutex
+	metrics []Metric
+	err     error
+}
+
+// SetErr sets query error message
+func (fq *FakeQuerier) SetErr(err error) {
+	fq.Lock()
+	fq.err = err
+	fq.Unlock()
+}
+
+// Reset reset querier's error message and results
+func (fq *FakeQuerier) Reset() {
+	fq.Lock()
+	fq.err = nil
+	fq.metrics = fq.metrics[:0]
+	fq.Unlock()
+}
+
+// Add appends metrics to querier result metrics
+func (fq *FakeQuerier) Add(metrics ...Metric) {
+	fq.Lock()
+	fq.metrics = append(fq.metrics, metrics...)
+	fq.Unlock()
+}
+
+// BuildWithParams return FakeQuerier itself
+func (fq *FakeQuerier) BuildWithParams(_ QuerierParams) Querier {
+	return fq
+}
+
+// QueryRange performs query
+func (fq *FakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) (Result, error) {
+	req, _, err := fq.Query(ctx, q, time.Now())
+	return req, err
+}
+
+// Query returns metrics restored in querier
+func (fq *FakeQuerier) Query(_ context.Context, _ string, _ time.Time) (Result, *http.Request, error) {
+	fq.Lock()
+	defer fq.Unlock()
+	if fq.err != nil {
+		return Result{}, nil, fq.err
+	}
+	cp := make([]Metric, len(fq.metrics))
+	copy(cp, fq.metrics)
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	return Result{Data: cp}, req, nil
+}
+
+// FakeQuerierWithRegistry can store different results for different query expr
+type FakeQuerierWithRegistry struct {
+	sync.Mutex
+	registry map[string][]Metric
+}
+
+// Set stores query result for given key
+func (fqr *FakeQuerierWithRegistry) Set(key string, metrics ...Metric) {
+	fqr.Lock()
+	if fqr.registry == nil {
+		fqr.registry = make(map[string][]Metric)
+	}
+	fqr.registry[key] = metrics
+	fqr.Unlock()
+}
+
+// Reset clean querier's results registry
+func (fqr *FakeQuerierWithRegistry) Reset() {
+	fqr.Lock()
+	fqr.registry = nil
+	fqr.Unlock()
+}
+
+// BuildWithParams returns itself
+func (fqr *FakeQuerierWithRegistry) BuildWithParams(_ QuerierParams) Querier {
+	return fqr
+}
+
+// QueryRange performs query
+func (fqr *FakeQuerierWithRegistry) QueryRange(ctx context.Context, q string, _, _ time.Time) (Result, error) {
+	req, _, err := fqr.Query(ctx, q, time.Now())
+	return req, err
+}
+
+// Query returns metrics restored in querier registry
+func (fqr *FakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time.Time) (Result, *http.Request, error) {
+	fqr.Lock()
+	defer fqr.Unlock()
+
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	metrics, ok := fqr.registry[expr]
+	if !ok {
+		return Result{}, req, nil
+	}
+	cp := make([]Metric, len(metrics))
+	copy(cp, metrics)
+	return Result{Data: cp}, req, nil
+}
+
+// FakeQuerierWithDelay mock querier with given delay duration
+type FakeQuerierWithDelay struct {
+	FakeQuerier
+	Delay time.Duration
+}
+
+// Query returns query result after delay duration
+func (fqd *FakeQuerierWithDelay) Query(ctx context.Context, expr string, ts time.Time) (Result, *http.Request, error) {
+	timer := time.NewTimer(fqd.Delay)
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+	return fqd.FakeQuerier.Query(ctx, expr, ts)
+}
+
+// BuildWithParams returns itself
+func (fqd *FakeQuerierWithDelay) BuildWithParams(_ QuerierParams) Querier {
+	return fqd
+}

app/vmalert/datasource/init.go

@@ -10,13 +10,14 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
 		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
-	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to show -datasource.url in the exported metrics. "+
+	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 
 	headers = flag.String("datasource.headers", "", "Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. "+
@@ -36,18 +37,24 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
-	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
-	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
-	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
-	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
-	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url")
+	oauth2EndpointParams   = flag.String("datasource.oauth2.endpointParams", "", "Optional OAuth2 endpoint parameters to use for -datasource.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	oauth2TokenURL = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url")
+	oauth2Scopes   = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
-	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
+	lookBack = flag.Duration("datasource.lookback", 0, `Will be deprecated soon, please adjust "-search.latencyOffset"  at datasource side `+
+		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
+		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
 		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
 		"If set to 0, rule's evaluation interval will be used instead.")
-	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
-		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Deprecated: please use "eval_alignment" in rule group instead. `+
+		`Whether to align "time" parameter with evaluation interval. `+
+		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. "+
+		"See more details at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
 		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
@@ -62,6 +69,11 @@ func InitSecretFlags() {
 	}
 }
 
+// ShowDatasourceURL whether to show -datasource.url with sensitive information
+func ShowDatasourceURL() bool {
+	return *showDatasourceURL
+}
+
 // Param represents an HTTP GET param
 type Param struct {
 	Key, Value string
@@ -74,6 +86,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
+	if !*queryTimeAlignment {
+		logger.Warnf("flag `-datasource.queryTimeAlignment` is deprecated and will be removed in next releases. Please use `eval_alignment` in rule group instead.")
+	}
+	if *lookBack != 0 {
+		logger.Warnf("flag `-datasource.lookback` will be deprecated soon. Please use `-rule.evalDelay` command-line flag instead. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
+	}
 
 	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
@@ -92,14 +110,22 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}
 
+	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
+	}
 	authCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		utils.WithBearer(*bearerToken, *bearerTokenFile),
-		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		utils.WithHeaders(*headers))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
+	_, err = authCfg.GetAuthHeader()
+	if err != nil {
+		return nil, fmt.Errorf("failed to set request auth header to datasource %q: %w", *addr, err)
+	}
 
 	return &VMStorage{
 		c:                &http.Client{Transport: tr},

app/vmalert/datasource/vm.go

@@ -39,14 +39,8 @@ type VMStorage struct {
 	queryStep        time.Duration
 	dataSourceType   datasourceType
 
-	// evaluationInterval will align the request's timestamp
-	// if `datasource.queryTimeAlignment` is enabled,
-	// will set request's `step` param as well.
+	// evaluationInterval will help setting request's `step` param.
 	evaluationInterval time.Duration
-	// evaluationOffset shifts the request's timestamp, will be equal
-	// to the offset specified evaluationInterval.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4693
-	evaluationOffset *time.Duration
 	// extraParams contains params to be attached to each HTTP request
 	extraParams url.Values
 	// extraHeaders are headers to be attached to each HTTP request
@@ -95,7 +89,6 @@ func (s *VMStorage) Clone() *VMStorage {
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
-	s.evaluationOffset = params.EvalOffset
 	if params.QueryParams != nil {
 		if s.extraParams == nil {
 			s.extraParams = url.Values{}
@@ -144,26 +137,35 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req := s.newQueryRequest(query, ts)
-	resp, err := s.do(ctx, req)
-	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
-		// something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		req = s.newQueryRequest(query, ts)
-		resp, err = s.do(ctx, req)
-	}
+	req, err := s.newQueryRequest(query, ts)
 	if err != nil {
-		return Result{}, req, err
+		return Result{}, nil, err
+	}
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+			// Return unexpected error to the caller.
+			return Result{}, nil, err
+		}
+		// Something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req, err = s.newQueryRequest(query, ts)
+		if err != nil {
+			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
+		}
+		resp, err = s.do(ctx, req)
+		if err != nil {
+			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
+		}
 	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
 
+	// Process the received response.
 	parseFn := parsePrometheusResponse
 	if s.dataSourceType != datasourcePrometheus {
 		parseFn = parseGraphiteResponse
 	}
 	result, err := parseFn(req, resp)
+	_ = resp.Body.Close()
 	return result, req, err
 }
 
@@ -180,47 +182,68 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	req := s.newQueryRangeRequest(query, start, end)
-	resp, err := s.do(ctx, req)
-	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
-		// something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		req = s.newQueryRangeRequest(query, start, end)
-		resp, err = s.do(ctx, req)
-	}
+	req, err := s.newQueryRangeRequest(query, start, end)
 	if err != nil {
 		return res, err
 	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	return parsePrometheusResponse(req, resp)
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+			// Return unexpected error to the caller.
+			return res, err
+		}
+		// Something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req, err = s.newQueryRangeRequest(query, start, end)
+		if err != nil {
+			return res, fmt.Errorf("second attempt: %w", err)
+		}
+		resp, err = s.do(ctx, req)
+		if err != nil {
+			return res, fmt.Errorf("second attempt: %w", err)
+		}
+	}
+
+	// Process the received response.
+	res, err = parsePrometheusResponse(req, resp)
+	_ = resp.Body.Close()
+	return res, err
 }
 
 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	ru := req.URL.Redacted()
+	if *showDatasourceURL {
+		ru = req.URL.String()
+	}
 	if s.debug {
-		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
+		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru)
 	}
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
+		return nil, fmt.Errorf("error getting response from %s: %w", ru, err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
 	}
 	return resp, nil
 }
 
-func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) *http.Request {
-	req := s.newRequest()
+func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) (*http.Request, error) {
+	req, err := s.newRequest()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", s.datasourceURL, err)
+	}
 	s.setPrometheusRangeReqParams(req, query, start, end)
-	return req
+	return req, nil
 }
 
-func (s *VMStorage) newQueryRequest(query string, ts time.Time) *http.Request {
-	req := s.newRequest()
+func (s *VMStorage) newQueryRequest(query string, ts time.Time) (*http.Request, error) {
+	req, err := s.newRequest()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create query request to datasource %q: %w", s.datasourceURL, err)
+	}
 	switch s.dataSourceType {
 	case "", datasourcePrometheus:
 		s.setPrometheusInstantReqParams(req, query, ts)
@@ -229,20 +252,23 @@ func (s *VMStorage) newQueryRequest(query string, ts time.Time) *http.Request {
 	default:
 		logger.Panicf("BUG: engine not found: %q", s.dataSourceType)
 	}
-	return req
+	return req, nil
 }
 
-func (s *VMStorage) newRequest() *http.Request {
+func (s *VMStorage) newRequest() (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
-		s.authCfg.SetHeaders(req, true)
+		err = s.authCfg.SetHeaders(req, true)
+		if err != nil {
+			return nil, err
+		}
 	}
 	for _, h := range s.extraHeaders {
 		req.Header.Set(h.key, h.value)
 	}
-	return req
+	return req, nil
 }

app/vmalert/datasource/vm_prom_api.go

@@ -112,14 +112,14 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 		return res, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
-		return res, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
+		return res, fmt.Errorf("unknown status: %s, Expected success or error", r.Status)
 	}
 	var parseFn func() ([]Metric, error)
 	switch r.Data.ResultType {
 	case rtVector:
 		var pi promInstant
 		if err := json.Unmarshal(r.Data.Result, &pi.Result); err != nil {
-			return res, fmt.Errorf("umarshal err %s; \n %#v", err, string(r.Data.Result))
+			return res, fmt.Errorf("unmarshal err %w; \n %#v", err, string(r.Data.Result))
 		}
 		parseFn = pi.metrics
 	case rtMatrix:
@@ -161,8 +161,9 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
-
-	timestamp = s.adjustReqTimestamp(timestamp)
+	if s.lookBack > 0 {
+		timestamp = timestamp.Add(-s.lookBack)
+	}
 	q.Set("time", timestamp.Format(time.RFC3339))
 	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
@@ -186,9 +187,6 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
-	if s.evaluationOffset != nil {
-		start = start.Truncate(s.evaluationInterval).Add(*s.evaluationOffset)
-	}
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
 	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
@@ -213,30 +211,3 @@ func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q.Set("query", query)
 	r.URL.RawQuery = q.Encode()
 }
-
-func (s *VMStorage) adjustReqTimestamp(timestamp time.Time) time.Time {
-	if s.evaluationOffset != nil {
-		// calculate the min timestamp on the evaluationInterval
-		intervalStart := timestamp.Truncate(s.evaluationInterval)
-		ts := intervalStart.Add(*s.evaluationOffset)
-		if timestamp.Before(ts) {
-			// if passed timestamp is before the expected evaluation offset,
-			// then we should adjust it to the previous evaluation round.
-			// E.g. request with evaluationInterval=1h and evaluationOffset=30m
-			// was evaluated at 11:20. Then the timestamp should be adjusted
-			// to 10:30, to the previous evaluationInterval.
-			return ts.Add(-s.evaluationInterval)
-		}
-		// evaluationOffset shouldn't interfere with queryTimeAlignment or lookBack,
-		// so we return it immediately
-		return ts
-	}
-	if *queryTimeAlignment {
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
-		timestamp = timestamp.Truncate(s.evaluationInterval)
-	}
-	if s.lookBack > 0 {
-		timestamp = timestamp.Add(-s.lookBack)
-	}
-	return timestamp
-}

app/vmalert/datasource/vm_prom_api_test.go

@@ -3,7 +3,6 @@ package datasource
 import (
 	"encoding/json"
 	"testing"
-	"time"
 )
 
 func BenchmarkMetrics(b *testing.B) {
@@ -19,74 +18,3 @@ func BenchmarkMetrics(b *testing.B) {
 		}
 	})
 }
-
-func TestGetPrometheusReqTimestamp(t *testing.T) {
-	offset := 30 * time.Minute
-	testCases := []struct {
-		name               string
-		s                  *VMStorage
-		queryTimeAlignment bool
-		originTS, expTS    string
-	}{
-		{
-			"with eval_offset, find previous offset point",
-			&VMStorage{
-				evaluationOffset:   &offset,
-				evaluationInterval: time.Hour,
-				lookBack:           1 * time.Minute,
-			},
-			false,
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T10:30:00+00:00",
-		},
-		{
-			"with eval_offset",
-			&VMStorage{
-				evaluationOffset:   &offset,
-				evaluationInterval: time.Hour,
-			},
-			true,
-			"2023-08-28T11:41:00+00:00",
-			"2023-08-28T11:30:00+00:00",
-		},
-		{
-			"with query align",
-			&VMStorage{
-				evaluationInterval: time.Hour,
-			},
-			true,
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T11:00:00+00:00",
-		},
-		{
-			"with query align and lookback",
-			&VMStorage{
-				evaluationInterval: time.Hour,
-				lookBack:           1 * time.Minute,
-			},
-			true,
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T10:59:00+00:00",
-		},
-		{
-			"without query align",
-			&VMStorage{
-				evaluationInterval: time.Hour,
-			},
-			false,
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T11:11:00+00:00",
-		},
-	}
-	for _, tc := range testCases {
-		oldAlignPara := *queryTimeAlignment
-		*queryTimeAlignment = tc.queryTimeAlignment
-		originT, _ := time.Parse(time.RFC3339, tc.originTS)
-		expT, _ := time.Parse(time.RFC3339, tc.expTS)
-		gotTS := tc.s.adjustReqTimestamp(originT)
-		if !gotTS.Equal(expT) {
-			t.Fatalf("get wrong prometheus request timestamp, expect %s, got %s", expT, gotTS)
-		}
-		*queryTimeAlignment = oldAlignPara
-	}
-}

app/vmalert/datasource/vm_test.go

@@ -506,8 +506,7 @@ func TestRequestParams(t *testing.T) {
 			},
 			func(t *testing.T, r *http.Request) {
 				evalInterval := 15 * time.Second
-				tt := timestamp.Truncate(evalInterval)
-				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
@@ -521,7 +520,6 @@ func TestRequestParams(t *testing.T) {
 			func(t *testing.T, r *http.Request) {
 				evalInterval := 15 * time.Second
 				tt := timestamp.Add(-time.Minute)
-				tt = tt.Truncate(evalInterval)
 				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
@@ -549,8 +547,7 @@ func TestRequestParams(t *testing.T) {
 			},
 			func(t *testing.T, r *http.Request) {
 				evalInterval := 3 * time.Hour
-				tt := timestamp.Truncate(evalInterval)
-				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {tt.Format(time.RFC3339)}}
+				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
@@ -640,7 +637,10 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req := tc.vm.newRequest()
+			req, err := tc.vm.newRequest()
+			if err != nil {
+				t.Fatal(err)
+			}
 			switch tc.vm.dataSourceType {
 			case "", datasourcePrometheus:
 				if tc.queryRange {
@@ -735,7 +735,10 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req := vm.newQueryRequest("foo", time.Now())
+			req, err := vm.newQueryRequest("foo", time.Now())
+			if err != nil {
+				t.Fatal(err)
+			}
 			tt.checkFn(t, req)
 		})
 	}

app/vmalert/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -46,8 +47,8 @@ all files with prefix rule_ in folder dir.
 See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
 `)
 
-	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
-	for rules annotations templating. Flag can be specified multiple times.
+	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions `+
+		`for rules annotations templating. Flag can be specified multiple times.
 Examples:
  -rule.templates="/path/to/file". Path to a single file with go templates
  -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
@@ -58,7 +59,7 @@ absolute path to all .tpl files in root.
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -tls and -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -66,11 +67,6 @@ absolute path to all .tpl files in root.
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
-	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
-		"which by default is 4 times evaluationInterval of the parent group.")
-	resendDelay            = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
-	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
-		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
 
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier. By default, hostname is used as address.")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
@@ -82,12 +78,8 @@ absolute path to all .tpl files in root.
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
 
-	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
-		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
 
-	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
-
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
@@ -101,9 +93,9 @@ func main() {
 	remoteread.InitSecretFlags()
 	remotewrite.InitSecretFlags()
 	datasource.InitSecretFlags()
+	notifier.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if !*remoteReadIgnoreRestoreErrors {
 		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
@@ -125,9 +117,9 @@ func main() {
 		return
 	}
 
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	eu, err := getExternalURL(*externalURL)
 	if err != nil {
-		logger.Fatalf("failed to init `external.url`: %s", err)
+		logger.Fatalf("failed to init `-external.url`: %s", err)
 	}
 
 	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
@@ -189,8 +181,11 @@ func main() {
 	rh := &requestHandler{m: manager}
 	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
 
+	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
+	pushmetrics.Stop()
+
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
@@ -201,7 +196,7 @@ func main() {
 var (
 	configReloads      = metrics.NewCounter(`vmalert_config_last_reload_total`)
 	configReloadErrors = metrics.NewCounter(`vmalert_config_last_reload_errors_total`)
-	configSuccess      = metrics.NewCounter(`vmalert_config_last_reload_successful`)
+	configSuccess      = metrics.NewGauge(`vmalert_config_last_reload_successful`, nil)
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -228,7 +223,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
 	manager := &manager{
-		groups:         make(map[uint64]*Group),
+		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
 		notifiers:      nts,
 		labels:         labels,
@@ -237,7 +232,9 @@ func newManager(ctx context.Context) (*manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to init remoteWrite: %w", err)
 	}
-	manager.rw = rw
+	if rw != nil {
+		manager.rw = rw
+	}
 
 	rr, err := remoteread.Init()
 	if err != nil {
@@ -248,14 +245,26 @@ func newManager(ctx context.Context) (*manager, error) {
 	return manager, nil
 }
 
-func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
-	if externalURL != "" {
-		return url.Parse(externalURL)
+func getExternalURL(customURL string) (*url.URL, error) {
+	if customURL == "" {
+		// use local hostname as external URL
+		return getHostnameAsExternalURL(*httpListenAddr, httpserver.IsTLS())
 	}
-	hname, err := os.Hostname()
+	u, err := url.Parse(customURL)
 	if err != nil {
 		return nil, err
 	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, fmt.Errorf("invalid scheme %q in url %q, only 'http' and 'https' are supported", u.Scheme, u.String())
+	}
+	return u, nil
+}
+
+func getHostnameAsExternalURL(httpListenAddr string, isSecure bool) (*url.URL, error) {
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get hostname: %w", err)
+	}
 	port := ""
 	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
 		port = ":" + ipport[1]

app/vmalert/main_test.go

@@ -8,28 +8,43 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+func init() {
+	// Disable rand sleep on group start during tests in order to speed up test execution.
+	// Rand sleep is needed only in prod code.
+	rule.SkipRandSleepOnGroupStart = true
+}
+
 func TestGetExternalURL(t *testing.T) {
-	expURL := "https://vicotriametrics.com/path"
-	u, err := getExternalURL(expURL, "", false)
+	invalidURL := "victoriametrics.com/path"
+	_, err := getExternalURL(invalidURL)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+
+	expURL := "https://victoriametrics.com/path"
+	u, err := getExternalURL(expURL)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
 	}
+
 	h, _ := os.Hostname()
-	expURL = fmt.Sprintf("https://%s:4242", h)
-	u, err = getExternalURL("", "0.0.0.0:4242", true)
+	expURL = fmt.Sprintf("http://%s:8880", h)
+	u, err = getExternalURL("")
 	if err != nil {
 		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -98,10 +113,10 @@ groups:
 	ctx, cancel := context.WithCancel(context.Background())
 
 	m := &manager{
-		querierBuilder: &fakeQuerier{},
-		groups:         make(map[uint64]*Group),
+		querierBuilder: &datasource.FakeQuerier{},
+		groups:         make(map[uint64]*rule.Group),
 		labels:         map[string]string{},
-		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 		rw:             &remotewrite.Client{},
 	}
 
@@ -126,7 +141,7 @@ groups:
 				t.Fatalf("expected to have config error %s; got nil instead", cErr)
 			}
 			if cfgSuc != 0 {
-				t.Fatalf("expected to have metric configSuccess to be set to 0; got %d instead", cfgSuc)
+				t.Fatalf("expected to have metric configSuccess to be set to 0; got %v instead", cfgSuc)
 			}
 			return
 		}
@@ -135,7 +150,7 @@ groups:
 			t.Fatalf("unexpected config error: %s", cErr)
 		}
 		if cfgSuc != 1 {
-			t.Fatalf("expected to have metric configSuccess to be set to 1; got %d instead", cfgSuc)
+			t.Fatalf("expected to have metric configSuccess to be set to 1; got %v instead", cfgSuc)
 		}
 	}
 

app/vmalert/manager.go

@@ -3,14 +3,13 @@ package main
 import (
 	"context"
 	"fmt"
-	"net/url"
-	"sort"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -19,7 +18,7 @@ type manager struct {
 	querierBuilder datasource.QuerierBuilder
 	notifiers      func() []notifier.Notifier
 
-	rw *remotewrite.Client
+	rw remotewrite.RWClient
 	// remote read builder.
 	rr datasource.QuerierBuilder
 
@@ -27,28 +26,28 @@ type manager struct {
 	labels map[string]string
 
 	groupsMu sync.RWMutex
-	groups   map[uint64]*Group
+	groups   map[uint64]*rule.Group
 }
 
-// RuleAPI generates APIRule object from alert by its ID(hash)
-func (m *manager) RuleAPI(gID, rID uint64) (APIRule, error) {
+// ruleAPI generates apiRule object from alert by its ID(hash)
+func (m *manager) ruleAPI(gID, rID uint64) (apiRule, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
 	g, ok := m.groups[gID]
 	if !ok {
-		return APIRule{}, fmt.Errorf("can't find group with id %d", gID)
+		return apiRule{}, fmt.Errorf("can't find group with id %d", gID)
 	}
 	for _, rule := range g.Rules {
 		if rule.ID() == rID {
-			return rule.ToAPI(), nil
+			return ruleToAPI(rule), nil
 		}
 	}
-	return APIRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
+	return apiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
 }
 
-// AlertAPI generates APIAlert object from alert by its ID(hash)
-func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
+// alertAPI generates apiAlert object from alert by its ID(hash)
+func (m *manager) alertAPI(gID, aID uint64) (*apiAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
@@ -56,12 +55,12 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 	if !ok {
 		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
-	for _, rule := range g.Rules {
-		ar, ok := rule.(*AlertingRule)
+	for _, r := range g.Rules {
+		ar, ok := r.(*rule.AlertingRule)
 		if !ok {
 			continue
 		}
-		if apiAlert := ar.AlertAPI(aID); apiAlert != nil {
+		if apiAlert := alertToAPI(ar, aID); apiAlert != nil {
 			return apiAlert, nil
 		}
 	}
@@ -82,15 +81,15 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error {
+func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) error {
 	m.wg.Add(1)
 	id := g.ID()
 	go func() {
 		defer m.wg.Done()
 		if restore {
-			g.start(ctx, m.notifiers, m.rw, m.rr)
+			g.Start(ctx, m.notifiers, m.rw, m.rr)
 		} else {
-			g.start(ctx, m.notifiers, m.rw, nil)
+			g.Start(ctx, m.notifiers, m.rw, nil)
 		}
 	}()
 	m.groups[id] = g
@@ -99,7 +98,7 @@ func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error
 
 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
 	var rrPresent, arPresent bool
-	groupsRegistry := make(map[uint64]*Group)
+	groupsRegistry := make(map[uint64]*rule.Group)
 	for _, cfg := range groupsCfg {
 		for _, r := range cfg.Rules {
 			if rrPresent && arPresent {
@@ -112,7 +111,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 				arPresent = true
 			}
 		}
-		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
+		ng := rule.NewGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
@@ -124,8 +123,8 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	}
 
 	type updateItem struct {
-		old *Group
-		new *Group
+		old *rule.Group
+		new *rule.Group
 	}
 	var toUpdate []updateItem
 
@@ -135,7 +134,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		if !ok {
 			// old group is not present in new list,
 			// so must be stopped and deleted
-			og.close()
+			og.Close()
 			delete(m.groups, og.ID())
 			og = nil
 			continue
@@ -157,81 +156,13 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
 			wg.Add(1)
-			go func(old *Group, new *Group) {
-				old.updateCh <- new
+			go func(old *rule.Group, new *rule.Group) {
+				old.UpdateWith(new)
 				wg.Done()
 			}(item.old, item.new)
-			item.old.interruptEval()
+			item.old.InterruptEval()
 		}
 		wg.Wait()
 	}
 	return nil
 }
-
-func (g *Group) toAPI() APIGroup {
-	g.mu.RLock()
-	defer g.mu.RUnlock()
-
-	ag := APIGroup{
-		// encode as string to avoid rounding
-		ID: fmt.Sprintf("%d", g.ID()),
-
-		Name:            g.Name,
-		Type:            g.Type.String(),
-		File:            g.File,
-		Interval:        g.Interval.Seconds(),
-		LastEvaluation:  g.LastEvaluation,
-		Concurrency:     g.Concurrency,
-		Params:          urlValuesToStrings(g.Params),
-		Headers:         headersToStrings(g.Headers),
-		NotifierHeaders: headersToStrings(g.NotifierHeaders),
-
-		Labels: g.Labels,
-	}
-	ag.Rules = make([]APIRule, 0)
-	for _, r := range g.Rules {
-		ag.Rules = append(ag.Rules, r.ToAPI())
-	}
-	return ag
-}
-
-func urlValuesToStrings(values url.Values) []string {
-	if len(values) < 1 {
-		return nil
-	}
-
-	keys := make([]string, 0, len(values))
-	for k := range values {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	var res []string
-	for _, k := range keys {
-		params := values[k]
-		for _, v := range params {
-			res = append(res, fmt.Sprintf("%s=%s", k, v))
-		}
-	}
-	return res
-}
-
-func headersToStrings(headers map[string]string) []string {
-	if len(headers) < 1 {
-		return nil
-	}
-
-	keys := make([]string, 0, len(headers))
-	for k := range headers {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	var res []string
-	for _, k := range keys {
-		v := headers[k]
-		res = append(res, fmt.Sprintf("%s: %s", k, v))
-	}
-
-	return res
-}

app/vmalert/manager_test.go

@@ -10,8 +10,10 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 )
 
@@ -26,7 +28,7 @@ func TestMain(m *testing.M) {
 // successful cases of
 // starting with empty rules folder
 func TestManagerEmptyRulesDir(t *testing.T) {
-	m := &manager{groups: make(map[uint64]*Group)}
+	m := &manager{groups: make(map[uint64]*rule.Group)}
 	cfg := loadCfg(t, []string{"foo/bar"}, true, true)
 	if err := m.update(context.Background(), cfg, false); err != nil {
 		t.Fatalf("expected to load successfully with empty rules dir; got err instead: %v", err)
@@ -38,9 +40,9 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
-		groups:         make(map[uint64]*Group),
-		querierBuilder: &fakeQuerier{},
-		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		groups:         make(map[uint64]*rule.Group),
+		querierBuilder: &datasource.FakeQuerier{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -91,7 +93,7 @@ func TestManagerUpdate(t *testing.T) {
 	}()
 
 	var (
-		VMRows = &AlertingRule{
+		VMRows = &rule.AlertingRule{
 			Name: "VMRows",
 			Expr: "vm_rows > 0",
 			For:  10 * time.Second,
@@ -104,7 +106,7 @@ func TestManagerUpdate(t *testing.T) {
 				"description": "{{$labels}}",
 			},
 		}
-		Conns = &AlertingRule{
+		Conns = &rule.AlertingRule{
 			Name: "Conns",
 			Expr: "sum(vm_tcplistener_conns) by(instance) > 1",
 			Annotations: map[string]string{
@@ -112,7 +114,7 @@ func TestManagerUpdate(t *testing.T) {
 				"description": "It is {{ $value }} connections for {{$labels.instance}}",
 			},
 		}
-		ExampleAlertAlwaysFiring = &AlertingRule{
+		ExampleAlertAlwaysFiring = &rule.AlertingRule{
 			Name: "ExampleAlertAlwaysFiring",
 			Expr: "sum by(job) (up == 1)",
 		}
@@ -122,20 +124,20 @@ func TestManagerUpdate(t *testing.T) {
 		name       string
 		initPath   string
 		updatePath string
-		want       []*Group
+		want       []*rule.Group
 	}{
 		{
 			name:       "update good rules",
 			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules1-good.rules",
-			want: []*Group{
+			want: []*rule.Group{
 				{
 					File:     "config/testdata/dir/rules1-good.rules",
 					Name:     "duplicatedGroupDiffFiles",
 					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
-					Rules: []Rule{
-						&AlertingRule{
+					Rules: []rule.Rule{
+						&rule.AlertingRule{
 							Name:   "VMRows",
 							Expr:   "vm_rows > 0",
 							For:    5 * time.Minute,
@@ -153,19 +155,20 @@ func TestManagerUpdate(t *testing.T) {
 			name:       "update good rules from 1 to 2 groups",
 			initPath:   "config/testdata/dir/rules/rules1-good.rules",
 			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*Group{
+			want: []*rule.Group{
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     config.NewPrometheusType(),
-					Rules:    []Rule{VMRows},
 					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
 				},
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup", Rules: []Rule{
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
 					},
@@ -176,20 +179,20 @@ func TestManagerUpdate(t *testing.T) {
 			name:       "update with one bad rule file",
 			initPath:   "config/testdata/rules/rules0-good.rules",
 			updatePath: "config/testdata/dir/rules2-bad.rules",
-			want: []*Group{
+			want: []*rule.Group{
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
-					Rules:    []Rule{VMRows},
+					Rules:    []rule.Rule{VMRows},
 				},
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Name:     "TestGroup",
 					Type:     config.NewPrometheusType(),
-					Rules: []Rule{
+					Rules: []rule.Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
 					},
@@ -200,19 +203,20 @@ func TestManagerUpdate(t *testing.T) {
 			name:       "update empty dir rules from 0 to 2 groups",
 			initPath:   "config/testdata/empty/*",
 			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*Group{
+			want: []*rule.Group{
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Name:     "groupGorSingleAlert",
 					Type:     config.NewPrometheusType(),
 					Interval: defaultEvalInterval,
-					Rules:    []Rule{VMRows},
+					Rules:    []rule.Rule{VMRows},
 				},
 				{
 					File:     "config/testdata/rules/rules0-good.rules",
 					Interval: defaultEvalInterval,
 					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup", Rules: []Rule{
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
 					},
@@ -224,9 +228,9 @@ func TestManagerUpdate(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
 			m := &manager{
-				groups:         make(map[uint64]*Group),
-				querierBuilder: &fakeQuerier{},
-				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 			}
 
 			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
@@ -255,11 +259,36 @@ func TestManagerUpdate(t *testing.T) {
 		})
 	}
 }
+func compareGroups(t *testing.T, a, b *rule.Group) {
+	t.Helper()
+	if a.Name != b.Name {
+		t.Fatalf("expected group name %q; got %q", a.Name, b.Name)
+	}
+	if a.File != b.File {
+		t.Fatalf("expected group %q file name %q; got %q", a.Name, a.File, b.File)
+	}
+	if a.Interval != b.Interval {
+		t.Fatalf("expected group %q interval %v; got %v", a.Name, a.Interval, b.Interval)
+	}
+	if len(a.Rules) != len(b.Rules) {
+		t.Fatalf("expected group %s to have %d rules; got: %d",
+			a.Name, len(a.Rules), len(b.Rules))
+	}
+	for i, r := range a.Rules {
+		got, want := r, b.Rules[i]
+		if a.ID() != b.ID() {
+			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+		}
+		if err := rule.CompareRules(t, want, got); err != nil {
+			t.Fatalf("comparison error: %s", err)
+		}
+	}
+}
 
 func TestManagerUpdateNegative(t *testing.T) {
 	testCases := []struct {
 		notifiers []notifier.Notifier
-		rw        *remotewrite.Client
+		rw        remotewrite.RWClient
 		cfg       config.Group
 		expErr    string
 	}{
@@ -286,7 +315,7 @@ func TestManagerUpdateNegative(t *testing.T) {
 			"contains alerting rules",
 		},
 		{
-			[]notifier.Notifier{&fakeNotifier{}},
+			[]notifier.Notifier{&notifier.FakeNotifier{}},
 			nil,
 			config.Group{
 				Name: "Recording and alerting rules",
@@ -316,8 +345,8 @@ func TestManagerUpdateNegative(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.cfg.Name, func(t *testing.T) {
 			m := &manager{
-				groups:         make(map[uint64]*Group),
-				querierBuilder: &fakeQuerier{},
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
 				rw:             tc.rw,
 			}
 			if tc.notifiers != nil {
@@ -346,21 +375,3 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	}
 	return cfg
 }
-
-func TestUrlValuesToStrings(t *testing.T) {
-	mapQueryParams := map[string][]string{
-		"param1": {"param1"},
-		"param2": {"anotherparam"},
-	}
-	expectedRes := []string{"param1=param1", "param2=anotherparam"}
-	res := urlValuesToStrings(mapQueryParams)
-
-	if len(res) != len(expectedRes) {
-		t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
-	}
-	for ind, val := range expectedRes {
-		if val != res[ind] {
-			t.Errorf("Expected %v; but got %v", val, res[ind])
-		}
-	}
-}

app/vmalert/notifier/alert.go

@@ -191,7 +191,7 @@ func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarsh
 	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
 		labels = append(labels, prompbmarshal.Label{
-			Name:  k,
+			Name:  promrelabel.SanitizeMetricName(k),
 			Value: v,
 		})
 	}

app/vmalert/notifier/alert_test.go

@@ -237,6 +237,11 @@ func TestAlert_toPromLabels(t *testing.T) {
 		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
 		nil,
 	)
+	fn(
+		map[string]string{"foo.bar": "baz", "service!name": "qux"},
+		[]prompbmarshal.Label{{Name: "foo_bar", Value: "baz"}, {Name: "service_name", Value: "qux"}},
+		nil,
+	)
 
 	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
 - target_label: "foo"

app/vmalert/notifier/alertmanager.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
@@ -17,7 +18,7 @@ import (
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	addr    string
+	addr    *url.URL
 	argFunc AlertURLGenerator
 	client  *http.Client
 	timeout time.Duration
@@ -48,7 +49,12 @@ func (am *AlertManager) Close() {
 }
 
 // Addr returns address where alerts are sent.
-func (am AlertManager) Addr() string { return am.addr }
+func (am AlertManager) Addr() string {
+	if *showNotifierURL {
+		return am.addr.String()
+	}
+	return am.addr.Redacted()
+}
 
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
@@ -64,7 +70,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
-	req, err := http.NewRequest(http.MethodPost, am.addr, b)
+	req, err := http.NewRequest(http.MethodPost, am.addr.String(), b)
 	if err != nil {
 		return err
 	}
@@ -82,7 +88,10 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 	req = req.WithContext(ctx)
 
 	if am.authCfg != nil {
-		am.authCfg.SetHeaders(req, true)
+		err = am.authCfg.SetHeaders(req, true)
+		if err != nil {
+			return err
+		}
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -91,12 +100,16 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 
 	defer func() { _ = resp.Body.Close() }()
 
+	amURL := am.addr.Redacted()
+	if *showNotifierURL {
+		amURL = am.addr.String()
+	}
 	if resp.StatusCode != http.StatusOK {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
+			return fmt.Errorf("failed to read response from %q: %w", amURL, err)
 		}
-		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body))
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, amURL, string(body))
 	}
 	return nil
 }
@@ -131,13 +144,20 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
+	amURL, err := url.Parse(alertManagerURL)
+	if err != nil {
+		return nil, fmt.Errorf("provided incorrect notifier url: %w", err)
+	}
+	if !*showNotifierURL {
+		alertManagerURL = amURL.Redacted()
+	}
 	return &AlertManager{
-		addr:           alertManagerURL,
+		addr:           amURL,
 		argFunc:        fn,
 		authCfg:        aCfg,
 		relabelConfigs: relabelCfg,

app/vmalert/notifier/config.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"crypto/md5"
 	"fmt"
-	"gopkg.in/yaml.v2"
 	"net/url"
 	"os"
 	"path"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
@@ -142,26 +143,23 @@ func parseLabels(target string, metaLabels *promutils.Labels, cfg *Config) (stri
 	if labels.Len() == 0 {
 		return "", nil, nil
 	}
-	schemeRelabeled := labels.Get("__scheme__")
-	if len(schemeRelabeled) == 0 {
-		schemeRelabeled = "http"
+	scheme := labels.Get("__scheme__")
+	if len(scheme) == 0 {
+		scheme = "http"
 	}
-	addressRelabeled := labels.Get("__address__")
-	if len(addressRelabeled) == 0 {
+	alertsPath := labels.Get("__alerts_path__")
+	if !strings.HasPrefix(alertsPath, "/") {
+		alertsPath = "/" + alertsPath
+	}
+	address := labels.Get("__address__")
+	if len(address) == 0 {
 		return "", nil, nil
 	}
-	if strings.Contains(addressRelabeled, "/") {
-		return "", nil, nil
-	}
-	addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled)
-	alertsPathRelabeled := labels.Get("__alerts_path__")
-	if !strings.HasPrefix(alertsPathRelabeled, "/") {
-		alertsPathRelabeled = "/" + alertsPathRelabeled
-	}
-	u := fmt.Sprintf("%s://%s%s", schemeRelabeled, addressRelabeled, alertsPathRelabeled)
+	address = addMissingPort(scheme, address)
+	u := fmt.Sprintf("%s://%s%s", scheme, address, alertsPath)
 	if _, err := url.Parse(u); err != nil {
 		return "", nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q, metrics_path=%q (%q): %w",
-			u, cfg.Scheme, schemeRelabeled, target, addressRelabeled, alertsPathRelabeled, err)
+			u, cfg.Scheme, scheme, target, address, alertsPath, err)
 	}
 	return u, labels, nil
 }
@@ -181,9 +179,24 @@ func addMissingPort(scheme, target string) string {
 func mergeLabels(target string, metaLabels *promutils.Labels, cfg *Config) *promutils.Labels {
 	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
 	m := promutils.NewLabels(3 + metaLabels.Len())
-	m.Add("__address__", target)
-	m.Add("__scheme__", cfg.Scheme)
-	m.Add("__alerts_path__", path.Join("/", cfg.PathPrefix, alertManagerPath))
+	address := target
+	scheme := cfg.Scheme
+	alertsPath := path.Join("/", cfg.PathPrefix, alertManagerPath)
+	// try to extract optional scheme and alertsPath from __address__.
+	if strings.HasPrefix(address, "http://") {
+		scheme = "http"
+		address = address[len("http://"):]
+	} else if strings.HasPrefix(address, "https://") {
+		scheme = "https"
+		address = address[len("https://"):]
+	}
+	if n := strings.IndexByte(address, '/'); n >= 0 {
+		alertsPath = address[n:]
+		address = address[:n]
+	}
+	m.Add("__address__", address)
+	m.Add("__scheme__", scheme)
+	m.Add("__alerts_path__", alertsPath)
 	m.AddFrom(metaLabels)
 	return m
 }

app/vmalert/notifier/config_watcher.go

@@ -87,7 +87,7 @@ func (cw *configWatcher) reload(path string) error {
 func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
 	targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
 	for _, err := range errors {
-		return fmt.Errorf("failed to init notifier for %q: %s", typeK, err)
+		return fmt.Errorf("failed to init notifier for %q: %w", typeK, err)
 	}
 
 	cw.setTargets(typeK, targets)
@@ -107,7 +107,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn
 			}
 			updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
 			for _, err := range errors {
-				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
 			}
 			cw.setTargets(typeK, updateTargets)
 		}
@@ -118,7 +118,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn
 func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
 	metaLabels, err := labelsFn()
 	if err != nil {
-		return nil, []error{fmt.Errorf("failed to get labels: %s", err)}
+		return nil, []error{fmt.Errorf("failed to get labels: %w", err)}
 	}
 	var targets []Target
 	var errors []error
@@ -167,11 +167,11 @@ func (cw *configWatcher) start() error {
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
-					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
+					return fmt.Errorf("failed to parse labels for target %q: %w", target, err)
 				}
 				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
-					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
+					return fmt.Errorf("failed to init alertmanager for addr %q: %w", address, err)
 				}
 				targets = append(targets, Target{
 					Notifier: notifier,
@@ -189,14 +189,14 @@ func (cw *configWatcher) start() error {
 				sdc := &cw.cfg.ConsulSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, fmt.Errorf("got labels err: %s", err)
+					return nil, fmt.Errorf("got labels err: %w", err)
 				}
 				labels = append(labels, targetLabels...)
 			}
 			return labels, nil
 		})
 		if err != nil {
-			return fmt.Errorf("failed to start consulSD discovery: %s", err)
+			return fmt.Errorf("failed to start consulSD discovery: %w", err)
 		}
 	}
 
@@ -207,14 +207,14 @@ func (cw *configWatcher) start() error {
 				sdc := &cw.cfg.DNSSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, fmt.Errorf("got labels err: %s", err)
+					return nil, fmt.Errorf("got labels err: %w", err)
 				}
 				labels = append(labels, targetLabels...)
 			}
 			return labels, nil
 		})
 		if err != nil {
-			return fmt.Errorf("failed to start DNSSD discovery: %s", err)
+			return fmt.Errorf("failed to start DNSSD discovery: %w", err)
 		}
 	}
 	return nil

app/vmalert/notifier/config_watcher_test.go

@@ -318,3 +318,47 @@ func TestMergeHTTPClientConfigs(t *testing.T) {
 		t.Fatalf("expected BasicAuth tp be present")
 	}
 }
+
+func TestParseLabels(t *testing.T) {
+	testCases := []struct {
+		name            string
+		target          string
+		cfg             *Config
+		expectedAddress string
+		expectedErr     bool
+	}{
+		{
+			"invalid address",
+			"invalid:*//url",
+			&Config{},
+			"",
+			true,
+		},
+		{
+			"use some default params",
+			"alertmanager:9093",
+			&Config{PathPrefix: "test"},
+			"http://alertmanager:9093/test/api/v2/alerts",
+			false,
+		},
+		{
+			"use target address",
+			"https://alertmanager:9093/api/v1/alerts",
+			&Config{Scheme: "http", PathPrefix: "test"},
+			"https://alertmanager:9093/api/v1/alerts",
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			address, _, err := parseLabels(tc.target, nil, tc.cfg)
+			if err == nil == tc.expectedErr {
+				t.Fatalf("unexpected error; got %t; want %t", err != nil, tc.expectedErr)
+			}
+			if address != tc.expectedAddress {
+				t.Fatalf("unexpected address; got %q; want %q", address, tc.expectedAddress)
+			}
+		})
+	}
+}