docs/CHANGELOG.md: cut v1.76.1

This is a follow-up for c4d2cd8336

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2409

.dockerignore

@@ -4,3 +4,4 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
+.vscode

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,10 +8,10 @@ assignees: ''
 
 **Describe the bug**
 A clear and concise description of what the bug is.
-It would be a great [upgrading](https://docs.victoriametrics.com/#how-to-upgrade) 
+It would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
 to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-and verifying whether the bug is reproducible there.
-It is also recommended reading [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
+and verify whether the bug is reproducible there.
+It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
 
 **To Reproduce**
 Steps to reproduce the behavior.
@@ -36,12 +36,11 @@ See how to setup monitoring here:
 * [montioring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
 
 **Version**
-The line returned when passing `--version` command line flag to binary. For example:
+The line returned when passing `--version` command line flag to the binary. For example:
 ```
 $ ./victoria-metrics-prod --version
 victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
 ```
 
 **Used command-line flags**
-Please provide applied command-line flags used for running VictoriaMetrics and its components. 
-
+Please provide the command-line flags used for running VictoriaMetrics and its components. 

.github/dependabot.yml

@@ -7,7 +7,7 @@ updates:
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
   - package-ecosystem: "bundler"
     directory: "/docs"
     schedule:
@@ -15,12 +15,14 @@ updates:
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "daily"
   - package-ecosystem: "npm"
-    directory: "/app/vmui"
+    directory: "/app/vmui/packages/vmui"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+    open-pull-requests-limit: 0

.github/workflows/check-licenses.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/main.yml

@@ -16,15 +16,15 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
-      - name: Dependencies
-        run: |
-          go get -u golang.org/x/lint/golint
-          go get -u github.com/kisielk/errcheck
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.29.0
       - name: Code checkout
         uses: actions/checkout@master
+      - name: Dependencies
+        run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
       - name: Build
         env:
           GO111MODULE: on
@@ -60,7 +60,7 @@ jobs:
           GOOS=darwin go build -mod=vendor ./app/vmctl
           CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
       - name: Publish coverage
-        uses: codecov/codecov-action@v2.0.2
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
 

.gitignore

@@ -4,6 +4,7 @@
 *.pprof
 /bin
 .idea
+.vscode
 *.test
 *.swp
 /gocache-for-docker

CODE_OF_CONDUCT.md

@@ -68,9 +68,9 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+available at <https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>
 
 [homepage]: https://www.contributor-covenant.org
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+<https://www.contributor-covenant.org/faq>

CODE_OF_CONDUCT_RU.md

@@ -79,7 +79,7 @@
 
 **Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
 На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие 
+включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
 в публичных пространствах, а так же во внешних каналах,
 таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
@@ -89,10 +89,10 @@
 **Общественное влияние**: Серьёзное нарушение стандартов сообщества,
 включая продолжительное неуместное поведение.
 
-**Последствия**: Временный запрет (бан) на любое взаимодействие 
+**Последствия**: Временный запрет (бан) на любое взаимодействие
 или публичное общение с сообществом на определенный период времени.
 На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие 
+вовлеченными в инцидент, включая незапрошенное взаимодействие
 с теми, кто обеспечивает соблюдение Кодекса.
 Нарушение этих правил влечет за собой вечный бан.
 
@@ -108,7 +108,7 @@
 
 Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
 версии 2.0, доступной по адресу
-https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
 
 Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
 enforcement ladder](https://github.com/mozilla/diversity).
@@ -116,5 +116,5 @@ enforcement ladder](https://github.com/mozilla/diversity).
 [homepage]: https://www.contributor-covenant.org
 
 Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-https://www.contributor-covenant.org/faq. Переводы доступны по адресу
-https://www.contributor-covenant.org/translations.
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2021 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -24,6 +24,8 @@ all: \
 
 include app/*/Makefile
 include deployment/*/Makefile
+include snap/local/Makefile
+
 
 clean:
 	rm -rf bin/*
@@ -84,9 +86,12 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
-release-snap:
-	snapcraft
-	snapcraft upload "victoriametrics_$(PKG_TAG)_multi.snap" --release beta,edge,candidate
+
+publish-release:
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
 
 release: \
 	release-victoria-metrics \
@@ -95,66 +100,82 @@ release: \
 release-victoria-metrics: \
 	release-victoria-metrics-amd64 \
 	release-victoria-metrics-arm \
-	release-victoria-metrics-arm64
+	release-victoria-metrics-arm64 \
+	release-victoria-metrics-darwin-amd64 \
+	release-victoria-metrics-darwin-arm64
 
 release-victoria-metrics-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=amd64 $(MAKE) release-victoria-metrics-generic
 
 release-victoria-metrics-arm:
-	GOARCH=arm $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm $(MAKE) release-victoria-metrics-generic
 
 release-victoria-metrics-arm64:
-	GOARCH=arm64 $(MAKE) release-victoria-metrics-generic
+	OSARCH=arm64 $(MAKE) release-victoria-metrics-generic
 
-release-victoria-metrics-generic: victoria-metrics-$(GOARCH)-prod
+release-victoria-metrics-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-victoria-metrics-generic
+
+release-victoria-metrics-generic: victoria-metrics-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+		&& sha256sum victoria-metrics-$(OSARCH)-$(PKG_TAG).tar.gz \
+			victoria-metrics-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > victoria-metrics-$(OSARCH)-$(PKG_TAG)_checksums.txt
 
 release-vmutils: \
 	release-vmutils-amd64 \
 	release-vmutils-arm64 \
 	release-vmutils-arm \
+	release-vmutils-darwin-amd64  \
+	release-vmutils-darwin-arm64 \
 	release-vmutils-windows-amd64
 
 release-vmutils-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-generic
+	OSARCH=amd64 $(MAKE) release-vmutils-generic
 
 release-vmutils-arm64:
-	GOARCH=arm64 $(MAKE) release-vmutils-generic
+	OSARCH=arm64 $(MAKE) release-vmutils-generic
 
 release-vmutils-arm:
-	GOARCH=arm $(MAKE) release-vmutils-generic
+	OSARCH=arm $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-amd64:
+	OSARCH=darwin-amd64 $(MAKE) release-vmutils-generic
+
+release-vmutils-darwin-arm64:
+	OSARCH=darwin-arm64 $(MAKE) release-vmutils-generic
 
 release-vmutils-windows-amd64:
 	GOARCH=amd64 $(MAKE) release-vmutils-windows-generic
 
 release-vmutils-generic: \
-	vmagent-$(GOARCH)-prod \
-	vmalert-$(GOARCH)-prod \
-	vmauth-$(GOARCH)-prod \
-	vmbackup-$(GOARCH)-prod \
-	vmrestore-$(GOARCH)-prod \
-	vmctl-$(GOARCH)-prod
+	vmagent-$(OSARCH)-prod \
+	vmalert-$(OSARCH)-prod \
+	vmauth-$(OSARCH)-prod \
+	vmbackup-$(OSARCH)-prod \
+	vmrestore-$(OSARCH)-prod \
+	vmctl-$(OSARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOARCH)||" -czf vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOARCH)-prod \
-			vmalert-$(GOARCH)-prod \
-			vmauth-$(GOARCH)-prod \
-			vmbackup-$(GOARCH)-prod \
-			vmrestore-$(GOARCH)-prod \
-			vmctl-$(GOARCH)-prod \
-			| sed s/-$(GOARCH)-prod/-prod/ > vmutils-$(GOARCH)-$(PKG_TAG)_checksums.txt
+		tar --transform="flags=r;s|-$(OSARCH)||" -czf vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+		&& sha256sum vmutils-$(OSARCH)-$(PKG_TAG).tar.gz \
+			vmagent-$(OSARCH)-prod \
+			vmalert-$(OSARCH)-prod \
+			vmauth-$(OSARCH)-prod \
+			vmbackup-$(OSARCH)-prod \
+			vmrestore-$(OSARCH)-prod \
+			vmctl-$(OSARCH)-prod \
+			| sed s/-$(OSARCH)-prod/-prod/ > vmutils-$(OSARCH)-$(PKG_TAG)_checksums.txt
 
 release-vmutils-windows-generic: \
 	vmagent-windows-$(GOARCH)-prod \
@@ -174,6 +195,7 @@ release-vmutils-windows-generic: \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 
+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -261,7 +283,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.40.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.45.1
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

app/victoria-metrics/Makefile

@@ -27,6 +27,12 @@ victoria-metrics-ppc64le-prod:
 victoria-metrics-386-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
 
+victoria-metrics-darwin-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
+
+victoria-metrics-darwin-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-arm64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 

app/victoria-metrics/main.go

@@ -25,9 +25,9 @@ import (
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
 )
 
 func main() {
@@ -51,7 +51,7 @@ func main() {
 
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	storage.SetDedupInterval(*minScrapeInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -86,17 +86,20 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/vmui", "Web UI"},
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/metrics", "available service metrics"},
-			{"/api/v1/status/tsdb", "tsdb status page"},
-			{"/api/v1/status/top_queries", "top queries"},
-			{"/api/v1/status/active_queries", "active queries"},
+			{"vmui", "Web UI"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"api/v1/status/tsdb", "tsdb status page"},
+			{"api/v1/status/top_queries", "top queries"},
+			{"api/v1/status/active_queries", "active queries"},
 		})
 		return true
 	}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -19,6 +19,7 @@
 	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-100s}","3"],
 	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-40s}","2"],

app/vmagent/Makefile

@@ -27,6 +27,15 @@ vmagent-ppc64le-prod:
 vmagent-386-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-386
 
+vmagent-darwin-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
+
+vmagent-darwin-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
+
+vmagent-windows-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
+
 package-vmagent:
 	APP_NAME=vmagent $(MAKE) package-via-docker
 
@@ -81,6 +90,3 @@ vmagent-pure:
 
 vmagent-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-with-goarch
-
-vmagent-windows-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,99 @@
+package datadog
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
+		for _, tag := range ss.Tags {
+			n := strings.IndexByte(tag, ':')
+			if n < 0 {
+				return fmt.Errorf("cannot find ':' in tag %q", tag)
+			}
+			name := tag[:n]
+			value := tag[n+1:]
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -21,9 +21,10 @@ import (
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
+	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )
 
 var (
@@ -35,9 +36,9 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(r io.Reader) error {
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 			return insertRows(nil, db, rows, nil)
 		})
 	})
@@ -80,7 +81,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			if tag.Key == "db" {
+			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
 			commonLabels = append(commonLabels, prompbmarshal.Label{
@@ -90,7 +91,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 		}
 		if len(db) > 0 && !hasDBKey {
 			commonLabels = append(commonLabels, prompbmarshal.Label{
-				Name:  "db",
+				Name:  *dbLabel,
 				Value: db,
 			})
 		}

app/vmagent/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
@@ -41,16 +43,17 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
-		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
 var (
@@ -93,7 +96,9 @@ func main() {
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
+		})
 	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
@@ -149,14 +154,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if r.Method != "GET" {
 			return false
 		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/metrics", "available service metrics"},
-			{"/-/reload", "reload configuration"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}
@@ -221,13 +229,64 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/target_response":
+		promscrapeTargetResponseRequests.Inc()
+		if err := promscrape.WriteTargetResponse(w, r); err != nil {
+			promscrapeTargetResponseErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
@@ -327,6 +386,35 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -349,14 +437,26 @@ var (
 	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
 
 	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
+	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
+	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
+
+	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 

app/vmagent/native/request_handler.go

@@ -30,8 +30,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(block *parser.Block) error {
+		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
 			return insertRows(at, block, extraLabels)
 		})
 	})

app/vmagent/prometheusimport/request_handler.go

@@ -1,6 +1,7 @@
 package prometheusimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -38,6 +39,15 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader with optional gzip format
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		}, nil)
+	})
+}
+
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmagent/promremotewrite/request_handler.go

@@ -1,6 +1,7 @@
 package promremotewrite
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -29,12 +30,21 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
 			return insertRows(at, tss, extraLabels)
 		})
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, nil)
+		})
+	})
+}
+
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmagent/remotewrite/client.go

@@ -67,7 +67,8 @@ type client struct {
 	fq             *persistentqueue.FastQueue
 	hc             *http.Client
 
-	authCfg *promauth.Config
+	sendBlock func(block []byte) bool
+	authCfg   *promauth.Config
 
 	rl rateLimiter
 
@@ -84,16 +85,16 @@ type client struct {
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
 		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
 	}
 	tlsCfg := authCfg.NewTLSConfig()
 	tr := &http.Transport{
-		Dial:                statDial,
+		DialContext:         statDial,
 		TLSClientConfig:     tlsCfg,
-		TLSHandshakeTimeout: 5 * time.Second,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -104,11 +105,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		if !strings.Contains(pURL, "://") {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
 		}
-		urlProxy, err := url.Parse(pURL)
+		pu, err := url.Parse(pURL)
 		if err != nil {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
 		}
-		tr.Proxy = http.ProxyURL(urlProxy)
+		tr.Proxy = http.ProxyURL(pu)
 	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
@@ -121,6 +122,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		},
 		stopCh: make(chan struct{}),
 	}
+	c.sendBlock = c.sendBlockHTTP
+	return c
+}
+
+func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
@@ -143,7 +149,6 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		}()
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
-	return c
 }
 
 func (c *client) MustStop() {
@@ -160,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if username != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
-			Password:     password,
+			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
 	}
@@ -174,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if clientSecretFile != "" || clientSecret != "" {
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     clientSecret,
+			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
@@ -237,9 +242,9 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlock returns false only if c.stopCh is closed.
+// sendBlockHTTP returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlock(block []byte) bool {
+func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
 	retryDuration := time.Second
 	retriesCount := 0
@@ -249,7 +254,7 @@ func (c *client) sendBlock(block []byte) bool {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -290,6 +295,17 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
+		body, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+		if err != nil {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
+		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149

app/vmagent/remotewrite/pendingseries.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -20,16 +21,10 @@ import (
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
-		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
+	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 )
 
-// the maximum number of rows to send per each block.
-const maxRowsPerBlock = 10000
-
-// the maximum number of labels to send per each block.
-const maxLabelsPerBlock = 10 * maxRowsPerBlock
-
 type pendingSeries struct {
 	mu sync.Mutex
 	wr writeRequest
@@ -153,10 +148,13 @@ func (wr *writeRequest) adjustSampleValues() {
 
 func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
-		if len(wr.samples) >= maxRowsPerBlock || len(wr.labels) >= maxLabelsPerBlock {
+		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss
@@ -213,7 +211,22 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		writeRequestBufPool.Put(bb)
 	}
 
-	// Too big block. Recursively split it into smaller parts.
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
+		samples := wr.Timeseries[0].Samples
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return
+		}
+		n := len(samples) / 2
+		wr.Timeseries[0].Samples = samples[:n]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples[n:]
+		pushWriteRequest(wr, pushBlock)
+		wr.Timeseries[0].Samples = samples
+		return
+	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]

app/vmagent/remotewrite/relabel.go

@@ -15,12 +15,14 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
-	relabelDebug       = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
 )

app/vmagent/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -53,9 +54,9 @@ var (
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
 	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See also -remoteWrite.maxDailySeries")
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -remoteWrite.maxHourlySeries")
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 )
 
 var (
@@ -170,28 +171,32 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
 
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / maxRowsPerBlock / 100
-	if maxInmemoryBlocks > 400 {
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
 		// There is no much sense in keeping higher number of blocks in memory,
 		// since this means that the producer outperforms consumer and the queue
 		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 400
+		maxInmemoryBlocks = 100 * *queues
 	}
 	if maxInmemoryBlocks < 2 {
 		maxInmemoryBlocks = 2
 	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
-	for i, remoteWriteURL := range urls {
+	for i, remoteWriteURLRaw := range urls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
 		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if at != nil {
 			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
-			remoteWriteURL = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL, at.AccountID, at.ProjectID)
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
 			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
 		}
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
 	return rwctxs
 }
@@ -218,6 +223,13 @@ func Stop() {
 		}
 	}
 	rwctxsMap = nil
+
+	if sl := hourlySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
+	if sl := dailySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
 }
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
@@ -262,6 +274,9 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -271,7 +286,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			samplesCount += len(tss[i].Samples)
 			labelsCount += len(tss[i].Labels)
 			i++
-			if samplesCount >= maxRowsPerBlock || labelsCount >= maxLabelsPerBlock {
+			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
 			}
 		}
@@ -289,11 +304,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if len(tssBlock) > 0 {
-			for _, rwctx := range rwctxs {
-				rwctx.Push(tssBlock)
-			}
-		}
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
 		if rctx != nil {
 			rctx.reset()
 		}
@@ -303,6 +314,23 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	}
 }
 
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+	if len(tssBlock) == 0 {
+		// Nothing to push
+		return
+	}
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	for _, rwctx := range rwctxs {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			rwctx.Push(tssBlock)
+		}(rwctx)
+	}
+	wg.Wait()
+}
+
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
 func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
 	if !*sortLabels {
@@ -362,6 +390,8 @@ var labelsHashBufPool bytesutil.ByteBufferPool
 func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
+		// Do not use logger.WithThrottler() here, since this will increase CPU usage
+		// because every call to logSkippedSeries will result to a call to labelsToString.
 		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
 	default:
 	}
@@ -396,17 +426,29 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
-	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+	// strip query params, otherwise changing params resets pq
+	pqURL := *remoteWriteURL
+	pqURL.RawQuery = ""
+	pqURL.Fragment = ""
+	h := xxhash.Sum64([]byte(pqURL.String()))
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
+	var c *client
+	switch remoteWriteURL.Scheme {
+	case "http", "https":
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+	default:
+		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+	}
+	c.init(argIdx, *queues, sanitizedURL)
+
 	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
 	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
@@ -425,7 +467,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 

app/vmagent/remotewrite/statconn.go

@@ -1,7 +1,9 @@
 package remotewrite
 
 import (
+	"context"
 	"net"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -9,9 +11,26 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func statDial(networkUnused, addr string) (conn net.Conn, err error) {
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
-	conn, err = net.DialTimeout(network, addr, 5*time.Second)
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
 	dialsTotal.Inc()
 	if err != nil {
 		dialErrors.Inc()

app/vmagent/vmimport/request_handler.go

@@ -1,6 +1,7 @@
 package vmimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -31,12 +32,22 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
 			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		})
+	})
+}
+
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmalert/Makefile

@@ -27,6 +27,15 @@ vmalert-ppc64le-prod:
 vmalert-386-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-386
 
+vmalert-darwin-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
+
+vmalert-darwin-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
+
+vmalert-windows-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
+
 package-vmalert:
 	APP_NAME=vmalert $(MAKE) package-via-docker
 
@@ -56,6 +65,7 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
+	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
@@ -69,6 +79,13 @@ run-vmalert: vmalert
 		-evaluationInterval=3s \
 		-rule.configCheckInterval=10s
 
+run-vmalert-sd: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/consul.good.yaml \
+		-configCheckInterval=10s
+
 replay-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
@@ -101,6 +118,3 @@ vmalert-pure:
 
 vmalert-windows-amd64:
 	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-with-goarch
-
-vmalert-windows-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64

app/vmalert/alerting.go

@@ -12,9 +12,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // AlertingRule is basic alert entity
@@ -38,6 +38,8 @@ type AlertingRule struct {
 	alerts map[uint64]*notifier.Alert
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -50,15 +52,15 @@ type AlertingRule struct {
 }
 
 type alertingRuleMetrics struct {
-	errors  *gauge
-	pending *gauge
-	active  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	pending *utils.Gauge
+	active  *utils.Gauge
+	samples *utils.Gauge
 }
 
 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
-		Type:         cfg.Type,
+		Type:         group.Type,
 		RuleID:       cfg.ID,
 		Name:         cfg.Alert,
 		Expr:         cfg.Expr,
@@ -69,16 +71,16 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
 		metrics: &alertingRuleMetrics{},
 	}
 
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
-	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -90,7 +92,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -102,7 +104,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -111,7 +113,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return 1
 		})
-	ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			ar.mu.RLock()
 			defer ar.mu.RUnlock()
@@ -122,10 +124,10 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 
 // Close unregisters rule metrics
 func (ar *AlertingRule) Close() {
-	metrics.UnregisterMetric(ar.metrics.active.name)
-	metrics.UnregisterMetric(ar.metrics.pending.name)
-	metrics.UnregisterMetric(ar.metrics.errors.name)
-	metrics.UnregisterMetric(ar.metrics.samples.name)
+	ar.metrics.active.Unregister()
+	ar.metrics.pending.Unregister()
+	ar.metrics.errors.Unregister()
+	ar.metrics.samples.Unregister()
 }
 
 // String implements Stringer interface
@@ -139,6 +141,53 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+type labelSet struct {
+	// origin labels from series
+	// used for templating
+	origin map[string]string
+	// processed labels with additional data
+	// used as Alert labels
+	processed map[string]string
+}
+
+// toLabels converts labels from given Metric
+// to labelSet which contains original and processed labels.
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn notifier.QueryFn) (*labelSet, error) {
+	ls := &labelSet{
+		origin:    make(map[string]string, len(m.Labels)),
+		processed: make(map[string]string),
+	}
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		ls.origin[l.Name] = l.Value
+		ls.processed[l.Name] = l.Value
+	}
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+		Labels: ls.origin,
+		Value:  m.Values[0],
+		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
+	}
+	for k, v := range extraLabels {
+		ls.processed[k] = v
+	}
+
+	// set additional labels to identify group and rule name
+	if ar.Name != "" {
+		ls.processed[alertNameLabel] = ar.Name
+	}
+	if !*disableAlertGroupLabel && ar.GroupName != "" {
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+	}
+	return ls, nil
+}
+
 // ExecRange executes alerting rule on the given time range similarly to Exec.
 // It doesn't update internal states of the Rule and meant to be used just
 // to get time series for backfilling.
@@ -153,18 +202,7 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(s, qFn, ar)
-		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
-		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			s.SetLabel(k, v)
-		}
-
-		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
 		}
@@ -178,16 +216,15 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 
 		// if alert with For > 0
 		prevT := time.Time{}
-		//activeAt := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
-				//activeAt = at
-				a.Start = at
-			} else if at.Sub(a.Start) >= ar.For {
+				a.ActiveAt = at
+			} else if at.Sub(a.ActiveAt) >= ar.For {
 				a.State = notifier.StateFiring
+				a.Start = at
 			}
 			prevT = at
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
@@ -196,15 +233,21 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 	return result, nil
 }
 
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequently repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := ar.q.Query(ctx, ar.Expr)
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
 	ar.mu.Lock()
 	defer ar.mu.Unlock()
 
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
 	ar.lastExecError = err
-	ar.lastExecTime = time.Now()
 	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
@@ -212,52 +255,55 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
-		if a.State == notifier.StateInactive {
+		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
 			delete(ar.alerts, h)
 		}
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
-		// extra labels could contain templates, so we expand them first
-		labels, err := expandLabels(m, qFn, ar)
+		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
 			return nil, fmt.Errorf("failed to expand labels: %s", err)
 		}
-		for k, v := range labels {
-			// apply extra labels to datasource
-			// so the hash key will be consistent on restore
-			m.SetLabel(k, v)
-		}
-		h := hash(m)
+		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
 			// conflicting with the metric labels
-			return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
+			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, ar.lastExecError
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
+			if a.State == notifier.StateInactive {
+				// alert could be in inactive state for resolvedRetention
+				// so when we again receive metrics for it - we switch it
+				// back to notifier.StatePending
+				a.State = notifier.StatePending
+				a.ActiveAt = ts
+			}
 			if a.Value != m.Values[0] {
 				// update Value field with latest value
 				a.Value = m.Values[0]
 				// and re-exec template since Value can be used
 				// in annotations
-				a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 				if err != nil {
 					return nil, err
 				}
 			}
 			continue
 		}
-		a, err := ar.newAlert(m, ar.lastExecTime, qFn)
+		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
 		if err != nil {
 			ar.lastExecError = err
 			return nil, fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = h
 		a.State = notifier.StatePending
+		a.ActiveAt = ts
 		ar.alerts[h] = a
 	}
 
@@ -271,28 +317,19 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 				delete(ar.alerts, h)
 				continue
 			}
-			a.State = notifier.StateInactive
+			if a.State == notifier.StateFiring {
+				a.State = notifier.StateInactive
+				a.ResolvedAt = ts
+			}
 			continue
 		}
-		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+		if a.State == notifier.StatePending && time.Since(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
+			a.Start = ts
 			alertsFired.Inc()
 		}
 	}
-	return ar.toTimeSeries(ar.lastExecTime.Unix()), nil
-}
-
-func expandLabels(m datasource.Metric, q notifier.QueryFn, ar *AlertingRule) (map[string]string, error) {
-	metricLabels := make(map[string]string)
-	for _, l := range m.Labels {
-		metricLabels[l.Name] = l.Value
-	}
-	tpl := notifier.AlertTplData{
-		Labels: metricLabels,
-		Value:  m.Values[0],
-		Expr:   ar.Expr,
-	}
-	return notifier.ExecTemplate(q, ar.Labels, tpl)
+	return ar.toTimeSeries(ts.Unix()), nil
 }
 
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
@@ -325,47 +362,43 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 }
 
 // TODO: consider hashing algorithm in VM
-func hash(m datasource.Metric) uint64 {
+func hash(labels map[string]string) uint64 {
 	hash := fnv.New64a()
-	labels := m.Labels
-	sort.Slice(labels, func(i, j int) bool {
-		return labels[i].Name < labels[j].Name
-	})
-	for _, l := range labels {
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
 		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
+		if k == "__name__" {
 			continue
 		}
-		hash.Write([]byte(l.Name))
-		hash.Write([]byte(l.Value))
+		name, value := k, labels[k]
+		hash.Write([]byte(name))
+		hash.Write([]byte(value))
 		hash.Write([]byte("\xff"))
 	}
 	return hash.Sum64()
 }
 
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
-	a := &notifier.Alert{
-		GroupID: ar.GroupID,
-		Name:    ar.Name,
-		Labels:  map[string]string{},
-		Value:   m.Values[0],
-		Start:   start,
-		Expr:    ar.Expr,
-	}
-	// label defined here to make override possible by
-	// time series labels.
-	if ar.GroupName != "" {
-		a.Labels[alertGroupNameLabel] = ar.GroupName
-	}
-	for _, l := range m.Labels {
-		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
-			continue
-		}
-		a.Labels[l.Name] = l.Value
-	}
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn notifier.QueryFn) (*notifier.Alert, error) {
 	var err error
-	a.Annotations, err = a.ExecTemplate(qFn, ar.Annotations)
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+	}
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
 	return a, err
 }
 
@@ -380,34 +413,54 @@ func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
 	return ar.newAlertAPI(*a)
 }
 
-// RuleAPI returns Rule representation in form
-// of APIAlertingRule
-func (ar *AlertingRule) RuleAPI() APIAlertingRule {
-	var lastErr string
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
 	if ar.lastExecError != nil {
-		lastErr = ar.lastExecError.Error()
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
 	}
-	return APIAlertingRule{
-		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", ar.ID()),
-		GroupID:     fmt.Sprintf("%d", ar.GroupID),
-		Type:        ar.Type.String(),
-		Name:        ar.Name,
-		Expression:  ar.Expr,
-		For:         ar.For.String(),
-		LastError:   lastErr,
-		LastSamples: ar.lastExecSamples,
-		LastExec:    ar.lastExecTime,
-		Labels:      ar.Labels,
-		Annotations: ar.Annotations,
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
 	}
+	return r
 }
 
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
 	var alerts []*APIAlert
 	ar.mu.RLock()
 	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
 		alerts = append(alerts, ar.newAlertAPI(*a))
 	}
 	ar.mu.RUnlock()
@@ -415,19 +468,25 @@ func (ar *AlertingRule) AlertsAPI() []*APIAlert {
 }
 
 func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
-	return &APIAlert{
+	aa := &APIAlert{
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", a.ID),
 		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
 
 		Name:        a.Name,
 		Expression:  ar.Expr,
 		Labels:      a.Labels,
 		Annotations: a.Annotations,
 		State:       a.State.String(),
-		ActiveAt:    a.Start,
-		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+		ActiveAt:    a.ActiveAt,
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
 }
 
 const (
@@ -442,44 +501,43 @@ const (
 	alertStateLabel = "alertstate"
 
 	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
 	alertGroupNameLabel = "alertgroup"
 )
 
-// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+// alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	tss = append(tss, alertToTimeSeries(a, timestamp))
 	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
 	return tss
 }
 
-func alertToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertMetricName
-	labels[alertNameLabel] = name
 	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
 // alertForToTimeSeries returns a timeseries that represents
 // state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	labels[alertNameLabel] = name
-	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
+	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written timeseries.
-// Restore restores only Start field. Field State will be always Pending and supposed
+// Restore restores the state of active alerts basing on previously written time series.
+// Restore restores only ActiveAt field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
 func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
@@ -487,7 +545,8 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		return fmt.Errorf("querier is nil")
 	}
 
-	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query) }
+	ts := time.Now()
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
 
 	// account for external labels in filter
 	var labelsFilter string
@@ -500,31 +559,61 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
 		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr)
+	qMetrics, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return err
 	}
 
 	for _, m := range qMetrics {
-		labels := m.Labels
-		m.Labels = make([]datasource.Label, 0)
-		// drop all extra labels, so hash key will
-		// be identical to time series received in Exec
-		for _, l := range labels {
-			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
+		ls := &labelSet{
+			origin:    make(map[string]string, len(m.Labels)),
+			processed: make(map[string]string, len(m.Labels)),
+		}
+		for _, l := range m.Labels {
+			if l.Name == "__name__" {
 				continue
 			}
-			m.Labels = append(m.Labels, l)
+			ls.origin[l.Name] = l.Value
+			ls.processed[l.Name] = l.Value
 		}
-
-		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
+		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
-		a.ID = hash(m)
+		a.ID = hash(ls.processed)
 		a.State = notifier.StatePending
+		a.Restored = true
 		ar.alerts[a.ID] = a
-		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
 	return nil
 }
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	needsSending := func(a *notifier.Alert) bool {
+		if a.State == notifier.StatePending {
+			return false
+		}
+		if a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(ts)
+	}
+
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		if !needsSending(a) {
+			continue
+		}
+		a.End = ts.Add(resolveDuration)
+		if a.State == notifier.StateInactive {
+			a.End = a.ResolvedAt
+		}
+		a.LastSent = ts
+		alerts = append(alerts, *a)
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"reflect"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -27,7 +28,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant",
 				}),
 			},
 		},
@@ -41,7 +41,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant extra labels",
 					"job":           "foo",
 					"instance":      "bar",
 				}),
@@ -57,41 +56,36 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant labels override",
 				}),
 			},
 		},
 		{
 			newTestAlertingRule("for", time.Second),
-			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "for",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
 		{
 			newTestAlertingRule("for pending", 10*time.Second),
-			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StatePending, ActiveAt: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StatePending.String(),
-					alertNameLabel:  "for pending",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for pending",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
@@ -109,23 +103,27 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 
 func TestAlertingRule_Exec(t *testing.T) {
 	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
 	testCases := []struct {
 		rule      *AlertingRule
 		steps     [][]datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
+		expAlerts []testAlert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
 			[][]datasource.Metric{},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("empty labels", 0),
 			[][]datasource.Metric{
 				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 			},
-			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -133,8 +131,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -143,8 +141,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -154,8 +152,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -166,12 +164,12 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
-			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -179,7 +177,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -191,8 +191,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -204,10 +204,10 @@ func TestAlertingRule_Exec(t *testing.T) {
 					metricWithLabels(t, "name", "foo2"),
 				},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -219,10 +219,11 @@ func TestAlertingRule_Exec(t *testing.T) {
 			},
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
-			// 3: fire third alert, set second inactive, delete first one
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			// 3: fire third alert, set second inactive
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -230,8 +231,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -240,8 +241,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -252,7 +253,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset and delete pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
@@ -262,8 +263,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -275,8 +276,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -289,8 +290,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 	}
@@ -303,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -312,7 +313,18 @@ func TestAlertingRule_Exec(t *testing.T) {
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
 				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
 			}
-			for key, exp := range tc.expAlerts {
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := make(map[string]string)
+				for i := 0; i < len(ta.labels); i += 2 {
+					k, v := ta.labels[i], ta.labels[i+1]
+					labels[k] = v
+				}
+				labels[alertNameLabel] = tc.rule.Name
+				h := hash(labels)
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
 				got, ok := tc.rule.alerts[key]
 				if !ok {
 					t.Fatalf("expected to have key %d", key)
@@ -374,9 +386,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(3, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
 			},
 		},
 		{
@@ -385,9 +397,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
 		},
 		{
@@ -396,11 +408,11 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(5, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(5, 0)},
-				{State: notifier.StatePending, Start: time.Unix(20, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
 			},
 		},
 		{
@@ -412,15 +424,15 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StatePending, Start: time.Unix(1, 0)},
-				{State: notifier.StateFiring, Start: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 				//
-				{State: notifier.StatePending, Start: time.Unix(1, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
-				{State: notifier.StatePending, Start: time.Unix(5, 0),
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
 					Labels: map[string]string{
 						"foo": "bar",
 					}},
@@ -468,7 +480,12 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			var j int
 			for _, series := range tc.data {
 				for _, timestamp := range series.Timestamps {
-					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
+					expTS = append(expTS, tc.rule.alertToTimeSeries(a, timestamp)...)
 					j++
 				}
 			}
@@ -496,12 +513,11 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(nil): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -509,18 +525,20 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
+					alertNameLabel, "metric labels",
 					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					"foo", "bar",
-					"namespace", "baz",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					alertNameLabel:      "metric labels",
+					alertGroupNameLabel: "groupID",
+					"foo":               "bar",
+					"namespace":         "baz",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -528,7 +546,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
 					// extra labels set by rule
@@ -536,12 +553,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t,
-					"foo", "bar",
-					"namespace", "baz",
-					"source", "vm",
-				)): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{
+					"foo":       "bar",
+					"namespace": "baz",
+					"source":    "vm",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
 			},
 		},
 		{
@@ -561,12 +578,12 @@ func TestAlertingRule_Restore(t *testing.T) {
 				),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(2 * time.Hour)},
-				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
-					Start: time.Now().Truncate(3 * time.Hour)},
+				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(2 * time.Hour)},
+				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(3 * time.Hour)},
 			},
 		},
 	}
@@ -591,8 +608,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 				if got.State != exp.State {
 					t.Fatalf("expected state %d; got %d", exp.State, got.State)
 				}
-				if got.Start != exp.Start {
-					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				if got.ActiveAt != exp.ActiveAt {
+					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
 				}
 			}
 		})
@@ -607,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO())
+	_, err := ar.Exec(context.TODO(), time.Now())
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now())
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -623,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO())
+	_, err = ar.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -645,18 +662,20 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "foo",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "bar",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
 					},
 				},
 			},
@@ -666,37 +685,70 @@ func TestAlertingRule_Template(t *testing.T) {
 				Name: "override label",
 				Labels: map[string]string{
 					"instance": "{{ $labels.instance }}",
-					"region":   "east",
 				},
 				Annotations: map[string]string{
-					"summary":     `Too high connection number for "{{ $labels.instance }}" for region {{ $labels.region }}`,
-					"description": `It is {{ $value }} connections for "{{ $labels.instance }}"`,
+					"summary":     `Too high connection number for "{{ $labels.instance }}"`,
+					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
 			},
 			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "instance", "foo"),
-				metricWithValueAndLabels(t, 10, "instance", "bar"),
+				metricWithValueAndLabels(t, 2, "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
 					Labels: map[string]string{
-						"instance": "foo",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "foo",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "foo" for region east`,
-						"description": `It is 2 connections for "foo"`,
+						"summary":     `Too high connection number for "foo"`,
+						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
 					Labels: map[string]string{
-						"instance": "bar",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "bar",
 					},
 					Annotations: map[string]string{
-						"summary":     `Too high connection number for "bar" for region east`,
-						"description": `It is 10 connections for "bar"`,
+						"summary":     `Too high connection number for "bar"`,
+						"description": `override: It is 10 connections for "bar"`,
+					},
+				},
+			},
+		},
+		{
+			&AlertingRule{
+				Name:      "OriginLabels",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1,
+					alertNameLabel, "originAlertname",
+					alertGroupNameLabel, "originGroupname",
+					"instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo"}): {
+					Labels: map[string]string{
+						alertNameLabel:      "OriginLabels",
+						alertGroupNameLabel: "Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
 					},
 				},
 			},
@@ -709,7 +761,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -729,6 +781,86 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 }
 
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+}
+
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)

app/vmalert/config/config.go

@@ -5,17 +5,19 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
+	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // Group contains list of Rules grouped into
@@ -24,16 +26,22 @@ type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
 	Name        string             `yaml:"name"`
-	Interval    utils.PromDuration `yaml:"interval,omitempty"`
+	Interval    promutils.Duration `yaml:"interval"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+	// DEPRECATED: use Params field instead
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	// It has priority over the external labels.
+	Labels map[string]string `yaml:"labels"`
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -53,12 +61,20 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if g.Type.Get() == "" {
 		g.Type.Set(datasource.NewPrometheusType())
 	}
-	// update rules with empty type.
-	for i, r := range g.Rules {
-		if r.Type.Get() == "" {
-			r.Type.Set(g.Type)
-			r.ID = HashRule(r)
-			g.Rules[i] = r
+
+	// backward compatibility with deprecated `ExtraFilterLabels` param
+	if len(g.ExtraFilterLabels) > 0 {
+		if g.Params == nil {
+			g.Params = url.Values{}
+		}
+		// Sort extraFilters for consistent order for query args across runs.
+		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
+		for k, v := range g.ExtraFilterLabels {
+			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(extraFilters)
+		for _, extraFilter := range extraFilters {
+			g.Params.Add("extra_label", extraFilter)
 		}
 	}
 
@@ -73,9 +89,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
-	if len(g.Rules) == 0 {
-		return fmt.Errorf("group %q can't contain no rules", g.Name)
-	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -94,9 +107,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
-			if r.Type.Get() != "" {
-				exprValidator = r.Type.ValidateExpr
-			}
 			if err := exprValidator(r.Expr); err != nil {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
@@ -117,11 +127,10 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Type        datasource.Type    `yaml:"type,omitempty"`
 	Record      string             `yaml:"record,omitempty"`
 	Alert       string             `yaml:"alert,omitempty"`
 	Expr        string             `yaml:"expr"`
-	For         utils.PromDuration `yaml:"for"`
+	For         promutils.Duration `yaml:"for"`
 	Labels      map[string]string  `yaml:"labels,omitempty"`
 	Annotations map[string]string  `yaml:"annotations,omitempty"`
 
@@ -159,7 +168,6 @@ func HashRule(r Rule) uint64 {
 		h.Write([]byte("alerting"))
 		h.Write([]byte(r.Alert))
 	}
-	h.Write([]byte(r.Type.Get()))
 	kv := sortMap(r.Labels)
 	for _, i := range kv {
 		h.Write([]byte(i.key))
@@ -191,6 +199,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
+	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -210,6 +219,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
+			if len(g.ExtraFilterLabels) > 0 {
+				isExtraFilterLabelsUsed = true
+			}
 			groups = append(groups, g)
 		}
 	}
@@ -219,6 +231,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
+	if isExtraFilterLabelsUsed {
+		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
+	}
 	return groups, nil
 }
 

app/vmalert/config/config_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"gopkg.in/yaml.v2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestMain(m *testing.M) {
@@ -95,10 +96,6 @@ func TestGroup_Validate(t *testing.T) {
 			group:  &Group{},
 			expErr: "group name must be set",
 		},
-		{
-			group:  &Group{Name: "test"},
-			expErr: "contain no rules",
-		},
 		{
 			group: &Group{Name: "test",
 				Rules: []Rule{
@@ -263,11 +260,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -279,11 +275,10 @@ func TestGroup_Validate(t *testing.T) {
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						For:  utils.NewPromDuration(10 * time.Millisecond),
+						For:  promutils.NewDuration(10 * time.Millisecond),
 					},
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -347,7 +342,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: utils.NewPromDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -436,7 +431,7 @@ rules:
 `)
 	})
 
-	t.Run("Ok, `for` must change cs", func(t *testing.T) {
+	t.Run("`for` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup
 rules:
@@ -445,10 +440,118 @@ rules:
     for: 5m
 `, `
 name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`interval` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+interval: 2s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+interval: 4s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`concurrency` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+concurrency: 2
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+concurrency: 16
 rules:
   - alert: ExampleAlertWithFor
     expr: sum by(job) (up == 1)
 `)
 	})
 
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+}
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+
+	t.Run("extra labels", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+  env: prod
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+
+	t.Run("extra labels and params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+params:
+  nocache: ["1"]
+  extra_label: ["env=prod"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
+	})
 }

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -1,13 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 2s
-    concurrency: 2
-    type: prometheus
-    rules:
-      - alert: up
-        expr: up == 0
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules-update1-good.rules

@@ -1,12 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 30s
-    type: graphite
-    rules:
-      - alert: up
-        expr: filterSeries(time('host.2',20),'>','0')
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: duplicatedGroupDiffFiles
+    labels:
+       dc: gcp
     rules:
       - alert: VMRows
         for: 5m

app/vmalert/config/testdata/rules0-good.rules

@@ -1,5 +1,8 @@
 groups:
   - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
     rules:
       - alert: VMRows
         for: 10s

app/vmalert/config/testdata/rules2-good.rules

@@ -2,8 +2,11 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
-    extra_filter_labels:
+    extra_filter_labels: # deprecated param, use `params` instead
         job: victoriametrics
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by(instance) > 1
@@ -22,6 +25,7 @@ groups:
           dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}'
         annotations:
           description: Job {{ $labels.job }} is up!
+          external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }}
           summary: All instances up {{ range query "up" }}
             {{ . | label "instance" }}
             {{ end }}

app/vmalert/config/testdata/rules3-good.rules

@@ -21,10 +21,3 @@ groups:
         annotations:
           summary: Too high connection number for {{$labels.instance}}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-      - alert: HostDown
-        type: graphite
-        expr: filterSeries(sumSeries(host.receiver.interface.up),'last','=', 0)
-        for: 3m
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules4-good.rules

@@ -0,0 +1,8 @@
+groups:
+  - name: TestEmptyRules
+    interval: 2s
+    concurrency: 2
+    rules:
+
+  - name: TestNoRules
+    type: prometheus

app/vmalert/config/testdata/rules_interval_good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupTest
+    interval: 1s
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/datasource/datasource.go

@@ -2,12 +2,13 @@ package datasource
 
 import (
 	"context"
+	"net/url"
 	"time"
 )
 
 // Querier interface wraps Query and QueryRange methods
 type Querier interface {
-	Query(ctx context.Context, query string) ([]Metric, error)
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, error)
 	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
 }
 
@@ -20,8 +21,7 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     *Type
 	EvaluationInterval time.Duration
-	// see https://docs.victoriametrics.com/#prometheus-querying-api-enhancements
-	ExtraLabels map[string]string
+	QueryParams        url.Values
 }
 
 // Metric is the basic entity which should be return by datasource

app/vmalert/datasource/init.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -12,9 +13,14 @@ import (
 var (
 	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
 		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
-	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+
+	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
+
+	bearerToken     = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
@@ -22,17 +28,34 @@ var (
 	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
 	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
 
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
+	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
+	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+
 	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
 	queryStep = flag.Duration("datasource.queryStep", 0, "queryStep defines how far a value can fallback to when evaluating queries. "+
 		"For example, if datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query."+
 		"If queryStep isn't specified, rule's evaluationInterval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
-	roundDigits        = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
+// Param represents an HTTP GET param
+type Param struct {
+	Key, Value string
+}
+
 // Init creates a Querier from provided flag values.
-func Init() (QuerierBuilder, error) {
+// Provided extraParams will be added as GET params for
+// each request.
+func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
@@ -41,22 +64,35 @@ func Init() (QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
 
-	var rd string
+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
 	if *roundDigits > 0 {
-		rd = fmt.Sprintf("%d", *roundDigits)
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
+	}
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
 	return &VMStorage{
 		c:                &http.Client{Transport: tr},
-		basicAuthUser:    *basicAuthUsername,
-		basicAuthPass:    *basicAuthPassword,
+		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,
 		queryStep:        *queryStep,
-		roundDigits:      rd,
 		dataSourceType:   NewPrometheusType(),
+		extraParams:      extraParams,
 	}, nil
 }

app/vmalert/datasource/type.go

@@ -7,9 +7,6 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-const graphiteType = "graphite"
-const prometheusType = "prometheus"
-
 // Type represents data source type
 type Type struct {
 	name string
@@ -17,12 +14,16 @@ type Type struct {
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
-	return Type{name: prometheusType}
+	return Type{
+		name: "prometheus",
+	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
-	return Type{name: graphiteType}
+	return Type{
+		name: "graphite",
+	}
 }
 
 // NewRawType returns datasource type from raw string
@@ -44,19 +45,19 @@ func (t *Type) Set(d Type) {
 // String implements String interface with default value.
 func (t Type) String() string {
 	if t.name == "" {
-		return prometheusType
+		return "prometheus"
 	}
 	return t.name
 }
 
 // ValidateExpr validates query expression with datasource ql.
 func (t *Type) ValidateExpr(expr string) error {
-	switch t.name {
-	case graphiteType:
+	switch t.String() {
+	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
 			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
 		}
-	case "", prometheusType:
+	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
@@ -72,12 +73,13 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
+	if s == "" {
+		s = "prometheus"
+	}
 	switch s {
-	case "":
-		s = prometheusType
-	case graphiteType, prometheusType:
+	case "graphite", "prometheus":
 	default:
-		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, prometheusType, graphiteType)
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
 	t.name = s
 	return nil

app/vmalert/datasource/vm.go

@@ -5,37 +5,39 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
+	authCfg          *promauth.Config
 	datasourceURL    string
-	basicAuthUser    string
-	basicAuthPass    string
 	appendTypePrefix bool
 	lookBack         time.Duration
 	queryStep        time.Duration
-	roundDigits      string
 
 	dataSourceType     Type
 	evaluationInterval time.Duration
-	extraLabels        []string
+	extraParams        url.Values
+	disablePathAppend  bool
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                s.c,
-		datasourceURL:    s.datasourceURL,
-		basicAuthUser:    s.basicAuthUser,
-		basicAuthPass:    s.basicAuthPass,
-		lookBack:         s.lookBack,
-		queryStep:        s.queryStep,
-		appendTypePrefix: s.appendTypePrefix,
-		dataSourceType:   s.dataSourceType,
+		c:                 s.c,
+		authCfg:           s.authCfg,
+		datasourceURL:     s.datasourceURL,
+		lookBack:          s.lookBack,
+		queryStep:         s.queryStep,
+		appendTypePrefix:  s.appendTypePrefix,
+		dataSourceType:    s.dataSourceType,
+		disablePathAppend: s.disablePathAppend,
 	}
 }
 
@@ -45,9 +47,7 @@ func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 		s.dataSourceType = *params.DataSourceType
 	}
 	s.evaluationInterval = params.EvaluationInterval
-	for k, v := range params.ExtraLabels {
-		s.extraLabels = append(s.extraLabels, fmt.Sprintf("%s=%s", k, v))
-	}
+	s.extraParams = params.QueryParams
 	return s
 }
 
@@ -57,31 +57,30 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
 	return &VMStorage{
-		c:                c,
-		basicAuthUser:    basicAuthUser,
-		basicAuthPass:    basicAuthPass,
-		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix: appendTypePrefix,
-		lookBack:         lookBack,
-		queryStep:        queryStep,
-		dataSourceType:   NewPrometheusType(),
+		c:                 c,
+		authCfg:           authCfg,
+		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix:  appendTypePrefix,
+		lookBack:          lookBack,
+		queryStep:         queryStep,
+		dataSourceType:    NewPrometheusType(),
+		disablePathAppend: disablePathAppend,
 	}
 }
 
 // Query executes the given query and returns parsed response
-func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, error) {
 	req, err := s.newRequestPOST()
 	if err != nil {
 		return nil, err
 	}
 
-	ts := time.Now()
-	switch s.dataSourceType.name {
-	case "", prometheusType:
+	switch s.dataSourceType.String() {
+	case "prometheus":
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case graphiteType:
+	case "graphite":
 		s.setGraphiteReqParams(req, query, ts)
 	default:
 		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
@@ -96,7 +95,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}()
 
 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		parseFn = parseGraphiteResponse
 	}
 	return parseFn(req, resp)
@@ -106,7 +105,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
 	}
 	req, err := s.newRequestPOST()
@@ -133,12 +132,12 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return resp, nil
 }
@@ -148,9 +147,11 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
-	if s.basicAuthPass != "" {
-		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	req.Header.Set("Content-Type", "application/json")
+	if s.authCfg != nil {
+		if auth := s.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_graphite_api.go

@@ -38,7 +38,7 @@ func (r graphiteResponse) metrics() []Metric {
 func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &graphiteResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	return r.metrics(), nil
 }
@@ -54,6 +54,14 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("format", "json")
 	q.Set("target", query)
 	from := "-5min"

app/vmalert/datasource/vm_prom_api.go

@@ -82,10 +82,10 @@ const (
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &promResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	if r.Status == statusError {
-		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
@@ -118,12 +118,14 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusInstantPath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusInstantPath
+	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
 	}
-	if s.evaluationInterval > 0 {
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
@@ -136,7 +138,9 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusRangePath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusRangePath
+	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))
 	q.Add("end", fmt.Sprintf("%d", end.Unix()))
@@ -146,20 +150,24 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 
 func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("query", query)
-	if s.evaluationInterval > 0 {
-		// set step as evaluationInterval by default
-		q.Set("step", s.evaluationInterval.String())
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if s.queryStep > 0 {
-		// override step with user-specified value
-		q.Set("step", s.queryStep.String())
-	}
-	if s.roundDigits != "" {
-		q.Set("round_digits", s.roundDigits)
-	}
-	for _, l := range s.extraLabels {
-		q.Add("extra_label", l)
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
 }

app/vmalert/datasource/vm_test.go

@@ -5,19 +5,27 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
 	ctx           = context.Background()
 	basicAuthName = "foo"
 	basicAuthPass = "bar"
-	query         = "vm_rows"
-	queryRender   = "constantLine(10)"
+	baCfg         = &promauth.BasicAuthConfig{
+		Username: basicAuthName,
+		Password: promauth.NewSecret(basicAuthPass),
+	}
+	query       = "vm_rows"
+	queryRender = "constantLine(10)"
 )
 
 func TestVMInstantQuery(t *testing.T) {
@@ -73,30 +81,35 @@ func TestVMInstantQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
+	ts := time.Now()
 
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected invalid response status error got nil")
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected response body error got nil")
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected error status got nil")
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected unknown status got nil")
 	}
-	if _, err := pq.Query(ctx, query); err == nil {
+	if _, err := pq.Query(ctx, query, ts); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
 	}
-	m, err := pq.Query(ctx, query)
+	m, err := pq.Query(ctx, query, ts)
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -122,7 +135,7 @@ func TestVMInstantQuery(t *testing.T) {
 	g := NewGraphiteType()
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: &g})
 
-	m, err = gq.Query(ctx, queryRender)
+	m, err = gq.Query(ctx, queryRender, ts)
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -179,12 +192,16 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
 
-	_, err := pq.QueryRange(ctx, query, time.Now(), time.Time{})
+	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
 
 	_, err = pq.QueryRange(ctx, query, time.Time{}, time.Now())
@@ -216,6 +233,10 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
 	testCases := []struct {
@@ -234,6 +255,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus path with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus prefix",
 			false,
@@ -245,6 +277,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus prefix with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"prometheus range path",
 			true,
@@ -255,6 +299,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range path with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus range prefix",
 			true,
@@ -266,6 +321,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range prefix with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"graphite path",
 			false,
@@ -308,10 +375,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth",
 			false,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -321,10 +385,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth range",
 			true,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -377,15 +438,28 @@ func TestRequestParams(t *testing.T) {
 				queryStep: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, time.Minute, timestamp.Unix())
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"round digits",
+			"step to seconds",
 			false,
 			&VMStorage{
-				roundDigits: "10",
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
 			},
 			func(t *testing.T, r *http.Request) {
 				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
@@ -393,31 +467,32 @@ func TestRequestParams(t *testing.T) {
 			},
 		},
 		{
-			"extra labels",
-			false,
+			"prometheus extra params range",
+			true,
 			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&time=%d", query, timestamp.Unix())
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
+					timestamp.Unix(), query, timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"extra labels range",
-			true,
+			"graphite extra params",
+			false,
 			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
+				dataSourceType: NewGraphiteType(),
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&start=%d",
-					timestamp.Unix(), query, timestamp.Unix())
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
@@ -429,14 +504,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.name {
-			case "", prometheusType:
+			switch tc.vm.dataSourceType.String() {
+			case "prometheus":
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case graphiteType:
+			case "graphite":
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -444,10 +519,63 @@ func TestRequestParams(t *testing.T) {
 	}
 }
 
+func TestAuthConfig(t *testing.T) {
+	var testCases = []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
+}
+
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {
-		t.Errorf("expected to get %q; got %q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 

app/vmalert/group.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"net/url"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -12,21 +15,26 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu                sync.RWMutex
-	Name              string
-	File              string
-	Rules             []Rule
-	Type              datasource.Type
-	Interval          time.Duration
-	Concurrency       int
-	Checksum          string
-	ExtraFilterLabels map[string]string
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           datasource.Type
+	Interval       time.Duration
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
+
+	Labels map[string]string
+	Params url.Values
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -38,27 +46,47 @@ type Group struct {
 }
 
 type groupMetrics struct {
-	iterationTotal    *counter
-	iterationDuration *summary
+	iterationTotal    *utils.Counter
+	iterationDuration *utils.Summary
+	iterationMissed   *utils.Counter
 }
 
 func newGroupMetrics(name, file string) *groupMetrics {
 	m := &groupMetrics{}
 	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
-	m.iterationTotal = getOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = getOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
 	return m
 }
 
+// merges group rule labels into result map
+// set2 has priority over set1.
+func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
+	r := map[string]string{}
+	for k, v := range set1 {
+		r[k] = v
+	}
+	for k, v := range set2 {
+		if prevV, ok := r[k]; ok {
+			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+				k, prevV, groupName, ruleName, k, v)
+		}
+		r[k] = v
+	}
+	return r
+}
+
 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
-		Type:              cfg.Type,
-		Name:              cfg.Name,
-		File:              cfg.File,
-		Interval:          cfg.Interval.Duration(),
-		Concurrency:       cfg.Concurrency,
-		Checksum:          cfg.Checksum,
-		ExtraFilterLabels: cfg.ExtraFilterLabels,
+		Type:        cfg.Type,
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval.Duration(),
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		Params:      cfg.Params,
+		Labels:      cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -73,17 +101,20 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	}
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
-		// override rule labels with external labels
-		for k, v := range labels {
-			if prevV, ok := r.Labels[k]; ok {
-				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
-					k, prevV, g.Name, r.Name(), k, v)
-			}
-			if r.Labels == nil {
-				r.Labels = map[string]string{}
-			}
-			r.Labels[k] = v
+		var extraLabels map[string]string
+		// apply external labels
+		if len(labels) > 0 {
+			extraLabels = labels
 		}
+		// apply group labels, it has priority on external labels
+		if len(cfg.Labels) > 0 {
+			extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels)
+		}
+		// apply rules labels, it has priority on other labels
+		if len(extraLabels) > 0 {
+			r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels)
+		}
+
 		rules[i] = g.newRule(qb, r)
 	}
 	g.Rules = rules
@@ -98,8 +129,11 @@ func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
 }
 
 // ID return unique group ID that consists of
-// rules file and group name
+// rules file and group Name
 func (g *Group) ID() uint64 {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -110,6 +144,7 @@ func (g *Group) ID() uint64 {
 
 // Restore restores alerts state for group rules
 func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
+	labels = mergeLabels(g.Name, "", labels, g.Labels)
 	for _, rule := range g.Rules {
 		rr, ok := rule.(*AlertingRule)
 		if !ok {
@@ -166,20 +201,18 @@ func (g *Group) updateWith(newGroup *Group) error {
 	for _, nr := range rulesRegistry {
 		newRules = append(newRules, nr)
 	}
+	// note that g.Interval is not updated here
+	// so the value can be compared later in
+	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
-	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Params = newGroup.Params
+	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
 }
 
-var (
-	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
-	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
-	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
-)
-
 func (g *Group) close() {
 	if g.doneCh == nil {
 		return
@@ -187,8 +220,8 @@ func (g *Group) close() {
 	close(g.doneCh)
 	<-g.finishedCh
 
-	metrics.UnregisterMetric(g.metrics.iterationDuration.name)
-	metrics.UnregisterMetric(g.metrics.iterationTotal.name)
+	g.metrics.iterationDuration.Unregister()
+	g.metrics.iterationTotal.Unregister()
 	for _, rule := range g.Rules {
 		rule.Close()
 	}
@@ -196,12 +229,19 @@ func (g *Group) close() {
 
 var skipRandSleepOnGroupStart bool
 
-func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
 
+	e := &executor{
+		rw:                       rw,
+		notifiers:                nts,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
+
+	evalTS := time.Now()
+
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
 		if randSleep < sleepOffset {
 			randSleep += uint64(g.Interval)
@@ -220,7 +260,31 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 	}
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{nts, rw}
+
+	eval := func(ts time.Time) {
+		g.metrics.iterationTotal.Inc()
+
+		start := time.Now()
+
+		if len(g.Rules) < 1 {
+			g.metrics.iterationDuration.UpdateDuration(start)
+			g.LastEvaluation = start
+			return
+		}
+
+		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration)
+		for err := range errs {
+			if err != nil {
+				logger.Errorf("group %q: %s", g.Name, err)
+			}
+		}
+		g.metrics.iterationDuration.UpdateDuration(start)
+		g.LastEvaluation = start
+	}
+
+	eval(evalTS)
+
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -247,32 +311,48 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 			g.mu.Unlock()
 			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
-			g.metrics.iterationTotal.Inc()
-			iterationStart := time.Now()
-
-			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
-			for err := range errs {
-				if err != nil {
-					logger.Errorf("group %q: %s", g.Name, err)
-				}
+			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed > 0 {
+				g.metrics.iterationMissed.Inc()
 			}
+			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
-			g.metrics.iterationDuration.UpdateDuration(iterationStart)
+			eval(evalTS)
 		}
 	}
 }
 
-type executor struct {
-	notifiers []notifier.Notifier
-	rw        *remotewrite.Client
+// getResolveDuration returns the duration after which firing alert
+// can be considered as resolved.
+func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Duration {
+	if groupInterval > delta {
+		delta = groupInterval
+	}
+	resolveDuration := delta * 4
+	if maxDuration > 0 && resolveDuration > maxDuration {
+		resolveDuration = maxDuration
+	}
+	return resolveDuration
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
+type executor struct {
+	notifiers func() []notifier.Notifier
+	rw        *remotewrite.Client
+
+	previouslySentSeriesToRWMu sync.Mutex
+	// previouslySentSeriesToRW stores series sent to RW on previous iteration
+	// map[ruleID]map[ruleLabels][]prompb.Label
+	// where `ruleID` is ID of the Rule within a Group
+	// and `ruleLabels` is []prompb.Label marshalled to a string
+	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
+}
+
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, interval)
+			res <- e.exec(ctx, rule, ts, resolveDuration)
 		}
 		close(res)
 		return res
@@ -285,7 +365,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, interval)
+				res <- e.exec(ctx, r, ts, resolveDuration)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -297,66 +377,101 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 }
 
 var (
-	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
-	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
-	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+	alertsFired = metrics.NewCounter(`vmalert_alerts_fired_total`)
+
+	execTotal  = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
 
 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration) error {
 	execTotal.Inc()
-	execStart := time.Now()
-	defer func() {
-		execDuration.UpdateDuration(execStart)
-	}()
 
-	tss, err := rule.Exec(ctx)
+	tss, err := rule.Exec(ctx, ts)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
 	}
 
-	if len(tss) > 0 && e.rw != nil {
-		for _, ts := range tss {
-			if err := e.rw.Push(ts); err != nil {
-				remoteWriteErrors.Inc()
-				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
+	errGr := new(utils.ErrGroup)
+	if e.rw != nil {
+		pushToRW := func(tss []prompbmarshal.TimeSeries) {
+			for _, ts := range tss {
+				remoteWriteTotal.Inc()
+				if err := e.rw.Push(ts); err != nil {
+					remoteWriteErrors.Inc()
+					errGr.Add(fmt.Errorf("rule %q: remote write failure: %w", rule, err))
+				}
 			}
 		}
+		pushToRW(tss)
+		staleSeries := e.getStaleSeries(rule, tss, ts)
+		pushToRW(staleSeries)
 	}
 
 	ar, ok := rule.(*AlertingRule)
 	if !ok {
 		return nil
 	}
-	var alerts []notifier.Alert
-	for _, a := range ar.alerts {
-		switch a.State {
-		case notifier.StateFiring:
-			// set End to execStart + 3 intervals
-			// so notifier can resolve it automatically if `vmalert`
-			// won't be able to send resolve for some reason
-			a.End = time.Now().Add(3 * interval)
-			alerts = append(alerts, *a)
-		case notifier.StateInactive:
-			// set End to execStart to notify
-			// that it was just resolved
-			a.End = time.Now()
-			alerts = append(alerts, *a)
-		}
-	}
+
+	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
 		return nil
 	}
 
-	alertsSent.Add(len(alerts))
-	errGr := new(utils.ErrGroup)
-	for _, nt := range e.notifiers {
+	for _, nt := range e.notifiers() {
 		if err := nt.Send(ctx, alerts); err != nil {
-			alertsSendErrors.Inc()
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
+			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
 		}
 	}
 	return errGr.Err()
 }
+
+// getStaledSeries checks whether there are stale series from previously sent ones.
+func (e *executor) getStaleSeries(rule Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
+	for _, ts := range tss {
+		// convert labels to strings so we can compare with previously sent series
+		key := labelsToString(ts.Labels)
+		ruleLabels[key] = ts.Labels
+	}
+
+	rID := rule.ID()
+	var staleS []prompbmarshal.TimeSeries
+	// check whether there are series which disappeared and need to be marked as stale
+	e.previouslySentSeriesToRWMu.Lock()
+	for key, labels := range e.previouslySentSeriesToRW[rID] {
+		if _, ok := ruleLabels[key]; ok {
+			continue
+		}
+		// previously sent series are missing in current series, so we mark them as stale
+		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
+		staleS = append(staleS, ss)
+	}
+	// set previous series to current
+	e.previouslySentSeriesToRW[rID] = ruleLabels
+	e.previouslySentSeriesToRWMu.Unlock()
+
+	return staleS
+}
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b strings.Builder
+	b.WriteRune('{')
+	for i, label := range labels {
+		if len(label.Name) == 0 {
+			b.WriteString("__name__")
+		} else {
+			b.WriteString(label.Name)
+		}
+		b.WriteRune('=')
+		b.WriteString(strconv.Quote(label.Value))
+		if i < len(labels)-1 {
+			b.WriteRune(',')
+		}
+	}
+	b.WriteRune('}')
+	return b.String()
+}

app/vmalert/group_test.go

@@ -2,14 +2,17 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func init() {
@@ -34,7 +37,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -46,7 +49,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   utils.NewPromDuration(time.Second),
+				For:   promutils.NewDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -107,17 +110,6 @@ func TestUpdateWith(t *testing.T) {
 				{Record: "foo5"},
 			},
 		},
-		{
-			"update datasource type",
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewPrometheusType()},
-				{Alert: "foo3", Type: datasource.NewGraphiteType()},
-			},
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewGraphiteType()},
-				{Alert: "foo10", Type: datasource.NewPrometheusType()},
-			},
-		},
 	}
 
 	for _, tc := range testCases {
@@ -169,10 +161,11 @@ func TestGroupStart(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
-	const evalInterval = time.Millisecond
+
 	fs := &fakeQuerier{}
 	fn := &fakeNotifier{}
 
+	const evalInterval = time.Millisecond
 	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
 	g.Concurrency = 2
 
@@ -181,7 +174,7 @@ func TestGroupStart(t *testing.T) {
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
 	r := g.Rules[0].(*AlertingRule)
-	alert1, err := r.newAlert(m1, time.Now(), nil)
+	alert1, err := r.newAlert(m1, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -191,9 +184,12 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert1.Labels["label"] = "bar"
 	alert1.Labels["host"] = inst1
-	alert1.ID = hash(m1)
+	// add service labels
+	alert1.Labels[alertNameLabel] = alert1.Name
+	alert1.Labels[alertGroupNameLabel] = g.Name
+	alert1.ID = hash(alert1.Labels)
 
-	alert2, err := r.newAlert(m2, time.Now(), nil)
+	alert2, err := r.newAlert(m2, nil, time.Now(), nil)
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
@@ -203,13 +199,16 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert2.Labels["label"] = "bar"
 	alert2.Labels["host"] = inst2
-	alert2.ID = hash(m2)
+	// add service labels
+	alert2.Labels[alertNameLabel] = alert2.Name
+	alert2.Labels[alertGroupNameLabel] = g.Name
+	alert2.ID = hash(alert2.Labels)
 
 	finished := make(chan struct{})
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), []notifier.Notifier{fn}, nil)
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil)
 		close(finished)
 	}()
 
@@ -220,6 +219,12 @@ func TestGroupStart(t *testing.T) {
 	expectedAlerts := []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
+	gotAlertsNum := fn.getCounter()
+	if gotAlertsNum < len(expectedAlerts)*2 {
+		t.Fatalf("expected to receive at least %d alerts; got %d instead",
+			len(expectedAlerts)*2, gotAlertsNum)
+	}
+
 	// reset previous data
 	fs.reset()
 	// and set only one datapoint for response
@@ -229,9 +234,124 @@ func TestGroupStart(t *testing.T) {
 	time.Sleep(20 * evalInterval)
 
 	gotAlerts = fn.getAlerts()
-	expectedAlerts = []notifier.Alert{*alert1}
+	alert2.State = notifier.StateInactive
+	expectedAlerts = []notifier.Alert{*alert1, *alert2}
 	compareAlerts(t, expectedAlerts, gotAlerts)
 
 	g.close()
 	<-finished
 }
+
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		resendDelay   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
+}
+
+func TestGetStaleSeries(t *testing.T) {
+	ts := time.Now()
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	f := func(rule Rule, labels, expLabels [][]prompbmarshal.Label) {
+		t.Helper()
+		var tss []prompbmarshal.TimeSeries
+		for _, l := range labels {
+			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
+		}
+		staleS := e.getStaleSeries(rule, tss, ts)
+		if staleS == nil && expLabels == nil {
+			return
+		}
+		if len(staleS) != len(expLabels) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expLabels), len(staleS))
+		}
+		for i, exp := range expLabels {
+			got := staleS[i]
+			if !reflect.DeepEqual(exp, got.Labels) {
+				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
+					exp, got.Labels)
+			}
+			if len(got.Samples) != 1 {
+				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
+			}
+			if !decimal.IsStaleNaN(got.Samples[0].Value) {
+				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
+			}
+		}
+	}
+
+	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
+
+	// single series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		nil)
+
+	// multiple series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
+
+	// multiple rules and series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 2},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+}

app/vmalert/helpers_test.go

@@ -44,10 +44,10 @@ func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Qu
 }
 
 func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
-	return fq.Query(ctx, q)
+	return fq.Query(ctx, q, time.Now())
 }
 
-func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, error) {
 	fq.Lock()
 	defer fq.Unlock()
 	if fq.err != nil {
@@ -61,15 +61,26 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric,
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert
+	// records number of received alerts in total
+	counter int
 }
 
+func (*fakeNotifier) Close()       {}
+func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
+	fn.counter += len(alerts)
 	fn.alerts = alerts
 	return nil
 }
 
+func (fn *fakeNotifier) getCounter() int {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.counter
+}
+
 func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	fn.Lock()
 	defer fn.Unlock()
@@ -105,6 +116,21 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
+func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	var ls []prompbmarshal.Label
+	for i := 0; i < len(labels); i += 2 {
+		ls = append(ls, prompbmarshal.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return ls
+}
+
 func compareGroups(t *testing.T, a, b *Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -204,7 +230,8 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 			}*/
 		}
 		if len(expTS.Labels) != len(gotTS.Labels) {
-			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+			return fmt.Errorf("expected number of labels %d (%v); got %d (%v)",
+				len(expTS.Labels), expTS.Labels, len(gotTS.Labels), gotTS.Labels)
 		}
 		for i, exp := range expTS.Labels {
 			got := gotTS.Labels[i]

app/vmalert/main.go

@@ -35,26 +35,37 @@ absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
 	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
-		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes")
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
 	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
+		"which is by default equal to 3 evaluation intervals of the parent group.")
+	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
-	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+
+	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
 
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
+
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
+var alertURLGeneratorFn notifier.AlertURLGenerator
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -75,21 +86,33 @@ func main() {
 		}
 		return
 	}
+
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("failed to init `external.url`: %s", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("failed to init `external.alert.source`: %s", err)
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
 		}
-		eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-		if err != nil {
-			logger.Fatalf("failed to init `external.url`: %s", err)
+		if rw == nil {
+			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
 		}
 		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
 		}
-		q, err := datasource.Init()
+		// prevent queries from caching and boundaries aligning
+		// when querying VictoriaMetrics datasource.
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
 		if err != nil {
 			logger.Fatalf("failed to init datasource: %s", err)
 		}
@@ -111,11 +134,16 @@ func main() {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
 
+	// Register SIGHUP handler for config re-read just before manager.start call.
+	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
 	if err := manager.start(ctx, groupsCfg); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
 
-	go configReload(ctx, manager, groupsCfg)
+	go configReload(ctx, manager, groupsCfg, sighupCh)
 
 	rh := &requestHandler{m: manager}
 	go httpserver.Serve(*httpListenAddr, rh.handler)
@@ -137,29 +165,32 @@ var (
 )
 
 func newManager(ctx context.Context) (*manager, error) {
-	q, err := datasource.Init()
+	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.url`: %w", err)
+
+	labels := make(map[string]string, 0)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
 	}
-	notifier.InitTemplateFunc(eu)
-	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.alert.source`: %w", err)
-	}
-	nts, err := notifier.Init(aug)
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
-
 	manager := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         map[string]string{},
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -173,16 +204,6 @@ func newManager(ctx context.Context) (*manager, error) {
 	}
 	manager.rr = rr
 
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		manager.labels[s[:n]] = s[n+1:]
-	}
 	return manager, nil
 }
 
@@ -222,7 +243,7 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		templated, err := alert.ExecTemplate(nil, m)
+		templated, err := alert.ExecTemplate(nil, nil, m)
 		if err != nil {
 			logger.Errorf("can not exec source template %s", err)
 		}
@@ -239,15 +260,15 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .
 	flagutil.Usage(s)
 }
 
-func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
-	// Register SIGHUP handler for config re-read just before manager.start call.
-	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-	sighupCh := procutil.NewSighupChan()
-
+func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
 	var configCheckCh <-chan time.Time
-	if *rulesCheckInterval > 0 {
-		ticker := time.NewTicker(*rulesCheckInterval)
+	checkInterval := *configCheckInterval
+	if checkInterval == 0 && *rulesCheckInterval > 0 {
+		logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
+		checkInterval = *rulesCheckInterval
+	}
+	if checkInterval > 0 {
+		ticker := time.NewTicker(checkInterval)
 		configCheckCh = ticker.C
 		defer ticker.Stop()
 	}
@@ -264,6 +285,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 			configReloads.Inc()
 		case <-configCheckCh:
 		}
+		if err := notifier.Reload(); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to reload notifier config: %s", err)
+			continue
+		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
@@ -272,16 +299,19 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			// set success to 1 since previous reload
+			// could have been unsuccessful
+			configSuccess.Set(1)
 			// config didn't change - skip it
 			continue
 		}
-		groupsCfg = newGroupsCfg
-		if err := m.update(ctx, groupsCfg, false); err != nil {
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)

app/vmalert/main_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
@@ -94,14 +95,21 @@ groups:
 	*rulesCheckInterval = 200 * time.Millisecond
 	*rulePath = []string{f.Name()}
 	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 
 	m := &manager{
 		querierBuilder: &fakeQuerier{},
 		groups:         make(map[uint64]*Group),
 		labels:         map[string]string{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		rw:             &remotewrite.Client{},
 	}
-	go configReload(ctx, m, nil)
+
+	syncCh := make(chan struct{})
+	sighupCh := procutil.NewSighupChan()
+	go func() {
+		configReload(ctx, m, nil, sighupCh)
+		close(syncCh)
+	}()
 
 	lenLocked := func(m *manager) int {
 		m.groupsMu.RLock()
@@ -138,6 +146,9 @@ groups:
 	if groupsLen != 1 { // should remain unchanged
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
+
+	cancel()
+	<-syncCh
 }
 
 func writeToFile(t *testing.T, file, b string) {

app/vmalert/manager.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"sort"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -15,7 +17,7 @@ import (
 // manager controls group states
 type manager struct {
 	querierBuilder datasource.QuerierBuilder
-	notifiers      []notifier.Notifier
+	notifiers      func() []notifier.Notifier
 
 	rw *remotewrite.Client
 	// remote read builder.
@@ -35,7 +37,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 
 	g, ok := m.groups[gID]
 	if !ok {
-		return nil, fmt.Errorf("can't find group with id %q", gID)
+		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
 	for _, rule := range g.Rules {
 		ar, ok := rule.(*AlertingRule)
@@ -46,7 +48,7 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 			return apiAlert, nil
 		}
 	}
-	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
+	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)
 }
 
 func (m *manager) start(ctx context.Context, groupsCfg []config.Group) error {
@@ -85,12 +87,31 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 }
 
 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
+	var rrPresent, arPresent bool
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
+		for _, r := range cfg.Rules {
+			if rrPresent && arPresent {
+				continue
+			}
+			if r.Record != "" {
+				rrPresent = true
+			}
+			if r.Alert != "" {
+				arPresent = true
+			}
+		}
 		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	if rrPresent && m.rw == nil {
+		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
+	}
+	if arPresent && m.notifiers == nil {
+		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
+	}
+
 	type updateItem struct {
 		old *Group
 		new *Group
@@ -142,20 +163,38 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:              g.Name,
-		Type:              g.Type.String(),
-		File:              g.File,
-		Interval:          g.Interval.String(),
-		Concurrency:       g.Concurrency,
-		ExtraFilterLabels: g.ExtraFilterLabels,
+		Name:           g.Name,
+		Type:           g.Type.String(),
+		File:           g.File,
+		Interval:       g.Interval.Seconds(),
+		LastEvaluation: g.LastEvaluation,
+		Concurrency:    g.Concurrency,
+		Params:         urlValuesToStrings(g.Params),
+		Labels:         g.Labels,
 	}
 	for _, r := range g.Rules {
-		switch v := r.(type) {
-		case *AlertingRule:
-			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
-		case *RecordingRule:
-			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
-		}
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return ag
 }
+
+func urlValuesToStrings(values url.Values) []string {
+	if len(values) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		params := values[k]
+		for _, v := range params {
+			res = append(res, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return res
+}

app/vmalert/manager_test.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 )
 
 func TestMain(m *testing.M) {
@@ -38,7 +40,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: &fakeQuerier{},
-		notifiers:      []notifier.Notifier{&fakeNotifier{}},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -113,18 +115,6 @@ func TestManagerUpdate(t *testing.T) {
 			Name: "ExampleAlertAlwaysFiring",
 			Expr: "sum by(job) (up == 1)",
 		}
-		ExampleAlertGraphite = &AlertingRule{
-			Name: "up graphite",
-			Expr: "filterSeries(time('host.1',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
-		ExampleAlertGraphite2 = &AlertingRule{
-			Name: "up",
-			Expr: "filterSeries(time('host.2',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
 	)
 
 	testCases := []struct {
@@ -148,7 +138,7 @@ func TestManagerUpdate(t *testing.T) {
 							Name:   "VMRows",
 							Expr:   "vm_rows > 0",
 							For:    5 * time.Minute,
-							Labels: map[string]string{"label": "bar"},
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
 							Annotations: map[string]string{
 								"summary":     "{{ $value }}",
 								"description": "{{$labels}}",
@@ -226,28 +216,15 @@ func TestManagerUpdate(t *testing.T) {
 				},
 			},
 		},
-		{
-			name:       "update prometheus to graphite type",
-			initPath:   "config/testdata/dir/rules-update0-good.rules",
-			updatePath: "config/testdata/dir/rules-update1-good.rules",
-			want: []*Group{
-				{
-					File:     "config/testdata/dir/rules-update1-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     datasource.NewGraphiteType(),
-					Name:     "TestUpdateGroup",
-					Rules: []Rule{
-						ExampleAlertGraphite2,
-						ExampleAlertGraphite,
-					},
-				},
-			},
-		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}}
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+			}
 
 			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
 			if err := m.update(ctx, cfgInit, false); err != nil {
@@ -276,6 +253,80 @@ func TestManagerUpdate(t *testing.T) {
 	}
 }
 
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        *remotewrite.Client
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&fakeNotifier{}},
+			nil,
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				rw:             tc.rw,
+			}
+			if tc.notifiers != nil {
+				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
+}
+
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
 	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)

app/vmalert/metrics.go

@@ -1,39 +0,0 @@
-package main
-
-import "github.com/VictoriaMetrics/metrics"
-
-type gauge struct {
-	name string
-	*metrics.Gauge
-}
-
-func getOrCreateGauge(name string, f func() float64) *gauge {
-	return &gauge{
-		name:  name,
-		Gauge: metrics.GetOrCreateGauge(name, f),
-	}
-}
-
-type counter struct {
-	name string
-	*metrics.Counter
-}
-
-func getOrCreateCounter(name string) *counter {
-	return &counter{
-		name:    name,
-		Counter: metrics.GetOrCreateCounter(name),
-	}
-}
-
-type summary struct {
-	name string
-	*metrics.Summary
-}
-
-func getOrCreateSummary(name string) *summary {
-	return &summary{
-		name:    name,
-		Summary: metrics.GetOrCreateSummary(name),
-	}
-}

app/vmalert/notifier/alert.go

@@ -9,22 +9,41 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // Alert the triggered alert
 // TODO: Looks like alert name isn't unique
 type Alert struct {
-	GroupID     uint64
-	Name        string
-	Labels      map[string]string
+	// GroupID contains the ID of the parent rules group
+	GroupID uint64
+	// Name represents Alert name
+	Name string
+	// Labels is the list of label-value pairs attached to the Alert
+	Labels map[string]string
+	// Annotations is the list of annotations generated on Alert evaluation
 	Annotations map[string]string
-	State       AlertState
-
-	Expr  string
+	// State represents the current state of the Alert
+	State AlertState
+	// Expr contains expression that was executed to generate the Alert
+	Expr string
+	// ActiveAt defines the moment of time when Alert has become active
+	ActiveAt time.Time
+	// Start defines the moment of time when Alert has become firing
 	Start time.Time
-	End   time.Time
+	// End defines the moment of time when Alert supposed to expire
+	End time.Time
+	// ResolvedAt defines the moment when Alert was switched from Firing to Inactive
+	ResolvedAt time.Time
+	// LastSent defines the moment when Alert was sent last time
+	LastSent time.Time
+	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	ID    uint64
+	// ID is the unique identifer for the Alert
+	ID uint64
+	// Restored is true if Alert was restored after restart
+	Restored bool
 }
 
 // AlertState type indicates the Alert state
@@ -59,14 +78,20 @@ type AlertTplData struct {
 	Expr   string
 }
 
-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+}
 
 // ExecTemplate executes the Alert template for given
 // map of annotations.
 // Every alert could have a different datasource, so function
 // requires a queryFunction as an argument.
-func (a *Alert) ExecTemplate(q QueryFn, annotations map[string]string) (map[string]string, error) {
-	tplData := AlertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
+func (a *Alert) ExecTemplate(q QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{Value: a.Value, Labels: labels, Expr: a.Expr}
 	return templateAnnotations(annotations, tplData, funcsWithQuery(q))
 }
 
@@ -89,13 +114,15 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	var buf bytes.Buffer
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
 	for key, text := range annotations {
 		buf.Reset()
 		builder.Reset()
-		builder.Grow(len(tplHeader) + len(text))
-		builder.WriteString(tplHeader)
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
 		builder.WriteString(text)
-		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+		if err := templateAnnotation(&buf, builder.String(), tData, funcs); err != nil {
 			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
@@ -105,7 +132,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	return r, eg.Err()
 }
 
-func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error {
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, funcs template.FuncMap) error {
 	t := template.New("").Funcs(funcs).Option("missingkey=zero")
 	tpl, err := t.Parse(text)
 	if err != nil {
@@ -116,3 +149,18 @@ func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs tem
 	}
 	return nil
 }
+
+func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
+	for k, v := range a.Labels {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	promrelabel.SortLabels(labels)
+	if relabelCfg != nil {
+		return relabelCfg.Apply(labels, 0, false)
+	}
+	return labels
+}

app/vmalert/notifier/alert_test.go

@@ -1,12 +1,27 @@
 package notifier
 
 import (
+	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string, 0)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
 	testCases := []struct {
 		name        string
 		alert       *Alert
@@ -74,6 +89,26 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"desc":    "bar 1;garply 2;",
 			},
 		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
 	}
 
 	qFn := func(q string) ([]datasource.Metric, error) {
@@ -98,7 +133,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			tpl, err := tc.alert.ExecTemplate(qFn, tc.annotations)
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -114,3 +149,48 @@ func TestAlert_ExecTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestAlert_toPromLabels(t *testing.T) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
+		t.Helper()
+		a := Alert{Labels: labels}
+		got := a.toPromLabels(relabel)
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
+				exp, got)
+		}
+	}
+
+	fn(nil, nil, nil)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		nil,
+	)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	fn(
+		map[string]string{"a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"foo": "bar", "a": "baz"},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		pcs,
+	)
+	fn(
+		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		pcs,
+	)
+}

app/vmalert/notifier/alertmanager.go

@@ -7,31 +7,81 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	alertURL      string
-	basicAuthUser string
-	basicAuthPass string
-	argFunc       AlertURLGenerator
-	client        *http.Client
+	addr    string
+	argFunc AlertURLGenerator
+	client  *http.Client
+	timeout time.Duration
+
+	authCfg *promauth.Config
+	// stores already parsed RelabelConfigs object
+	relabelConfigs *promrelabel.ParsedConfigs
+
+	metrics *metrics
 }
 
+type metrics struct {
+	alertsSent       *utils.Counter
+	alertsSendErrors *utils.Counter
+}
+
+func newMetrics(addr string) *metrics {
+	return &metrics{
+		alertsSent:       utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+	}
+}
+
+// Close is a destructor method for AlertManager
+func (am *AlertManager) Close() {
+	am.metrics.alertsSent.Unregister()
+	am.metrics.alertsSendErrors.Unregister()
+}
+
+// Addr returns address where alerts are sent.
+func (am AlertManager) Addr() string { return am.addr }
+
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
-	b := &bytes.Buffer{}
-	writeamRequest(b, alerts, am.argFunc)
+	am.metrics.alertsSent.Add(len(alerts))
+	err := am.send(ctx, alerts)
+	if err != nil {
+		am.metrics.alertsSendErrors.Add(len(alerts))
+	}
+	return err
+}
 
-	req, err := http.NewRequest("POST", am.alertURL, b)
+func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
+	b := &bytes.Buffer{}
+	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
+
+	req, err := http.NewRequest("POST", am.addr, b)
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
+
+	if am.timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, am.timeout)
+		defer cancel()
+	}
+
 	req = req.WithContext(ctx)
-	if am.basicAuthPass != "" {
-		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
+
+	if am.authCfg != nil {
+		if auth := am.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {
@@ -43,9 +93,9 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return fmt.Errorf("failed to read response from %q: %w", am.alertURL, err)
+			return fmt.Errorf("failed to read response from %q: %w", am.addr, err)
 		}
-		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.addr, string(body))
 	}
 	return nil
 }
@@ -56,13 +106,41 @@ type AlertURLGenerator func(Alert) string
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
-	addr := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
-	return &AlertManager{
-		alertURL:      addr,
-		argFunc:       fn,
-		client:        c,
-		basicAuthUser: user,
-		basicAuthPass: pass,
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
+	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration) (*AlertManager, error) {
+	tls := &promauth.TLSConfig{}
+	if authCfg.TLSConfig != nil {
+		tls = authCfg.TLSConfig
 	}
+	tr, err := utils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+
+	ba := new(promauth.BasicAuthConfig)
+	oauth := new(promauth.OAuth2Config)
+	if authCfg.BasicAuth != nil {
+		ba = authCfg.BasicAuth
+	}
+	if authCfg.OAuth2 != nil {
+		oauth = authCfg.OAuth2
+	}
+
+	aCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";")))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
+	return &AlertManager{
+		addr:           alertManagerURL,
+		argFunc:        fn,
+		authCfg:        aCfg,
+		relabelConfigs: relabelCfg,
+		client:         &http.Client{Transport: tr},
+		timeout:        timeout,
+		metrics:        newMetrics(alertManagerURL),
+	}, nil
 }

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,9 +1,11 @@
 {% import (
     "time"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
 {
@@ -14,8 +16,9 @@
     {% endif %}
     "labels": {
         "alertname":{%q= alert.Name %}
-        {% for k,v := range alert.Labels %}
-            ,{%q= k %}:{%q= v %}
+        {% code lbls := alert.toPromLabels(relabelCfg) %}
+        {% for _, l := range lbls %}
+            ,{%q= l.Name %}:{%q= l.Value %}
         {% endfor %}
     },
     "annotations": {

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -7,124 +7,129 @@ package notifier
 //line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:6
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:8
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		qw422016.N().S(`"labels": {"alertname":`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-		for k, v := range alert.Labels {
-//line app/vmalert/notifier/alertmanager_request.qtpl:17
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().Q(alert.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:19
-		qw422016.N().S(`},"annotations": {`)
+		lbls := alert.toPromLabels(relabelCfg)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		for _, l := range lbls {
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			qw422016.N().Q(l.Value)
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 		c := len(alert.Annotations)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		for k, v := range alert.Annotations {
-//line app/vmalert/notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
 			c = c - 1
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	streamamRequest(qw422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
-	writeamRequest(qb422016, alerts, generatorURL)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
+	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:35
 }

app/vmalert/notifier/alertmanager_test.go

@@ -8,8 +8,21 @@ import (
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
+func TestAlertManager_Addr(t *testing.T) {
+	const addr = "http://localhost"
+	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	if am.Addr() != addr {
+		t.Errorf("expected to have %q; got %q", addr, am.Addr())
+	}
+}
+
 func TestAlertManager_Send(t *testing.T) {
 	const baUser, baPass = "foo", "bar"
 	mux := http.NewServeMux()
@@ -67,9 +80,19 @@ func TestAlertManager_Send(t *testing.T) {
 	})
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewAlertManager(srv.URL, baUser, baPass, func(alert Alert) string {
+
+	aCfg := promauth.HTTPClientConfig{
+		BasicAuth: &promauth.BasicAuthConfig{
+			Username: baUser,
+			Password: promauth.NewSecret(baPass),
+		},
+	}
+	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, srv.Client())
+	}, aCfg, nil, 0)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil {
 		t.Error("expected connection error got nil")
 	}

app/vmalert/notifier/config.go

@@ -0,0 +1,190 @@
+package notifier
+
+import (
+	"crypto/md5"
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// Config contains list of supported configuration settings
+// for Notifier
+type Config struct {
+	// Scheme defines the HTTP scheme for Notifier address
+	Scheme string `yaml:"scheme,omitempty"`
+	// PathPrefix is added to URL path before adding alertManagerPath value
+	PathPrefix string `yaml:"path_prefix,omitempty"`
+
+	// ConsulSDConfigs contains list of settings for service discovery via Consul
+	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// StaticConfigs contains list of static targets
+	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
+
+	// HTTPClientConfig contains HTTP configuration for Notifier clients
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
+	// RelabelConfigs contains list of relabeling rules for entities discovered via SD
+	RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
+	// AlertRelabelConfigs contains list of relabeling rules alert labels
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
+	// The timeout used when sending alerts.
+	Timeout promutils.Duration `yaml:"timeout,omitempty"`
+
+	// Checksum stores the hash of yaml definition for the config.
+	// May be used to detect any changes to the config file.
+	Checksum string
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+
+	// This is set to the directory from where the config has been loaded.
+	baseDir string
+
+	// stores already parsed RelabelConfigs object
+	parsedRelabelConfigs *promrelabel.ParsedConfigs
+	// stores already parsed AlertRelabelConfigs object
+	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
+}
+
+// StaticConfig contains list of static targets in the following form:
+// 	targets:
+//  	[ - '<host>' ]
+type StaticConfig struct {
+	Targets []string `yaml:"targets"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type config Config
+	if err := unmarshal((*config)(cfg)); err != nil {
+		return err
+	}
+	if cfg.Scheme == "" {
+		cfg.Scheme = "http"
+	}
+	if cfg.Timeout.Duration() == 0 {
+		cfg.Timeout = promutils.NewDuration(time.Second * 10)
+	}
+	rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse relabeling config: %w", err)
+	}
+	cfg.parsedRelabelConfigs = rCfg
+	arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false)
+	if err != nil {
+		return fmt.Errorf("failed to parse alert relabeling config: %w", err)
+	}
+	cfg.parsedAlertRelabelConfigs = arCfg
+
+	b, err := yaml.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)
+	}
+	h := md5.New()
+	h.Write(b)
+	cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
+func parseConfig(path string) (*Config, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading config file: %w", err)
+	}
+	var cfg *Config
+	err = yaml.Unmarshal(data, &cfg)
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.XXX) > 0 {
+		var keys []string
+		for k := range cfg.XXX {
+			keys = append(keys, k)
+		}
+		return nil, fmt.Errorf("unknown fields in %s", strings.Join(keys, ", "))
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
+	}
+	cfg.baseDir = filepath.Dir(absPath)
+	return cfg, nil
+}
+
+func parseLabels(target string, metaLabels map[string]string, cfg *Config) (string, []prompbmarshal.Label, error) {
+	labels := mergeLabels(target, metaLabels, cfg)
+	labels = cfg.parsedRelabelConfigs.Apply(labels, 0, false)
+	labels = promrelabel.RemoveMetaLabels(labels[:0], labels)
+	// Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels).
+	// This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 for details.
+	labels = append([]prompbmarshal.Label{}, labels...)
+
+	if len(labels) == 0 {
+		return "", nil, nil
+	}
+	schemeRelabeled := promrelabel.GetLabelValueByName(labels, "__scheme__")
+	if len(schemeRelabeled) == 0 {
+		schemeRelabeled = "http"
+	}
+	addressRelabeled := promrelabel.GetLabelValueByName(labels, "__address__")
+	if len(addressRelabeled) == 0 {
+		return "", nil, nil
+	}
+	if strings.Contains(addressRelabeled, "/") {
+		return "", nil, nil
+	}
+	addressRelabeled = addMissingPort(schemeRelabeled, addressRelabeled)
+	alertsPathRelabeled := promrelabel.GetLabelValueByName(labels, "__alerts_path__")
+	if !strings.HasPrefix(alertsPathRelabeled, "/") {
+		alertsPathRelabeled = "/" + alertsPathRelabeled
+	}
+	u := fmt.Sprintf("%s://%s%s", schemeRelabeled, addressRelabeled, alertsPathRelabeled)
+	if _, err := url.Parse(u); err != nil {
+		return "", nil, fmt.Errorf("invalid url %q for scheme=%q (%q), target=%q, metrics_path=%q (%q): %w",
+			u, cfg.Scheme, schemeRelabeled, target, addressRelabeled, alertsPathRelabeled, err)
+	}
+	return u, labels, nil
+}
+
+func addMissingPort(scheme, target string) string {
+	if strings.Contains(target, ":") {
+		return target
+	}
+	if scheme == "https" {
+		target += ":443"
+	} else {
+		target += ":80"
+	}
+	return target
+}
+
+func mergeLabels(target string, metaLabels map[string]string, cfg *Config) []prompbmarshal.Label {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+	m := make(map[string]string)
+	m["__address__"] = target
+	m["__scheme__"] = cfg.Scheme
+	m["__alerts_path__"] = path.Join("/", cfg.PathPrefix, alertManagerPath)
+	for k, v := range metaLabels {
+		m[k] = v
+	}
+	result := make([]prompbmarshal.Label, 0, len(m))
+	for k, v := range m {
+		result = append(result, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	return result
+}

app/vmalert/notifier/config_test.go

@@ -0,0 +1,31 @@
+package notifier
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestConfigParseGood(t *testing.T) {
+	f := func(path string) {
+		_, err := parseConfig(path)
+		checkErr(t, err)
+	}
+	f("testdata/mixed.good.yaml")
+	f("testdata/consul.good.yaml")
+	f("testdata/static.good.yaml")
+}
+
+func TestConfigParseBad(t *testing.T) {
+	f := func(path, expErr string) {
+		_, err := parseConfig(path)
+		if err == nil {
+			t.Fatalf("expected to get non-nil err for config %q", path)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("testdata/unknownFields.bad.yaml", "unknown field")
+	f("non-existing-file", "error reading")
+}

app/vmalert/notifier/config_watcher.go

@@ -0,0 +1,235 @@
+package notifier
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+)
+
+// configWatcher supports dynamic reload of Notifier objects
+// from static configuration and service discovery.
+// Use newWatcher to create a new object.
+type configWatcher struct {
+	cfg   *Config
+	genFn AlertURLGenerator
+	wg    sync.WaitGroup
+
+	reloadCh chan struct{}
+	syncCh   chan struct{}
+
+	targetsMu sync.RWMutex
+	targets   map[TargetType][]Target
+}
+
+func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) {
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return nil, err
+	}
+	cw := &configWatcher{
+		cfg:       cfg,
+		wg:        sync.WaitGroup{},
+		reloadCh:  make(chan struct{}, 1),
+		syncCh:    make(chan struct{}),
+		genFn:     gen,
+		targetsMu: sync.RWMutex{},
+		targets:   make(map[TargetType][]Target),
+	}
+	return cw, cw.start()
+}
+
+func (cw *configWatcher) notifiers() []Notifier {
+	cw.targetsMu.RLock()
+	defer cw.targetsMu.RUnlock()
+
+	var notifiers []Notifier
+	for _, ns := range cw.targets {
+		for _, n := range ns {
+			notifiers = append(notifiers, n.Notifier)
+		}
+
+	}
+	return notifiers
+}
+
+func (cw *configWatcher) reload(path string) error {
+	select {
+	case cw.reloadCh <- struct{}{}:
+	default:
+		return nil
+	}
+
+	defer func() { <-cw.reloadCh }()
+
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return err
+	}
+	if cfg.Checksum == cw.cfg.Checksum {
+		return nil
+	}
+
+	// stop existing discovery
+	cw.mustStop()
+
+	// re-start cw with new config
+	cw.syncCh = make(chan struct{})
+	cw.cfg = cfg
+	return cw.start()
+}
+
+func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
+	targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+	for _, err := range errors {
+		return fmt.Errorf("failed to init notifier for %q: %s", typeK, err)
+	}
+
+	cw.setTargets(typeK, targets)
+
+	cw.wg.Add(1)
+	go func() {
+		defer cw.wg.Done()
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-cw.syncCh:
+				return
+			case <-ticker.C:
+			}
+			updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
+			for _, err := range errors {
+				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+			}
+			cw.setTargets(typeK, updateTargets)
+		}
+	}()
+	return nil
+}
+
+func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
+	metaLabels, err := labelsFn()
+	if err != nil {
+		return nil, []error{fmt.Errorf("failed to get labels: %s", err)}
+	}
+	var targets []Target
+	var errors []error
+	duplicates := make(map[string]struct{})
+	for _, labels := range metaLabels {
+		target := labels["__address__"]
+		u, processedLabels, err := parseLabels(target, labels, cfg)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		if len(u) == 0 {
+			continue
+		}
+		if _, ok := duplicates[u]; ok { // check for duplicates
+			if !*suppressDuplicateTargetErrors {
+				logger.Errorf("skipping duplicate target with identical address %q; "+
+					"make sure service discovery and relabeling is set up properly; "+
+					"original labels: %s; resulting labels: %s",
+					u, labels, processedLabels)
+			}
+			continue
+		}
+		duplicates[u] = struct{}{}
+
+		am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		targets = append(targets, Target{
+			Notifier: am,
+			Labels:   processedLabels,
+		})
+	}
+	return targets, errors
+}
+
+type getLabels func() ([]map[string]string, error)
+
+func (cw *configWatcher) start() error {
+	if len(cw.cfg.StaticConfigs) > 0 {
+		var targets []Target
+		for _, cfg := range cw.cfg.StaticConfigs {
+			for _, target := range cfg.Targets {
+				address, labels, err := parseLabels(target, nil, cw.cfg)
+				if err != nil {
+					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
+				}
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
+				if err != nil {
+					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
+				}
+				targets = append(targets, Target{
+					Notifier: notifier,
+					Labels:   labels,
+				})
+			}
+		}
+		cw.setTargets(TargetStatic, targets)
+	}
+
+	if len(cw.cfg.ConsulSDConfigs) > 0 {
+		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]map[string]string, error) {
+			var labels []map[string]string
+			for i := range cw.cfg.ConsulSDConfigs {
+				sdc := &cw.cfg.ConsulSDConfigs[i]
+				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
+				if err != nil {
+					return nil, fmt.Errorf("got labels err: %s", err)
+				}
+				labels = append(labels, targetLabels...)
+			}
+			return labels, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start consulSD discovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func (cw *configWatcher) mustStop() {
+	close(cw.syncCh)
+	cw.wg.Wait()
+
+	cw.targetsMu.Lock()
+	for _, targets := range cw.targets {
+		for _, t := range targets {
+			t.Close()
+		}
+	}
+	cw.targets = make(map[TargetType][]Target)
+	cw.targetsMu.Unlock()
+
+	for i := range cw.cfg.ConsulSDConfigs {
+		cw.cfg.ConsulSDConfigs[i].MustStop()
+	}
+	cw.cfg = nil
+}
+
+func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
+	cw.targetsMu.Lock()
+	newT := make(map[string]Target)
+	for _, t := range targets {
+		newT[t.Addr()] = t
+	}
+	oldT := cw.targets[key]
+
+	for _, ot := range oldT {
+		if _, ok := newT[ot.Addr()]; !ok {
+			ot.Notifier.Close()
+		}
+	}
+	cw.targets[key] = targets
+	cw.targetsMu.Unlock()
+}

app/vmalert/notifier/config_watcher_test.go

@@ -0,0 +1,301 @@
+package notifier
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"testing"
+)
+
+func TestConfigWatcherReload(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f.Name()) }()
+
+	writeToFile(t, f.Name(), `
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9094
+`)
+	cw, err := newWatcher(f.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+	ns := cw.notifiers()
+	if len(ns) != 2 {
+		t.Fatalf("expected to have 2 notifiers; got %d %#v", len(ns), ns)
+	}
+
+	f2, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(f2.Name()) }()
+
+	writeToFile(t, f2.Name(), `
+static_configs:
+  - targets:
+      - 127.0.0.1:9093
+`)
+	checkErr(t, cw.reload(f2.Name()))
+
+	ns = cw.notifiers()
+	if len(ns) != 1 {
+		t.Fatalf("expected to have 1 notifier; got %d", len(ns))
+	}
+	expAddr := "http://127.0.0.1:9093/api/v2/alerts"
+	if ns[0].Addr() != expAddr {
+		t.Fatalf("expected to get %q; got %q instead", expAddr, ns[0].Addr())
+	}
+}
+
+func TestConfigWatcherStart(t *testing.T) {
+	consulSDServer := newFakeConsulServer()
+	defer consulSDServer.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+scheme: https
+path_prefix: proxy
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+`, consulSDServer.URL))
+
+	cw, err := newWatcher(consulSDFile.Name(), nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	if len(cw.notifiers()) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers()))
+	}
+
+	expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1)
+	expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2)
+
+	n1, n2 := cw.notifiers()[0], cw.notifiers()[1]
+	if n1.Addr() != expAddr1 {
+		t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr())
+	}
+	if n2.Addr() != expAddr2 {
+		t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
+	}
+}
+
+// TestConfigWatcherReloadConcurrent supposed to test concurrent
+// execution of configuration update.
+// Should be executed with -race flag
+func TestConfigWatcherReloadConcurrent(t *testing.T) {
+	consulSDServer1 := newFakeConsulServer()
+	defer consulSDServer1.Close()
+	consulSDServer2 := newFakeConsulServer()
+	defer consulSDServer2.Close()
+
+	consulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
+
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	staticAndConsulSDFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Remove(staticAndConsulSDFile.Name()) }()
+
+	writeToFile(t, staticAndConsulSDFile.Name(), fmt.Sprintf(`
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+consul_sd_configs:
+  - server: %s
+    services:
+      - alertmanager
+  - server: %s
+    services:
+      - consul
+`, consulSDServer1.URL, consulSDServer2.URL))
+
+	paths := []string{
+		staticAndConsulSDFile.Name(),
+		consulSDFile.Name(),
+		"testdata/static.good.yaml",
+		"unknownFields.bad.yaml",
+	}
+
+	cw, err := newWatcher(paths[0], nil)
+	if err != nil {
+		t.Fatalf("failed to start config watcher: %s", err)
+	}
+	defer cw.mustStop()
+
+	const workers = 500
+	const iterations = 10
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				_ = cw.reload(paths[rnd]) // update can fail and this is expected
+				_ = cw.notifiers()
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	checkErr(t, ioutil.WriteFile(file, []byte(b), 0644))
+}
+
+func checkErr(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+}
+
+const (
+	fakeConsulService1 = "127.0.0.1:9093"
+	fakeConsulService2 = "127.0.0.1:9095"
+)
+
+func newFakeConsulServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
+	})
+	mux.HandleFunc("/v1/catalog/services", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`{
+    "alertmanager": [
+        "alertmanager",
+        "__scheme__=http"
+    ]
+}`))
+	})
+	mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
+		rw.Header().Set("X-Consul-Index", "1")
+		rw.Write([]byte(`
+[
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am1",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "__scheme__=http"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9093,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 16,
+            "ModifyIndex": 16
+        }
+    },
+    {
+        "Node": {
+            "ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
+            "Node": "machine",
+            "Address": "127.0.0.1",
+            "Datacenter": "dc1",
+            "TaggedAddresses": {
+                "lan": "127.0.0.1",
+                "lan_ipv4": "127.0.0.1",
+                "wan": "127.0.0.1",
+                "wan_ipv4": "127.0.0.1"
+            },
+            "Meta": {
+                "consul-network-segment": ""
+            },
+            "CreateIndex": 13,
+            "ModifyIndex": 14
+        },
+        "Service": {
+            "ID": "am2",
+            "Service": "alertmanager",
+            "Tags": [
+                "alertmanager",
+                "bad-node"
+            ],
+            "Address": "",
+            "Meta": null,
+            "Port": 9095,
+            "Weights": {
+                "Passing": 1,
+                "Warning": 1
+            },
+            "EnableTagOverride": false,
+            "Proxy": {
+                "Mode": "",
+                "MeshGateway": {},
+                "Expose": {}
+            },
+            "Connect": {},
+            "CreateIndex": 15,
+            "ModifyIndex": 15
+        }
+    }
+]`))
+	})
+
+	return httptest.NewServer(mux)
+}

app/vmalert/notifier/init.go

@@ -1,17 +1,28 @@
 package notifier
 
 import (
+	"flag"
 	"fmt"
-	"net/http"
+	"strings"
+	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
-	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
-	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
+	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
+
+	addrs = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+
+	basicAuthUsername     = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthPassword     = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
+	basicAuthPasswordFile = flagutil.NewArray("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
+
+	bearerToken     = flagutil.NewArray("notifier.bearerToken", "Optional bearer token for -notifier.url")
+	bearerTokenFile = flagutil.NewArray("notifier.bearerTokenFile", "Optional path to bearer token file for -notifier.url")
 
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("notifier.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to -notifier.url")
 	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
@@ -20,26 +31,158 @@ var (
 		"By default system CA is used")
 	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
 		"By default the server name from -notifier.url is used")
+
+	oauth2ClientID = flagutil.NewArray("notifier.oauth2.clientID", "Optional OAuth2 clientID to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecret = flagutil.NewArray("notifier.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2ClientSecretFile = flagutil.NewArray("notifier.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2TokenURL = flagutil.NewArray("notifier.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for -notifier.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	oauth2Scopes = flagutil.NewArray("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
+		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 )
 
-// Init creates a Notifier object based on provided flags.
-func Init(gen AlertURLGenerator) ([]Notifier, error) {
-	if len(*addrs) == 0 {
-		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
+// cw holds a configWatcher for configPath configuration file
+// configWatcher provides a list of Notifier objects discovered
+// from static config or via service discovery.
+// cw is not nil only if configPath is provided.
+var cw *configWatcher
+
+// Reload checks the changes in configPath configuration file
+// and applies changes if any.
+func Reload() error {
+	if cw == nil {
+		return nil
+	}
+	return cw.reload(*configPath)
+}
+
+var staticNotifiersFn func() []Notifier
+
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
+// Init returns a function for retrieving actual list of Notifier objects.
+// Init works in two mods:
+//   * configuration via flags (for backward compatibility). Is always static
+//     and don't support live reloads.
+//   * configuration via file. Supports live reloads and service discovery.
+// Init returns an error if both mods are used.
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	if externalLabels != nil || externalURL != "" {
+		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
 	}
 
+	externalURL = extURL
+	externalLabels = extLabels
+
+	if *configPath == "" && len(*addrs) == 0 {
+		return nil, nil
+	}
+	if *configPath != "" && len(*addrs) > 0 {
+		return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
+	}
+
+	if len(*addrs) > 0 {
+		notifiers, err := notifiersFromFlags(gen)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create notifier from flag values: %s", err)
+		}
+		staticNotifiersFn = func() []Notifier {
+			return notifiers
+		}
+		return staticNotifiersFn, nil
+	}
+
+	var err error
+	cw, err = newWatcher(*configPath, gen)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init config watcher: %s", err)
+	}
+	return cw.notifiers, nil
+}
+
+func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 	var notifiers []Notifier
 	for i, addr := range *addrs {
-		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)
-		ca, serverName := tlsCAFile.GetOptionalArg(i), tlsServerName.GetOptionalArg(i)
-		tr, err := utils.Transport(addr, cert, key, ca, serverName, tlsInsecureSkipVerify.GetOptionalArg(i))
-		if err != nil {
-			return nil, fmt.Errorf("failed to create transport: %w", err)
+		authCfg := promauth.HTTPClientConfig{
+			TLSConfig: &promauth.TLSConfig{
+				CAFile:             tlsCAFile.GetOptionalArg(i),
+				CertFile:           tlsCertFile.GetOptionalArg(i),
+				KeyFile:            tlsKeyFile.GetOptionalArg(i),
+				ServerName:         tlsServerName.GetOptionalArg(i),
+				InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(i),
+			},
+			BasicAuth: &promauth.BasicAuthConfig{
+				Username:     basicAuthUsername.GetOptionalArg(i),
+				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
+				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
+			},
+			BearerToken:     promauth.NewSecret(bearerToken.GetOptionalArg(i)),
+			BearerTokenFile: bearerTokenFile.GetOptionalArg(i),
+			OAuth2: &promauth.OAuth2Config{
+				ClientID:         oauth2ClientID.GetOptionalArg(i),
+				ClientSecret:     promauth.NewSecret(oauth2ClientSecret.GetOptionalArg(i)),
+				ClientSecretFile: oauth2ClientSecretFile.GetOptionalArg(i),
+				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
+				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
+			},
+		}
+
+		addr = strings.TrimSuffix(addr, "/")
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
+		if err != nil {
+			return nil, err
 		}
-		user, pass := basicAuthUsername.GetOptionalArg(i), basicAuthPassword.GetOptionalArg(i)
-		am := NewAlertManager(addr, user, pass, gen, &http.Client{Transport: tr})
 		notifiers = append(notifiers, am)
 	}
-
 	return notifiers, nil
 }
+
+// Target represents a Notifier and optional
+// list of labels added during discovery.
+type Target struct {
+	Notifier
+	Labels []prompbmarshal.Label
+}
+
+// TargetType defines how the Target was discovered
+type TargetType string
+
+const (
+	// TargetStatic is for targets configured statically
+	TargetStatic TargetType = "static"
+	// TargetConsul is for targets discovered via Consul
+	TargetConsul TargetType = "consulSD"
+)
+
+// GetTargets returns list of static or discovered targets
+// via notifier configuration.
+func GetTargets() map[TargetType][]Target {
+	var targets = make(map[TargetType][]Target)
+
+	if staticNotifiersFn != nil {
+		for _, ns := range staticNotifiersFn() {
+			targets[TargetStatic] = append(targets[TargetStatic], Target{
+				Notifier: ns,
+			})
+		}
+	}
+
+	if cw != nil {
+		cw.targetsMu.RLock()
+		for key, ns := range cw.targets {
+			targets[key] = append(targets[key], ns...)
+		}
+		cw.targetsMu.RUnlock()
+	}
+	return targets
+}

app/vmalert/notifier/notifier.go

@@ -2,7 +2,14 @@ package notifier
 
 import "context"
 
-// Notifier is common interface for alert manager provider
+// Notifier is a common interface for alert manager provider
 type Notifier interface {
+	// Send sends the given list of alerts.
+	// Returns an error if fails to send the alerts.
+	// Must unblock if the given ctx is cancelled.
 	Send(ctx context.Context, alerts []Alert) error
+	// Addr returns address where alerts are sent.
+	Addr() string
+	// Close is a destructor for the Notifier
+	Close()
 }

app/vmalert/notifier/template_func.go

@@ -17,8 +17,10 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net"
 	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"time"
 
@@ -26,6 +28,7 @@ import (
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // metric is private copy of datasource.Metric,
@@ -61,6 +64,7 @@ var tmplFunc textTpl.FuncMap
 
 // InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
 	tmplFunc = textTpl.FuncMap{
 		/* Strings */
 
@@ -91,6 +95,24 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// alias for https://golang.org/pkg/strings/#ToLower
 		"toLower": strings.ToLower,
 
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
 		/* Numbers */
 
 		// humanize converts given number to a human readable format
@@ -261,6 +283,14 @@ func InitTemplateFunc(externalURL *url.URL) {
 			return m.Labels[label]
 		},
 
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
 		// value returns the value of the given metric.
 		// usually used alongside with `query` template function.
 		"value": func(m metric) float64 {

app/vmalert/notifier/testdata/consul.good.yaml

@@ -0,0 +1,16 @@
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -0,0 +1,18 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+  - server: localhost:8500
+    services:
+      - consul
+relabel_configs:
+  - source_labels: [__meta_consul_tags]
+    regex: .*,__scheme__=([^,]+),.*
+    replacement: '${1}'
+    target_label: __scheme__

app/vmalert/notifier/testdata/static.good.yaml

@@ -0,0 +1,7 @@
+static_configs:
+  - targets:
+      - localhost:9093
+      - localhost:9095
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/unknownFields.bad.yaml

@@ -0,0 +1,5 @@
+scheme: https
+unknown: field
+static_configs:
+  - targets:
+      - localhost:9093

app/vmalert/recording.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // RecordingRule is a Rule that supposed
@@ -31,6 +31,8 @@ type RecordingRule struct {
 	mu sync.RWMutex
 	// stores last moment of time Exec was called
 	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
 	// stores last error that happened in Exec func
 	// resets on every successful Exec
 	// may be used as Health state
@@ -43,8 +45,8 @@ type RecordingRule struct {
 }
 
 type recordingRuleMetrics struct {
-	errors  *gauge
-	samples *gauge
+	errors  *utils.Gauge
+	samples *utils.Gauge
 }
 
 // String implements Stringer interface
@@ -60,7 +62,7 @@ func (rr *RecordingRule) ID() uint64 {
 
 func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
 	rr := &RecordingRule{
-		Type:    cfg.Type,
+		Type:    group.Type,
 		RuleID:  cfg.ID,
 		Name:    cfg.Record,
 		Expr:    cfg.Expr,
@@ -68,14 +70,14 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 	}
 
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
-	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
+	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -84,7 +86,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 			}
 			return 1
 		})
-	rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			rr.mu.RLock()
 			defer rr.mu.RUnlock()
@@ -95,8 +97,8 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 
 // Close unregisters rule metrics
 func (rr *RecordingRule) Close() {
-	metrics.UnregisterMetric(rr.metrics.errors.name)
-	metrics.UnregisterMetric(rr.metrics.samples.name)
+	rr.metrics.errors.Unregister()
+	rr.metrics.samples.Unregister()
 }
 
 // ExecRange executes recording rule on the given time range similarly to Exec.
@@ -122,12 +124,13 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }
 
 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error) {
-	qMetrics, err := rr.q.Query(ctx, rr.Expr)
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
 
-	rr.lastExecTime = time.Now()
+	rr.lastExecTime = ts
+	rr.lastExecDuration = time.Since(ts)
 	rr.lastExecError = err
 	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
@@ -193,23 +196,27 @@ func (rr *RecordingRule) UpdateWith(r Rule) error {
 	return nil
 }
 
-// RuleAPI returns Rule representation in form
-// of APIRecordingRule
-func (rr *RecordingRule) RuleAPI() APIRecordingRule {
-	var lastErr string
-	if rr.lastExecError != nil {
-		lastErr = rr.lastExecError.Error()
-	}
-	return APIRecordingRule{
+// ToAPI returns Rule's representation in form
+// of APIRule
+func (rr *RecordingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "recording",
+		DatasourceType: rr.Type.String(),
+		Name:           rr.Name,
+		Query:          rr.Expr,
+		Labels:         rr.Labels,
+		LastEvaluation: rr.lastExecTime,
+		EvaluationTime: rr.lastExecDuration.Seconds(),
+		Health:         "ok",
+		LastSamples:    rr.lastExecSamples,
 		// encode as strings to avoid rounding
-		ID:          fmt.Sprintf("%d", rr.ID()),
-		GroupID:     fmt.Sprintf("%d", rr.GroupID),
-		Name:        rr.Name,
-		Type:        rr.Type.String(),
-		Expression:  rr.Expr,
-		LastError:   lastErr,
-		LastSamples: rr.lastExecSamples,
-		LastExec:    rr.lastExecTime,
-		Labels:      rr.Labels,
+		ID:      fmt.Sprintf("%d", rr.ID()),
+		GroupID: fmt.Sprintf("%d", rr.GroupID),
 	}
+
+	if rr.lastExecError != nil {
+		r.LastError = rr.lastExecError.Error()
+		r.Health = "err"
+	}
+	return r
 }

app/vmalert/recording_test.go

@@ -77,7 +77,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now())
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -178,7 +178,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO())
+	_, err := rr.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +193,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO())
+	_, err = rr.Exec(context.TODO(), time.Now())
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remoteread/init.go

@@ -12,9 +12,15 @@ import (
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
 		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428")
+		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
+
+	bearerToken     = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
 	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
@@ -22,6 +28,14 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
+
+	oauth2ClientID         = flag.String("remoteRead.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteRead.url.")
+	oauth2ClientSecret     = flag.String("remoteRead.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteRead.url.")
+	oauth2ClientSecretFile = flag.String("remoteRead.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteRead.url.")
+	oauth2TokenURL         = flag.String("remoteRead.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -remoteRead.url. ")
+	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
+
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
 )
 
 // Init creates a Querier from provided flag values.
@@ -34,6 +48,14 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
 }

app/vmalert/remotewrite/init.go

@@ -13,8 +13,13 @@ var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
-	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+
+	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
+
+	bearerToken     = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -28,6 +33,13 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
+
+	oauth2ClientID         = flag.String("remoteWrite.oauth2.clientID", "", "Optional OAuth2 clientID to use for -remoteWrite.url.")
+	oauth2ClientSecret     = flag.String("remoteWrite.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -remoteWrite.url.")
+	oauth2ClientSecretFile = flag.String("remoteWrite.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -remoteWrite.url.")
+	oauth2TokenURL         = flag.String("remoteWrite.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -notifier.url.")
+	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
+
 	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 )
 
@@ -43,14 +55,21 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
 	return NewClient(ctx, Config{
 		Addr:              *addr,
+		AuthCfg:           authCfg,
 		Concurrency:       *concurrency,
 		MaxQueueSize:      *maxQueueSize,
 		MaxBatchSize:      *maxBatchSize,
 		FlushInterval:     *flushInterval,
-		BasicAuthUser:     *basicAuthUsername,
-		BasicAuthPass:     *basicAuthPassword,
 		DisablePathAppend: *disablePathAppend,
 		Transport:         t,
 	})

app/vmalert/remotewrite/remotewrite.go

@@ -6,14 +6,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 // Client is an asynchronous HTTP client for writing
@@ -21,8 +24,8 @@ import (
 type Client struct {
 	addr              string
 	c                 *http.Client
+	authCfg           *promauth.Config
 	input             chan prompbmarshal.TimeSeries
-	baUser, baPass    string
 	flushInterval     time.Duration
 	maxBatchSize      int
 	maxQueueSize      int
@@ -35,10 +38,8 @@ type Client struct {
 // Config is config for remote write.
 type Config struct {
 	// Addr of remote storage
-	Addr string
-
-	BasicAuthUser string
-	BasicAuthPass string
+	Addr    string
+	AuthCfg *promauth.Config
 
 	// Concurrency defines number of readers that
 	// concurrently read from the queue and flush data
@@ -92,14 +93,17 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Transport == nil {
 		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
 	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
 		addr:              strings.TrimSuffix(cfg.Addr, "/"),
-		baUser:            cfg.BasicAuthUser,
-		baPass:            cfg.BasicAuthPass,
+		authCfg:           cfg.AuthCfg,
 		flushInterval:     cfg.FlushInterval,
 		maxBatchSize:      cfg.MaxBatchSize,
 		maxQueueSize:      cfg.MaxQueueSize,
@@ -107,10 +111,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 		disablePathAppend: cfg.DisablePathAppend,
 	}
-	cc := defaultConcurrency
-	if cfg.Concurrency > 0 {
-		cc = cfg.Concurrency
-	}
+
 	for i := 0; i < cc; i++ {
 		c.run(ctx)
 	}
@@ -183,10 +184,11 @@ func (c *Client) run(ctx context.Context) {
 }
 
 var (
-	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -197,6 +199,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		return
 	}
 	defer prompbmarshal.ResetWriteRequest(wr)
+	defer bufferFlushDuration.UpdateDuration(time.Now())
 
 	data, err := wr.Marshal()
 	if err != nil {
@@ -222,7 +225,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 	droppedRows.Add(len(wr.Timeseries))
 	droppedBytes.Add(len(b))
-	logger.Errorf("all %d attempts to send request failed - dropping %d timeseries",
+	logger.Errorf("all %d attempts to send request failed - dropping %d time series",
 		attempts, len(wr.Timeseries))
 }
 
@@ -232,22 +235,24 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
-	if c.baPass != "" {
-		req.SetBasicAuth(c.baUser, c.baPass)
+	if c.authCfg != nil {
+		if auth := c.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	if !c.disablePathAppend {
-		req.URL.Path += writePath
+		req.URL.Path = path.Join(req.URL.Path, writePath)
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
-			req.URL, err, len(data), r.Size())
+			req.URL.Redacted(), err, len(data), r.Size())
 	}
 	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != http.StatusNoContent {
+	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
-			resp.StatusCode, req.URL, body)
+			resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return nil
 }

app/vmalert/replay_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 type fakeReplayQuerier struct {
@@ -83,7 +83,7 @@ func TestReplay(t *testing.T) {
 			to:    "2021-01-01T15:02:30.000Z",
 			maxDP: 60,
 			cfg: []config.Group{
-				{Interval: utils.NewPromDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 			},
 			qb: &fakeReplayQuerier{
 				registry: map[string]map[string]struct{}{

app/vmalert/rule.go

@@ -3,8 +3,9 @@ package main
 import (
 	"context"
 	"errors"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Rule represents alerting or recording rule
@@ -14,13 +15,15 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context
-	Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, error)
+	// Exec executes the rule with given context at the given timestamp
+	Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error)
 	// ExecRange executes the rule on the given time range
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.
 	UpdateWith(Rule) error
+	// ToAPI converts Rule into APIRule
+	ToAPI() APIRule
 	// Close performs the shutdown procedures for rule
 	// such as metrics unregister
 	Close()

app/vmalert/tpl/footer.qtpl

@@ -0,0 +1,36 @@
+{% func Footer() %}
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+{% endfunc %}

app/vmalert/tpl/footer.qtpl.go

@@ -0,0 +1,86 @@
+// Code generated by qtc from "footer.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/footer.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/footer.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+func StreamFooter(qw422016 *qt422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:1
+	qw422016.N().S(`
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+`)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func WriteFooter(qq422016 qtio422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:36
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	StreamFooter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func Footer() string {
+//line app/vmalert/tpl/footer.qtpl:36
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/footer.qtpl:36
+	WriteFooter(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	return qs422016
+//line app/vmalert/tpl/footer.qtpl:36
+}

app/vmalert/tpl/header.qtpl

@@ -0,0 +1,43 @@
+{% func Header(title string, pages []NavItem) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    {%= PrintNavItems(title, pages) %}
+    <main class="px-2">
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -0,0 +1,107 @@
+// Code generated by qtc from "header.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/header.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/header.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:1
+	qw422016.N().S(`
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert`)
+//line app/vmalert/tpl/header.qtpl:5
+	if title != "" {
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.N().S(` - `)
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.E().S(title)
+//line app/vmalert/tpl/header.qtpl:5
+	}
+//line app/vmalert/tpl/header.qtpl:5
+	qw422016.N().S(`</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    `)
+//line app/vmalert/tpl/header.qtpl:41
+	StreamPrintNavItems(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:41
+	qw422016.N().S(`
+    <main class="px-2">
+`)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:43
+	StreamHeader(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func Header(title string, pages []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:43
+	WriteHeader(qb422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:43
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:43
+}

app/vmalert/tpl/nav.qtpl

@@ -0,0 +1,25 @@
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func PrintNavItems(current string, items []NavItem) %}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}
+
+

app/vmalert/tpl/nav.qtpl.go

@@ -0,0 +1,96 @@
+// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/nav.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/nav.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/nav.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/nav.qtpl:2
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/nav.qtpl:8
+func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:8
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/nav.qtpl:13
+	for _, item := range items {
+//line app/vmalert/tpl/nav.qtpl:13
+		qw422016.N().S(`
+                <li class="nav-item">
+                    <a class="nav-link`)
+//line app/vmalert/tpl/nav.qtpl:15
+		if current == item.Name {
+//line app/vmalert/tpl/nav.qtpl:15
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/nav.qtpl:15
+		}
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`" href="`)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.E().S(item.Url)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/nav.qtpl:19
+	}
+//line app/vmalert/tpl/nav.qtpl:19
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	StreamPrintNavItems(qw422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func PrintNavItems(current string, items []NavItem) string {
+//line app/vmalert/tpl/nav.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/nav.qtpl:23
+	WritePrintNavItems(qb422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	return qs422016
+//line app/vmalert/tpl/nav.qtpl:23
+}

app/vmalert/utils.go

@@ -30,3 +30,20 @@ func newTimeSeries(values []float64, timestamps []int64, labels map[string]strin
 	}
 	return ts
 }
+
+// newTimeSeriesPB creates prompbmarshal.TimeSeries with given
+// values, timestamps and labels.
+// It expects that labels are already sorted.
+func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{
+		Samples: make([]prompbmarshal.Sample, len(values)),
+	}
+	for i := range values {
+		ts.Samples[i] = prompbmarshal.Sample{
+			Value:     values[i],
+			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
+		}
+	}
+	ts.Labels = labels
+	return ts
+}

app/vmalert/utils/auth.go

@@ -0,0 +1,60 @@
+package utils
+
+import (
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+// AuthConfigOptions options which helps build promauth.Config
+type AuthConfigOptions func(config *promauth.HTTPClientConfig)
+
+// AuthConfig returns promauth.Config based on the given params
+func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
+	authCfg := &promauth.HTTPClientConfig{}
+	for _, option := range filterOptions {
+		option(authCfg)
+	}
+
+	return authCfg.NewConfig(".")
+}
+
+// WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
+func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if username != "" || password != "" || passwordFile != "" {
+			config.BasicAuth = &promauth.BasicAuthConfig{
+				Username:     username,
+				Password:     promauth.NewSecret(password),
+				PasswordFile: passwordFile,
+			}
+		}
+	}
+}
+
+// WithBearer returns AuthConfigOptions and set BearerToken or BearerTokenFile based on given params
+func WithBearer(token, tokenFile string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if token != "" {
+			config.BearerToken = promauth.NewSecret(token)
+		}
+		if tokenFile != "" {
+			config.BearerTokenFile = tokenFile
+		}
+	}
+}
+
+// WithOAuth returns AuthConfigOptions and set OAuth params based on given params
+func WithOAuth(clientID, clientSecret, clientSecretFile, tokenURL, scopes string) AuthConfigOptions {
+	return func(config *promauth.HTTPClientConfig) {
+		if clientSecretFile != "" || clientSecret != "" {
+			config.OAuth2 = &promauth.OAuth2Config{
+				ClientID:         clientID,
+				ClientSecret:     promauth.NewSecret(clientSecret),
+				ClientSecretFile: clientSecretFile,
+				TokenURL:         tokenURL,
+				Scopes:           strings.Split(scopes, ";"),
+			}
+		}
+	}
+}

app/vmalert/utils/metrics.go

@@ -0,0 +1,54 @@
+package utils
+
+import "github.com/VictoriaMetrics/metrics"
+
+type namedMetric struct {
+	Name string
+}
+
+// Unregister removes the metric by name from default registry
+func (nm namedMetric) Unregister() {
+	metrics.UnregisterMetric(nm.Name)
+}
+
+// Gauge is a metrics.Gauge with Name
+type Gauge struct {
+	namedMetric
+	*metrics.Gauge
+}
+
+// GetOrCreateGauge creates a new Gauge with the given name
+func GetOrCreateGauge(name string, f func() float64) *Gauge {
+	return &Gauge{
+		namedMetric: namedMetric{Name: name},
+		Gauge:       metrics.GetOrCreateGauge(name, f),
+	}
+}
+
+// Counter is a metrics.Counter with Name
+type Counter struct {
+	namedMetric
+	*metrics.Counter
+}
+
+// GetOrCreateCounter creates a new Counter with the given name
+func GetOrCreateCounter(name string) *Counter {
+	return &Counter{
+		namedMetric: namedMetric{Name: name},
+		Counter:     metrics.GetOrCreateCounter(name),
+	}
+}
+
+// Summary is a metrics.Summary with Name
+type Summary struct {
+	namedMetric
+	*metrics.Summary
+}
+
+// GetOrCreateSummary creates a new Summary with the given name
+func GetOrCreateSummary(name string) *Summary {
+	return &Summary{
+		namedMetric: namedMetric{Name: name},
+		Summary:     metrics.GetOrCreateSummary(name),
+	}
+}

app/vmalert/utils/prom_duration.go

@@ -1,43 +0,0 @@
-package utils
-
-import (
-	"time"
-
-	"github.com/VictoriaMetrics/metricsql"
-)
-
-// PromDuration is Prometheus duration.
-type PromDuration struct {
-	milliseconds int64
-}
-
-// NewPromDuration returns PromDuration for given d.
-func NewPromDuration(d time.Duration) PromDuration {
-	return PromDuration{
-		milliseconds: d.Milliseconds(),
-	}
-}
-
-// MarshalYAML implements yaml.Marshaler interface.
-func (pd PromDuration) MarshalYAML() (interface{}, error) {
-	return pd.Duration().String(), nil
-}
-
-// UnmarshalYAML implements yaml.Unmarshaler interface.
-func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var s string
-	if err := unmarshal(&s); err != nil {
-		return err
-	}
-	ms, err := metricsql.DurationValue(s, 0)
-	if err != nil {
-		return err
-	}
-	pd.milliseconds = ms
-	return nil
-}
-
-// Duration returns duration for pd.
-func (pd *PromDuration) Duration() time.Duration {
-	return time.Duration(pd.milliseconds) * time.Millisecond
-}

app/vmalert/vmalert_cluster.excalidraw

@@ -0,0 +1,525 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 794,
+      "versionNonce": 1855937036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 659,
+      "versionNonce": 247957684,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347948032,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1670,
+      "versionNonce": 2021681972,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 750.3317260742188,
+      "y": 226.5509033203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348054411
+    },
+    {
+      "type": "text",
+      "version": 1311,
+      "versionNonce": 1283453068,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 759.2862243652344,
+      "y": 238.68240356445312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert:8480",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 897,
+      "versionNonce": 1983434892,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 61.744873046875,
+      "y": 224.9600830078125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638348077724
+    },
+    {
+      "type": "text",
+      "version": 719,
+      "versionNonce": 457402292,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 70.69937133789062,
+      "y": 237.33523559570312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect:8481",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1098,
+      "versionNonce": 1480603788,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 371.7434387207031,
+      "y": 398.50787353515625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 864,
+      "versionNonce": 1115813900,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 393.73809814453125,
+      "y": 410.5976257324219,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348009180,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2405,
+      "versionNonce": 959767732,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 556.6860961914062,
+      "y": 252.2582773408825,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 184.9195556640625,
+      "height": 1.6022679018915937,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.0344528515859526,
+        "gap": 10.57574462890625
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.039393828258510157,
+        "gap": 8.72607421875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.9195556640625,
+          -1.6022679018915937
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1173,
+      "versionNonce": 1248255756,
+      "isDeleted": false,
+      "id": "wRO0q9xKPHc8e8XPPsQWh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.0439244722469,
+      "y": 246.80533728741074,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 157.86774649373126,
+      "height": 0.1417938392881979,
+      "seed": 656189364,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.13736472619498497,
+        "gap": 16.306295254315614
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.013200835330936087,
+        "gap": 14.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -157.86774649373126,
+          0.1417938392881979
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 557,
+      "versionNonce": 995289780,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 552.4987182617188,
+      "y": 212.27996826171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 76,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348059525,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist alerts state\n\n\nand recording rules",
+      "baseline": 72,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 803,
+      "versionNonce": 1576507444,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 290.34130859375,
+      "y": 210.56927490234375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1471,
+      "versionNonce": 1361321140,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 484.18669893674246,
+      "y": 302.3424013553929,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 1.0484739253853945,
+      "height": 84.72775855671654,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348083348,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.010768924644894236,
+        "gap": 14.650750964767894
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.051051952959743775,
+        "gap": 11.437713623046818
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0484739253853945,
+          84.72775855671654
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 576,
+      "versionNonce": 2112088460,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 530.5612182617188,
+      "y": 318.60687255859375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348023735,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 1835119500,
+      "isDeleted": false,
+      "id": "ff5OkfgmkKLifS13_TFj3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 291.37474060058594,
+      "y": 261.4861297607422,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 264004620,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "restore state",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_ha.excalidraw

@@ -0,0 +1,911 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 906,
+      "versionNonce": 448716468,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 302.2676696777344,
+      "y": 275.59356689453125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "OTKoeHmKtqCxFArDbY-sP"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 762,
+      "versionNonce": 223660724,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 318.0538024902344,
+      "y": 296.6778106689453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2"
+      ],
+      "updated": 1638355102070,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert1",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 2067,
+      "versionNonce": 817347852,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 329.6426086425781,
+      "y": 90.3275146484375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355058507
+    },
+    {
+      "type": "text",
+      "version": 1717,
+      "versionNonce": 2040797452,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 349.01177978515625,
+      "y": 102.45901489257812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1553,
+      "versionNonce": 39381044,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 173.67257690429688,
+      "y": 470.5100402832031,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "OTKoeHmKtqCxFArDbY-sP",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 1309,
+      "versionNonce": 1112872844,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 189.667236328125,
+      "y": 482.59979248046875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager1:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 835,
+      "versionNonce": 2036483596,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 525.83837890625,
+      "y": 147.33470153808594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 777,
+      "versionNonce": 990468492,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 617.0690307617188,
+      "y": 376.9822692871094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638355227582,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1112,
+      "versionNonce": 999136652,
+      "isDeleted": false,
+      "id": "mIu-d0lmShCxzMLD5iA_p",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 492.29505920410156,
+      "y": 272.3052215576172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 756416780,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "S_dOHQrhGmu8SFJzobJK7",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "Ar-hcDLlzVSoTPs2MaywO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 970,
+      "versionNonce": 430760500,
+      "isDeleted": false,
+      "id": "ZqIR6SaLNDQl8s0zZbdnE",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 508.08119201660156,
+      "y": 293.38946533203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 1477404084,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355105020,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalertN",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "qZFezRGU4Chwxgvb2451t",
+      "type": "line",
+      "x": 449.93653869628906,
+      "y": 306.30010986328125,
+      "width": 19.48321533203125,
+      "height": 0.3865966796875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1833426828,
+      "version": 69,
+      "versionNonce": 871948300,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          19.48321533203125,
+          0.3865966796875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": null,
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": null
+    },
+    {
+      "type": "rectangle",
+      "version": 1649,
+      "versionNonce": 93981708,
+      "isDeleted": false,
+      "id": "gNHiZJKo0ap69ALDobtZ-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518.5596466064453,
+      "y": 471.4539489746094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 454422412,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "Ar-hcDLlzVSoTPs2MaywO",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 1412,
+      "versionNonce": 75038348,
+      "isDeleted": false,
+      "id": "O-zgjZBvt4RI1PrkBNQnb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 534.3148040771484,
+      "y": 483.543701171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1026268980,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanagerN:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "m9_BptFOFxbV2sS_xJDu2",
+      "type": "arrow",
+      "x": 378.57185562792466,
+      "y": 262.1596984863281,
+      "width": 79.96146539019026,
+      "height": 111.53411865234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 141224884,
+      "version": 521,
+      "versionNonce": 283254540,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          79.96146539019026,
+          -111.53411865234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.23996018586441184,
+        "gap": 13.433868408203125
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.14207100087207922,
+        "gap": 15.550811767578125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "S_dOHQrhGmu8SFJzobJK7",
+      "type": "arrow",
+      "x": 558.1990515577129,
+      "y": 263.4271240234375,
+      "width": 88.18940317574186,
+      "height": 114.15780639648438,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1004789812,
+      "version": 314,
+      "versionNonce": 1259171724,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355105019,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -88.18940317574186,
+          -114.15780639648438
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4315094049079832,
+        "gap": 8.878097534179688
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": 0.14840834302306677,
+        "gap": 14.194549560546875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "fsGFp4NW4JlrCdF0HR3uA",
+      "type": "arrow",
+      "x": 356.6613630516658,
+      "y": 354.95416259765625,
+      "width": 278.74351860741064,
+      "height": 106.90020751953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 712384692,
+      "version": 334,
+      "versionNonce": 266493324,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          278.74351860741064,
+          106.90020751953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.7721210277890177,
+        "gap": 7.228485107421875
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4493598001028791,
+        "gap": 9.599578857421875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "OTKoeHmKtqCxFArDbY-sP",
+      "type": "arrow",
+      "x": 361.02563221226757,
+      "y": 356.2252197265625,
+      "width": 95.3594006000182,
+      "height": 105.52130126953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1992252596,
+      "version": 466,
+      "versionNonce": 472553100,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355221749,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -95.3594006000182,
+          105.52130126953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.39325294425055324,
+        "gap": 8.499542236328125
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.400636314282374,
+        "gap": 8.763519287109375
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "XhPgLRBk-YhWAFcSQi9TJ",
+      "type": "arrow",
+      "x": 561.7189961327852,
+      "y": 349.238037109375,
+      "width": 266.271510370216,
+      "height": 112.383544921875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 588869260,
+      "version": 453,
+      "versionNonce": 584723212,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355150131,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -266.271510370216,
+          112.383544921875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": -0.7084007026732048,
+        "gap": 4.8007049560546875
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.4180430111580123,
+        "gap": 8.888458251953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "Ar-hcDLlzVSoTPs2MaywO",
+      "type": "arrow",
+      "x": 557.5999880535809,
+      "y": 352.71795654296875,
+      "width": 112.87813113656136,
+      "height": 106.3255615234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1508782476,
+      "version": 331,
+      "versionNonce": 1137561908,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355153683,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          112.87813113656136,
+          106.3255615234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4358123206211808,
+        "gap": 8.280624389648438
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4783744286829653,
+        "gap": 12.410430908203125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "WqHnv9_g3SdkLZuy4WSHa",
+      "type": "text",
+      "x": 186.5891876220703,
+      "y": 523.145263671875,
+      "width": 272,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1310147468,
+      "version": 281,
+      "versionNonce": 949157044,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "text": "Alertmanagers in cluster mode",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "id": "yA0kgvdF71wZbHJ7Cg2p8",
+      "type": "rectangle",
+      "x": 159.41685485839844,
+      "y": 440.7666015625,
+      "width": 625.9590148925781,
+      "height": 110.13494873046878,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1465986996,
+      "version": 366,
+      "versionNonce": 196040844,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939
+    },
+    {
+      "id": "D6fkQH1E_MFbCuL697ArO",
+      "type": "arrow",
+      "x": 422.4853057861328,
+      "y": 494.8779132311229,
+      "width": 90.5517578125,
+      "height": 0.12427004064892344,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 2025208204,
+      "version": 316,
+      "versionNonce": 2144102156,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          90.5517578125,
+          0.12427004064892344
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.08064204025505255,
+        "gap": 8.706283569335938
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": -0.05976220061952895,
+        "gap": 5.5225830078125
+      },
+      "startArrowhead": "arrow",
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 463,
+      "versionNonce": 2118914484,
+      "isDeleted": false,
+      "id": "p4EDvSKPqZzfM_SReybeq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 274.6086883544922,
+      "y": 56.77886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 394,
+      "height": 19,
+      "seed": 1214462348,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355074872,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VictoriaMetrics with deduplication enabled",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 712,
+      "versionNonce": 737833612,
+      "isDeleted": false,
+      "id": "YfNFeOucMo8BJk2P2JRex",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 229.77923583984375,
+      "y": 227.84378051757812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 448.5542297363281,
+      "height": 134.06329345703122,
+      "seed": 932207756,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355133244
+    },
+    {
+      "type": "text",
+      "version": 427,
+      "versionNonce": 1458527796,
+      "isDeleted": false,
+      "id": "oldM7Q_aJtpWd2jXQ0iKf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 238.9442901611328,
+      "y": 230.862548828125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 225,
+      "height": 38,
+      "seed": 2131899572,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355259358,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "vmalerts with identical \nconfigurations",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_multicluster.excalidraw

@@ -0,0 +1,915 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 791,
+      "versionNonce": 48874036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "pD9DcILMxa6GaR1U5YyMO",
+        "HPEwr85wL4IedW0AgdArp",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 658,
+      "versionNonce": 1653816076,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347796775,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 802,
+      "versionNonce": 995326644,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.05322265625,
+      "y": 228.65371704101562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 624,
+      "versionNonce": 707755700,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 640.7635803222656,
+      "y": 241.02886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701075,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 802,
+      "versionNonce": 1974403340,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 401.241943359375,
+      "y": 342.4627990722656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 178,
+      "height": 38,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347863144,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute aggregating\nrecording rules",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "jrFZeFNsxlss7IsEZpA-J",
+      "type": "rectangle",
+      "x": 594.1509857177734,
+      "y": 192.09194946289062,
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 48379060,
+      "version": 665,
+      "versionNonce": 617456652,
+      "isDeleted": false,
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347763716
+    },
+    {
+      "id": "6ibhLp94HJFIdfP5HCEv8",
+      "type": "text",
+      "x": 609.7205657958984,
+      "y": 199.81723022460938,
+      "width": 225,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1053164852,
+      "version": 256,
+      "versionNonce": 538595124,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347761092,
+      "text": "VM cluster with raw data",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "type": "rectangle",
+      "version": 914,
+      "versionNonce": 1576666164,
+      "isDeleted": false,
+      "id": "R5v-yZCVJ97BkJqr0Qb7H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 807.5664215087891,
+      "y": 287.8628387451172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1794212620,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "bZXA8PH9gYu-clotqJ4f7",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 725,
+      "versionNonce": 267455500,
+      "isDeleted": false,
+      "id": "pWRC_smX7TuOI8_8UrA4H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 840.0209197998047,
+      "y": 300.2379913330078,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 421856180,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 843,
+      "versionNonce": 1244193972,
+      "isDeleted": false,
+      "id": "EqROOfYulSPsZm7ovxfQN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 604.9091339111328,
+      "y": 345.2847442626953,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 2043521972,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347718537
+    },
+    {
+      "type": "text",
+      "version": 676,
+      "versionNonce": 143370892,
+      "isDeleted": false,
+      "id": "ddQH1nnmT7HbKW7Xmv4zx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 642.6199798583984,
+      "y": 357.90354919433594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 335223180,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "bZXA8PH9gYu-clotqJ4f7",
+      "type": "arrow",
+      "x": 785.2667708463345,
+      "y": 367.2342882272049,
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1153029388,
+      "version": 420,
+      "versionNonce": 1726025228,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347704644,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "EqROOfYulSPsZm7ovxfQN",
+        "focus": 0.5247890891198189,
+        "gap": 8.36404562660789
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": -0.6931056940383433,
+        "gap": 9.943033572650961
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "arrow",
+      "version": 528,
+      "versionNonce": 1908259468,
+      "isDeleted": false,
+      "id": "MGdu6GDIPNBAaEUr0Gt-a",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 781.4456641707259,
+      "y": 248.777857603323,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1695061516,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347737174,
+      "startBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 883,
+      "versionNonce": 845352076,
+      "isDeleted": false,
+      "id": "4pW8hvBu3bo1eMtFvg_gS",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 601.6520690917969,
+      "y": 473.7677536010742,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1678097076,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 703,
+      "versionNonce": 519371956,
+      "isDeleted": false,
+      "id": "w_i2hO06oLa0bWbyAfFzU",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 639.3624267578125,
+      "y": 486.14290618896484,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 340753036,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 745,
+      "versionNonce": 879581324,
+      "isDeleted": false,
+      "id": "U5U-67wL5fPwvzlxOAF5A",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 592.7498321533203,
+      "y": 437.2059860229492,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "seed": 912540724,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347776748
+    },
+    {
+      "type": "text",
+      "version": 345,
+      "versionNonce": 1628116148,
+      "isDeleted": false,
+      "id": "0IGVrvICMVeZp2RCaqTeP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 608.3194122314453,
+      "y": 444.93126678466797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 291,
+      "height": 19,
+      "seed": 68700428,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347784373,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VM cluster with aggregated data",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 996,
+      "versionNonce": 2112461580,
+      "isDeleted": false,
+      "id": "9QMf0HXPE1W4M9S-zMJVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 806.1652679443359,
+      "y": 532.9768753051758,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 523607476,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "9WBH34em4CdU2OwzqYIl5",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 804,
+      "versionNonce": 1660832052,
+      "isDeleted": false,
+      "id": "WOrOD5vn6EtMUQRJomt3-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 838.6197662353516,
+      "y": 545.3520278930664,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 1321420684,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 925,
+      "versionNonce": 2052807604,
+      "isDeleted": false,
+      "id": "4dtJZpXEUxSK3biwFG7vd",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.5079803466797,
+      "y": 590.3987808227539,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1738524468,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "9WBH34em4CdU2OwzqYIl5",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 756,
+      "versionNonce": 2040967820,
+      "isDeleted": false,
+      "id": "rAbyooo-X08-86qjoK0WR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 641.2188262939453,
+      "y": 603.0175857543945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 948598284,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "9WBH34em4CdU2OwzqYIl5"
+      ],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 660,
+      "versionNonce": 1560678196,
+      "isDeleted": false,
+      "id": "9WBH34em4CdU2OwzqYIl5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.8656172818814,
+      "y": 612.3483247872634,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "seed": 1141424308,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": 0.5247890891198189,
+        "gap": 8.364045626608004
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": -0.6931056940383404,
+        "gap": 9.943033572650847
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 768,
+      "versionNonce": 1653264948,
+      "isDeleted": false,
+      "id": "5W90eDBjtfZvkSuQTG0Iw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.0445106062728,
+      "y": 493.8918941633816,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1852298380,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4pW8hvBu3bo1eMtFvg_gS",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "type": "arrow",
+      "x": 423.70701599121094,
+      "y": 428.34434509277344,
+      "width": 179.54901123046875,
+      "height": 182.9937744140625,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 389863732,
+      "version": 47,
+      "versionNonce": 1364399028,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347805500,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          179.54901123046875,
+          -182.9937744140625
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "type": "arrow",
+      "x": 424.7585906982422,
+      "y": 441.12806701660156,
+      "width": 174.29449462890625,
+      "height": 170.56607055664062,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 981082124,
+      "version": 92,
+      "versionNonce": 1261546252,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347812431,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          174.29449462890625,
+          170.56607055664062
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": -0.8207814548026305,
+        "gap": 4.45489501953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 775,
+      "versionNonce": 95530764,
+      "isDeleted": false,
+      "id": "o-yIJ_WZzVubhbVODvhcC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 407.27012634277344,
+      "y": 489.33091735839844,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 775116812,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347824876,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_single.excalidraw

@@ -0,0 +1,354 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 795,
+      "versionNonce": 1195460364,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 660,
+      "versionNonce": 1034125236,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1690,
+      "versionNonce": 236788620,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 344.9837951660156,
+      "y": 64.64248657226562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348176044
+    },
+    {
+      "type": "text",
+      "version": 1331,
+      "versionNonce": 945335476,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 364.35296630859375,
+      "y": 76.77398681640625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1099,
+      "versionNonce": 671872012,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 360.6495666503906,
+      "y": 382.2536315917969,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 865,
+      "versionNonce": 635839156,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 382.64422607421875,
+      "y": 394.3433837890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2444,
+      "versionNonce": 361351692,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.0465703465111,
+      "y": 204.98379516601562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.7962088410123442,
+      "height": 86.86798095703125,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "gap": 10.57574462890625,
+        "focus": 0.0344528515859526
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "gap": 8.72607421875,
+        "focus": -0.039393828258510157
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.7962088410123442,
+          -86.86798095703125
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 640,
+      "versionNonce": 1892951180,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 500.609619140625,
+      "y": 134.11544799804688,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348163424,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1472,
+      "versionNonce": 768565516,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.5245361328125,
+      "y": 300.5478957313172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.3521007656264601,
+      "height": 70.26802223743277,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "startBinding": {
+        "elementId": "E9Run6wCm2chQ6JHrmc_y",
+        "focus": 1.1925203824459496,
+        "gap": 11.77154541015625
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.0441077573536454,
+        "gap": 11.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.3521007656264601,
+          70.26802223743277
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 577,
+      "versionNonce": 1646003636,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 498.29608154296875,
+      "y": 298.6573791503906,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/web.go

@@ -4,40 +4,84 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var (
+	once     = sync.Once{}
+	apiLinks [][2]string
+	navItems []tpl.NavItem
+)
+
+func initLinks() {
+	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
+	apiLinks = [][2]string{
+		{path.Join(pathPrefix, "api/v1/rules"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
+		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
+		{path.Join(pathPrefix, "flags"), "command-line flags"},
+		{path.Join(pathPrefix, "metrics"), "list of application metrics"},
+		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
+	}
+	navItems = []tpl.NavItem{
+		{Name: "vmalert", Url: path.Join(pathPrefix, "/")},
+		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
+		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
+		{Name: "Notifiers", Url: path.Join(pathPrefix, "notifiers")},
+		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
+	}
+}
+
 type requestHandler struct {
 	m *manager
 }
 
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	once.Do(func() {
+		initLinks()
+	})
+
+	pathPrefix := httpserver.GetPathPrefix()
+	if pathPrefix == "" {
+		pathPrefix = "/"
+	}
+
 	switch r.URL.Path {
 	case "/":
 		if r.Method != "GET" {
 			return false
 		}
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/api/v1/groups", "list all loaded groups and rules"},
-			{"/api/v1/alerts", "list all active alerts"},
-			{"/api/v1/groupID/alertID/status", "get alert status by ID"},
-			{"/metrics", "list of application metrics"},
-			{"/-/reload", "reload configuration"},
-		})
+		WriteWelcome(w)
 		return true
-	case "/api/v1/groups":
+	case "/alerts":
+		WriteListAlerts(w, pathPrefix, rh.groupAlerts())
+		return true
+	case "/groups":
+		WriteListGroups(w, rh.groups())
+		return true
+	case "/notifiers":
+		WriteListTargets(w, notifier.GetTargets())
+		return true
+	case "/api/v1/rules":
 		data, err := rh.listGroups()
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
@@ -46,7 +90,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -58,39 +102,57 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		if !strings.HasSuffix(r.URL.Path, "/status") {
 			return false
 		}
-		// /api/v1/<groupName>/<alertID>/status
-		data, err := rh.alert(r.URL.Path)
+		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.Write(data)
+
+		// /api/v1/<groupID>/<alertID>/status
+		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
+			data, err := json.Marshal(alert)
+			if err != nil {
+				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+				return true
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.Write(data)
+			return true
+		}
+
+		// <groupID>/<alertID>/status
+		WriteAlert(w, pathPrefix, alert)
 		return true
 	}
 }
 
 type listGroupsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Groups []APIGroup `json:"groups"`
 	} `json:"data"`
-	Status string `json:"status"`
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) groups() []APIGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	lr := listGroupsResponse{Status: "success"}
+	var groups []APIGroup
 	for _, g := range rh.m.groups {
-		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+		groups = append(groups, g.toAPI())
 	}
 
 	// sort list of alerts for deterministic output
-	sort.Slice(lr.Data.Groups, func(i, j int) bool {
-		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	sort.Slice(groups, func(i, j int) bool {
+		return groups[i].Name < groups[j].Name
 	})
 
+	return groups
+}
+
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	lr := listGroupsResponse{Status: "success"}
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{
@@ -102,10 +164,34 @@ func (rh *requestHandler) listGroups() ([]byte, error) {
 }
 
 type listAlertsResponse struct {
-	Data struct {
+	Status string `json:"status"`
+	Data   struct {
 		Alerts []*APIAlert `json:"alerts"`
 	} `json:"data"`
-	Status string `json:"status"`
+}
+
+func (rh *requestHandler) groupAlerts() []GroupAlerts {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	var groupAlerts []GroupAlerts
+	for _, g := range rh.m.groups {
+		var alerts []*APIAlert
+		for _, r := range g.Rules {
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			alerts = append(alerts, a.AlertsToAPI()...)
+		}
+		if len(alerts) > 0 {
+			groupAlerts = append(groupAlerts, GroupAlerts{
+				Group:  g.toAPI(),
+				Alerts: alerts,
+			})
+		}
+	}
+	return groupAlerts
 }
 
 func (rh *requestHandler) listAlerts() ([]byte, error) {
@@ -119,7 +205,7 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 			if !ok {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
 		}
 	}
 
@@ -138,18 +224,17 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
-func (rh *requestHandler) alert(path string) ([]byte, error) {
+func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", 3)
 	if len(parts) != 3 {
 		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
 			StatusCode: http.StatusBadRequest,
 		}
 	}
-
 	groupID, err := uint64FromPath(parts[0])
 	if err != nil {
 		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
@@ -162,7 +247,7 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 	if err != nil {
 		return nil, errResponse(err, http.StatusNotFound)
 	}
-	return json.Marshal(resp)
+	return resp, nil
 }
 
 func uint64FromPath(path string) (uint64, error) {