docs/CHANGELOG.md: cut v1.49.0

This fixes issues found in Go since v1.15.5 - see https://github.com/golang/go/issues?q=milestone%3AGo1.15.6+label%3ACherryPickApproved

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@master
+        uses: actions/setup-go@main
         with:
-          go-version: 1.14
+          go-version: 1.15
         id: go
       - name: Dependencies
         env:
@@ -24,7 +24,7 @@ jobs:
         run: |
           go get -u golang.org/x/lint/golint
           go get -u github.com/kisielk/errcheck
-          go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
       - name: Code checkout
         uses: actions/checkout@master
       - name: Build
@@ -43,7 +43,20 @@ jobs:
           make victoria-metrics-arm64
           make vmutils
           GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=freebsd go build -mod=vendor ./app/vmagent
+          GOOS=freebsd go build -mod=vendor ./app/vmalert
+          GOOS=freebsd go build -mod=vendor ./app/vmbackup
+          GOOS=freebsd go build -mod=vendor ./app/vmrestore
+          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=openbsd go build -mod=vendor ./app/vmagent
+          GOOS=openbsd go build -mod=vendor ./app/vmalert
+          GOOS=openbsd go build -mod=vendor ./app/vmbackup
+          GOOS=openbsd go build -mod=vendor ./app/vmrestore
           GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/vmagent
+          GOOS=darwin go build -mod=vendor ./app/vmalert
+          GOOS=darwin go build -mod=vendor ./app/vmbackup
+          GOOS=darwin go build -mod=vendor ./app/vmrestore
       - name: Publish coverage
         uses: codecov/codecov-action@v1.0.6
         with:

Makefile

@@ -122,8 +122,8 @@ benchmark-pure:
 	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
-	GO111MODULE=on go get -u ./lib/...
-	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go get -u -d ./lib/...
+	GO111MODULE=on go get -u -d ./app/...
 	GO111MODULE=on go mod tidy
 	GO111MODULE=on go mod vendor
 
@@ -133,6 +133,9 @@ app-local:
 app-local-pure:
 	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
+app-local-with-goarch:
+	GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
 quicktemplate-gen: install-qtc
 	qtc
 
@@ -145,3 +148,11 @@ golangci-lint: install-golangci-lint
 
 install-golangci-lint:
 	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+
+docs-sync:
+	cp app/vmagent/README.md docs/vmagent.md
+	cp app/vmalert/README.md docs/vmalert.md
+	cp app/vmauth/README.md docs/vmauth.md
+	cp app/vmbackup/README.md docs/vmbackup.md
+	cp app/vmrestore/README.md docs/vmrestore.md
+	cp README.md docs/Single-server-VictoriaMetrics.md

app/victoria-metrics/Makefile

@@ -108,3 +108,10 @@ victoria-metrics-package-deb-rpm-all: \
         victoria-metrics-package-deb-arm64 \
         victoria-metrics-package-rpm \
         victoria-metrics-package-rpm-arm64
+
+### Packaging as snap
+victoria-metrics-package-snap:
+	which snapcraft || snap install snapcraft
+	which multipass || snap install multipass
+	snapcraft
+

app/victoria-metrics/main.go

@@ -2,18 +2,22 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -22,12 +26,29 @@ var (
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
 		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
 		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
+		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
 
 func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	cgroup.UpdateGOMAXPROCSToCPUQuota()
+
+	if promscrape.IsDryRun() {
+		*dryRun = true
+	}
+	if *dryRun {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		return
+	}
+
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
@@ -61,6 +82,10 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+		return true
+	}
 	if vminsert.RequestHandler(w, r) {
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -373,7 +373,7 @@ func checkMetricsResult(got, want []Metric) error {
 		want = removeIfFoundMetrics(r, want)
 	}
 	if len(want) > 0 {
-		return fmt.Errorf("exptected metrics %+v not found in %+v", want, got)
+		return fmt.Errorf("expected metrics %+v not found in %+v", want, got)
 	}
 	return nil
 }

app/victoria-metrics/self_scraper.go

@@ -85,7 +85,6 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr.Timestamp = currentTimestamp
 			mr.Value = r.Value
 		}
-		logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp)
 		vmstorage.AddRows(mrs)
 	}
 }

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -5,12 +5,12 @@
     "empty_label_match 1 {TIME_S-1m}",
     "empty_label_match;foo=bar 2 {TIME_S-1m}",
     "empty_label_match;foo=baz 3 {TIME_S-1m}"],
-  "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S}&end={TIME_S}&step=60"],
+  "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
   "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[
-	      {"metric":{"__name__":"empty_label_match"},"values":[["{TIME_S}","1"]]},
-	      {"metric":{"__name__":"empty_label_match","foo":"bar"},"values":[["{TIME_S}","2"]]}
+	      {"metric":{"__name__":"empty_label_match"},"values":[["{TIME_S-1m}","1"],["{TIME_S}","1"]]},
+	      {"metric":{"__name__":"empty_label_match","foo":"bar"},"values":[["{TIME_S-1m}","2"],["{TIME_S}","2"]]}
       ]}}
 }

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -16,6 +16,8 @@
 	      ["{TIME_S-120s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-30s}","1"],
-	      ["{TIME_S-20s}","1"]
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S-0s}","1"]
       ]}]}}
 }

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -9,6 +9,6 @@
   "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
   "result_query": {
     "status":"success",
-    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","1"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","3"]}]}
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","2"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","4"]}]}
   }
 }

app/vmagent/Makefile

@@ -52,25 +52,29 @@ publish-vmagent:
 	APP_NAME=vmagent $(MAKE) publish-via-docker
 
 run-vmagent:
-	mkdir -p vmagent-data
-	DOCKER_OPTS='-v $(shell pwd)/vmagent-data:/vmagent-data' \
+	mkdir -p vmagent-remotewrite-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-remotewrite-data:/vmagent-remotewrite-data' \
+	ARGS='-remoteWrite.url=http://localhost:8428/api/v1/write' \
 	APP_NAME=vmagent \
 	$(MAKE) run-via-docker
 
 vmagent-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
+	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmagent-local-with-goarch
 
 vmagent-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
+	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmagent-local-with-goarch
 
 vmagent-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
+	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmagent-local-with-goarch
 
 vmagent-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
+	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmagent-local-with-goarch
 
 vmagent-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
+	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmagent-local-with-goarch
+
+vmagent-local-with-goarch:
+	APP_NAME=vmagent $(MAKE) app-local-with-goarch
 
 vmagent-pure:
 	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -1,8 +1,8 @@
 ## vmagent
 
-`vmagent` is a tiny but brave agent, which helps you collecting metrics from various sources
-and storing them to [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
-or any other Prometheus-compatible storage system that supports `remote_write` protocol.
+`vmagent` is a tiny but brave agent, which helps you collect metrics from various sources
+and stores them in [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports the `remote_write` protocol.
 
 <img alt="vmagent" src="vmagent.png">
 
@@ -11,7 +11,7 @@ or any other Prometheus-compatible storage system that supports `remote_write` p
 
 While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
 and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
-Also, we found that users’ infrastructure is like snowflakes - never alike, and we decided to add more flexibility
+Also, we found that users’ infrastructure are snowflakes - no two are alike, and we decided to add more flexibility
 to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
 
 
@@ -25,13 +25,15 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-native-format).
+  * Data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
 * Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
   to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
-* Uses lower amounts of RAM, CPU, disk IO and network bandwidth comparing to Prometheus.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared to Prometheus.
 
 
 ### Quick Start
@@ -40,8 +42,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. Multiple `-remoteWrite.url` args can be set in parallel
-  in order to replicate data concurrently to multiple remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
 
@@ -49,7 +50,7 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-If you need collecting only Influx data, then the following command line would be enough:
+If you only need to collect Influx data, then the following is sufficient:
 
 ```
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
@@ -62,6 +63,22 @@ Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://gi
 Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
 
 
+### Configuration update
+
+`vmagent` should be restarted in order to update config options set via command-line args.
+
+`vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
+
+* Sending `SUGHUP` signal to `vmagent` process:
+  ```bash
+  kill -SIGHUP `pidof vmagent`
+  ```
+
+* Sending HTTP request to `http://vmagent:8429/-/reload` endpoint.
+
+There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
+
+
 ### Use cases
 
 
@@ -79,14 +96,14 @@ See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/Victor
 #### Drop-in replacement for Prometheus
 
 If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
-then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such setup.
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such a setup.
 See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
 
 
 #### Replication and high availability
 
 `vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
-If a single remote storage instance temporarily goes out of service, then the collected data remains available in another remote storage instances.
+If a single remote storage instance temporarily is out of service, then the collected data remains available in another remote storage instances.
 `vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
 Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
 
@@ -94,13 +111,13 @@ Then it sends the buffered data to the remote storage in order to prevent data g
 #### Relabeling and filtering
 
 `vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
-it can remove unneeded samples via Prometheus-like relabeling before sending the collected data to remote storage.
+it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
 See [these docs](#relabeling) for details.
 
 
 #### Splitting data streams among multiple systems
 
-`vmagent` supports splitting of the collected data among muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+`vmagent` supports splitting the collected data between muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
 which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
 data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
 Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
@@ -135,7 +152,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `ec2_sd_configs` - for scraping targets in Amazon EC2.
   See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
-  `vmagent` doesn't support `role_arn` config param yet.
+  `vmagent` doesn't support `profile` config param and aws credentials file yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
   `vmagent` provides the following additional functionality for `gce_sd_config`:
@@ -147,10 +164,29 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
 * `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
   See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
-
+* `openstack_sd_configs` - for scraping OpenStack targets.
+  See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
+  [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
+  See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
+* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
+  See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
 
 File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
+`vmagent` also support the following additional options in `scrape_config` section:
+
+* `disable_compression: true` - for disabling response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets
+  in order to save network bandwidth.
+* `disable_keepalive: true` - for disabling [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
+  By default `vmagent` uses keep-alive connections to scrape targets in order to reduce overhead on connection re-establishing.
+
+Note that `vmagent` doesn't support `refresh_interval` option these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
+command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
+entries to 60s. Run `vmagent -help` in order to see default values for `-promscrape.*CheckInterval` flags.
+
+The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
+
 
 ### Adding labels to metrics
 
@@ -167,6 +203,8 @@ Additionally it provides the following extra actions:
 
 * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
 * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal.
+* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
 
 The relabeling can be defined in the following places:
 
@@ -177,6 +215,7 @@ The relabeling can be defined in the following places:
 
 Read more about relabeling in the following articles:
 
+* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
 * [Life of a label](https://www.robustperception.io/life-of-a-label)
 * [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
 * [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
@@ -188,24 +227,92 @@ Read more about relabeling in the following articles:
 
 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
 either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+If you have suggestions, improvements or found a bug - feel free to open an issue on github or add review to the dashboard.
 
-`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`vmagent` also exports target statuses at the following handlers:
+
+* `http://vmagent-host:8429/targets`. This handler returns human-readable plaintext status for every active target.
+This page is convenient to query from command line with `wget`, `curl` or similar tools.
+It accepts optional `show_original_labels=1` query arg, which shows the original labels per each target before applying relabeling.
+This information may be useful for debugging target relabeling.
+* `http://vmagent-host:8429/api/v1/targets`. This handler returns data compatible with [the corresponding page from Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/#targets).
+
+* `http://vmagent-host:8429/ready`. This handler returns http 200 status code when `vmagent` finishes initialization for all service_discovery configs.
+It may be useful for performing `vmagent` rolling update without scrape loss.
 
 
 ### Troubleshooting
 
+* It is recommended [setting up the official Grafana dashboard](#monitoring) in order to monitor `vmagent` state.
+
 * It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
   since `vmagent` establishes at least a single TCP connection per each target.
 
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
-  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
+  and `http://vmagent-host:8429/api/v1/targets`.
 
-* It is recommended increasing `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
-  and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
+  This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
+
+* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
+  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+
+* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
+  then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
+  by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
+
+  ```yml
+  scrape_configs:
+  - job_name: 'big-federate'
+    stream_parse: true
+    static_configs:
+    - targets:
+      - big-prometeus1
+      - big-prometeus2
+    honor_labels: true
+    metrics_path: /federate
+    params:
+      'match[]': ['{__name__!=""}']
+  ```
+
+* It is recommended to increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+
+* If you see gaps on the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, then try increasing `-remoteWrite.queues`.
+  Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage, so it starts dropping the buffered data
+  if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
 
 * `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
-  The directory can grow big when remote storage is unavailable during extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
-  If you don't want sending all the data from the directory to remote storage, just stop `vmagent` and delete the directory.
+  The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
+
+* By default `vmagent` masks `-remoteWrite.url` with `secret-url` values in logs and at `/metrics` page because
+  the url may contain sensitive information such as auth tokens or passwords.
+  Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
+
+* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
+  or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
+  See available options below if you prefer fixing the root cause of the error:
+
+  The following `relabel_configs` section may help determining `__meta_*` labels resulting in duplicate targets:
+  ```yml
+  - action: labelmap
+    regex: __meta_(.*)
+  ```
+
+  The following relabeling rule may be added to `relabel_configs` section in order to filter out pods with unneeded ports:
+  ```yml
+  - action: keep_if_equal
+    source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]
+  ```
+
+  The following relabeling rule may be added to `relabel_configs` section in order to filter out init container pods:
+  ```yml
+  - action: drop
+    source_labels: [__meta_kubernetes_pod_container_init]
+    regex: true
+  ```
 
 
 ### How to build from sources
@@ -231,13 +338,29 @@ Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 
-By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
-by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
-ROOT_IMAGE=alpine:3.11 make package-vmagent
+ROOT_IMAGE=scratch make package-vmagent
 ```
 
+#### ARM build
+
+ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
+
+#### Development ARM build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmagent-arm` or `make vmagent-arm64` from the root folder of the repository.
+   It builds `vmagent-arm` or `vmagent-arm64` binary respectively and puts it into the `bin` folder.
+
+#### Production ARM build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-arm-prod` or `make vmagent-arm64-prod` from the root folder of the repository.
+   It builds `vmagent-arm-prod` or `vmagent-arm64-prod` binary respectively and puts it into the `bin` folder.
+
 
 ### Profiling
 

app/vmagent/common/push_ctx.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // PushCtx is a context used for populating WriteRequest.
@@ -28,12 +29,7 @@ func (ctx *PushCtx) Reset() {
 	}
 	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
 
-	labels := ctx.Labels
-	for i := range labels {
-		label := &labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(ctx.Labels)
 	ctx.Labels = ctx.Labels[:0]
 
 	ctx.Samples = ctx.Samples[:0]

app/vmagent/csvimport/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -18,12 +19,18 @@ var (
 
 // InsertHandler processes csv data from req.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, insertRows)
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -44,6 +51,7 @@ func insertRows(rows []parser.Row) error {
 				Value: tag.Value,
 			})
 		}
+		labels = append(labels, extraLabels...)
 		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,

app/vmagent/influx/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -19,6 +20,7 @@ import (
 var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
 	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 )
 
 var (
@@ -61,25 +63,29 @@ func insertRows(db string, rows []parser.Row) error {
 	buf := ctx.buf[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		commonLabels = commonLabels[:0]
-		hasDBLabel := false
+		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
 			if tag.Key == "db" {
-				hasDBLabel = true
+				hasDBKey = true
 			}
 			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		if len(db) > 0 && !hasDBLabel {
+		if len(db) > 0 && !hasDBKey {
 			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  "db",
 				Value: db,
 			})
 		}
-		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+		if !*skipMeasurement {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, r.Measurement...)
+		}
 		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
 		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
 			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
@@ -107,7 +113,6 @@ func insertRows(db string, rows []parser.Row) error {
 				Samples: samples[len(samples)-1:],
 			})
 		}
-		rowsTotal += len(r.Fields)
 	}
 	ctx.buf = buf
 	ctx.ctx.WriteRequest.Timeseries = tssDst
@@ -131,12 +136,8 @@ type pushCtx struct {
 func (ctx *pushCtx) reset() {
 	ctx.ctx.Reset()
 
-	commonLabels := ctx.commonLabels
-	for i := range commonLabels {
-		label := &commonLabels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(ctx.commonLabels)
+	ctx.commonLabels = ctx.commonLabels[:0]
 
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
 	ctx.buf = ctx.buf[:0]

app/vmagent/main.go

@@ -4,19 +4,26 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
@@ -25,6 +32,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -33,12 +41,16 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to `http://<vmagent>:8429/write`")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
+		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
 
 var (
@@ -49,12 +61,37 @@ var (
 )
 
 func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
 	envflag.Parse()
+	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	cgroup.UpdateGOMAXPROCSToCPUQuota()
+
+	if promscrape.IsDryRun() {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		return
+	}
+	if *dryRun {
+		if err := remotewrite.CheckRelabelConfigs(); err != nil {
+			logger.Fatalf("error when checking relabel configs: %s", err)
+		}
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("all the configs are ok; exitting with 0 status code")
+		return
+	}
+
 	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
 	remotewrite.Init()
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -102,19 +139,24 @@ func main() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
+		return true
+	}
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -123,7 +165,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -132,7 +174,25 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -141,7 +201,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -154,14 +214,31 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		return true
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/ready":
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
+			http.Error(w, errMsg, http.StatusTooEarly)
+		} else {
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("OK"))
+		}
+		return true
 	}
 	return false
 }
@@ -176,12 +253,28 @@ var (
 	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 
+	prometheusimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+	prometheusimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+
+	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
 
-	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
+
+func usage() {
+	const s = `
+vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md .
+`
+	flagutil.Usage(s)
+}

app/vmagent/native/request_handler.go

@@ -0,0 +1,85 @@
+package native
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	mn := &block.MetricName
+	labelsLen := len(labels)
+	labels = append(labels, prompbmarshal.Label{
+		Name:  "__name__",
+		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
+	})
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		labels = append(labels, prompbmarshal.Label{
+			Name:  bytesutil.ToUnsafeString(tag.Key),
+			Value: bytesutil.ToUnsafeString(tag.Value),
+		})
+	}
+	labels = append(labels, extraLabels...)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	samplesLen := len(samples)
+	for j, value := range values {
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     value,
+			Timestamp: timestamps[j],
+		})
+	}
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
+		Labels:  labels[labelsLen:],
+		Samples: samples[samplesLen:],
+	})
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	return nil
+}

app/vmagent/prometheusimport/request_handler.go

@@ -0,0 +1,76 @@
+package prometheusimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+)
+
+// InsertHandler processes `/api/v1/import/prometheus` request.
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
+	})
+}
+
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,7 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
 		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
 		labelsLen := len(labels)
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
@@ -55,7 +56,6 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(ts.Samples)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmagent/remotewrite/client.go

@@ -1,10 +1,14 @@
 package remotewrite
 
 import (
+	"bytes"
 	"crypto/tls"
 	"encoding/base64"
 	"flag"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -13,12 +17,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/fasthttp"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. "+
+		"Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
@@ -39,24 +44,47 @@ var (
 )
 
 type client struct {
-	urlLabelValue  string
+	sanitizedURL   string
 	remoteWriteURL string
-	host           string
-	requestURI     string
 	authHeader     string
 	fq             *persistentqueue.FastQueue
-	hc             *fasthttp.HostClient
+	hc             *http.Client
 
 	requestDuration *metrics.Histogram
 	requestsOKCount *metrics.Counter
 	errorsCount     *metrics.Counter
+	packetsDropped  *metrics.Counter
 	retriesCount    *metrics.Counter
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	tlsCfg, err := getTLSConfig(argIdx)
+	if err != nil {
+		logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+	}
+	tr := &http.Transport{
+		Dial:                statDial,
+		TLSClientConfig:     tlsCfg,
+		TLSHandshakeTimeout: 5 * time.Second,
+		MaxConnsPerHost:     2 * concurrency,
+		MaxIdleConnsPerHost: 2 * concurrency,
+		IdleConnTimeout:     time.Minute,
+		WriteBufferSize:     64 * 1024,
+	}
+	pURL := proxyURL.GetOptionalArg(argIdx)
+	if len(pURL) > 0 {
+		if !strings.Contains(pURL, "://") {
+			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
+		}
+		urlProxy, err := url.Parse(pURL)
+		if err != nil {
+			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
+		}
+		tr.Proxy = http.ProxyURL(urlProxy)
+	}
 	authHeader := ""
 	username := basicAuthUsername.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
@@ -69,72 +97,26 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 	token := bearerToken.GetOptionalArg(argIdx)
 	if len(token) > 0 {
 		if authHeader != "" {
-			logger.Panicf("FATAL: `-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", token)
+			logger.Fatalf("`-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", token)
 		}
 		authHeader = "Bearer " + token
 	}
-
-	readTimeout := *sendTimeout
-	if readTimeout <= 0 {
-		readTimeout = time.Minute
-	}
-	writeTimeout := readTimeout
-	var u fasthttp.URI
-	u.Update(remoteWriteURL)
-	scheme := string(u.Scheme())
-	switch scheme {
-	case "http", "https":
-	default:
-		logger.Panicf("FATAL: unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
-	}
-	host := string(u.Host())
-	if len(host) == 0 {
-		logger.Panicf("FATAL: invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
-	}
-	requestURI := string(u.RequestURI())
-	isTLS := scheme == "https"
-	var tlsCfg *tls.Config
-	if isTLS {
-		var err error
-		tlsCfg, err = getTLSConfig(argIdx)
-		if err != nil {
-			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
-		}
-	}
-	if !strings.Contains(host, ":") {
-		if isTLS {
-			host += ":443"
-		} else {
-			host += ":80"
-		}
-	}
-	maxConns := 2 * concurrency
-	hc := &fasthttp.HostClient{
-		Addr:                host,
-		Name:                "vmagent",
-		Dial:                statDial,
-		IsTLS:               isTLS,
-		TLSConfig:           tlsCfg,
-		MaxConns:            maxConns,
-		MaxIdleConnDuration: 10 * readTimeout,
-		ReadTimeout:         readTimeout,
-		WriteTimeout:        writeTimeout,
-		MaxResponseBodySize: 1024 * 1024,
-	}
 	c := &client{
-		urlLabelValue:  urlLabelValue,
+		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
-		host:           host,
-		requestURI:     requestURI,
 		authHeader:     authHeader,
 		fq:             fq,
-		hc:             hc,
-		stopCh:         make(chan struct{}),
+		hc: &http.Client{
+			Transport: tr,
+			Timeout:   *sendTimeout,
+		},
+		stopCh: make(chan struct{}),
 	}
-	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
-	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
-	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
-	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
+	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -142,27 +124,30 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 			c.runWorker()
 		}()
 	}
-	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 	return c
 }
 
 func (c *client) MustStop() {
 	close(c.stopCh)
 	c.wg.Wait()
-	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
 }
 
 func getTLSConfig(argIdx int) (*tls.Config, error) {
-	tlsConfig := &promauth.TLSConfig{
+	c := &promauth.TLSConfig{
 		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
 		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
 		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
 		ServerName:         tlsServerName.GetOptionalArg(argIdx),
 		InsecureSkipVerify: *tlsInsecureSkipVerify,
 	}
-	cfg, err := promauth.NewConfig(".", nil, "", "", tlsConfig)
+	if c.CAFile == "" && c.CertFile == "" && c.KeyFile == "" && c.ServerName == "" && !c.InsecureSkipVerify {
+		return nil, nil
+	}
+	cfg, err := promauth.NewConfig(".", nil, "", "", c)
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate TLS config: %s", err)
+		return nil, fmt.Errorf("cannot populate TLS config: %w", err)
 	}
 	tlsCfg := cfg.NewTLSConfig()
 	return tlsCfg, nil
@@ -193,7 +178,7 @@ func (c *client) runWorker() {
 				// The block has been sent successfully.
 			case <-time.After(graceDuration):
 				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
-					len(block), c.remoteWriteURL, graceDuration.Seconds())
+					len(block), c.sanitizedURL, graceDuration.Seconds())
 			}
 			return
 		}
@@ -201,32 +186,25 @@ func (c *client) runWorker() {
 }
 
 func (c *client) sendBlock(block []byte) {
-	req := fasthttp.AcquireRequest()
-	req.SetRequestURI(c.requestURI)
-	req.SetHost(c.host)
-	req.Header.SetMethod("POST")
-	req.Header.Add("Content-Type", "application/x-protobuf")
-	req.Header.Add("Content-Encoding", "snappy")
-	req.Header.Add("X-Prometheus-Remote-Write-Version", "0.1.0")
+	retryDuration := time.Second
+	retriesCount := 0
+
+again:
+	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
+	if err != nil {
+		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+	}
+	h := req.Header
+	h.Set("User-Agent", "vmagent")
+	h.Set("Content-Type", "application/x-protobuf")
+	h.Set("Content-Encoding", "snappy")
+	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 	if c.authHeader != "" {
 		req.Header.Set("Authorization", c.authHeader)
 	}
-	req.SetBody(block)
-
-	retryDuration := time.Second
-	resp := fasthttp.AcquireResponse()
-
-again:
-	select {
-	case <-c.stopCh:
-		fasthttp.ReleaseRequest(req)
-		fasthttp.ReleaseResponse(resp)
-		return
-	default:
-	}
 
 	startTime := time.Now()
-	err := doRequestWithPossibleRetry(c.hc, req, resp)
+	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
@@ -235,40 +213,56 @@ again:
 			retryDuration = time.Minute
 		}
 		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
-		time.Sleep(retryDuration)
-		c.retriesCount.Inc()
-		goto again
-	}
-	statusCode := resp.StatusCode()
-	if statusCode/100 != 2 {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
-		retryDuration *= 2
-		if retryDuration > time.Minute {
-			retryDuration = time.Minute
+			len(block), c.sanitizedURL, err, retryDuration.Seconds())
+		t := time.NewTimer(retryDuration)
+		select {
+		case <-c.stopCh:
+			t.Stop()
+			return
+		case <-t.C:
 		}
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
-			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
-		time.Sleep(retryDuration)
 		c.retriesCount.Inc()
 		goto again
 	}
-	c.requestsOKCount.Inc()
-
-	// The block has been successfully sent to the remote storage.
-	fasthttp.ReleaseResponse(resp)
-	fasthttp.ReleaseRequest(req)
-}
-
-func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) error {
-	// There is no need in calling DoTimeout, since the timeout must be already set in hc.ReadTimeout.
-	err := hc.Do(req, resp)
-	if err == nil {
-		return nil
+	statusCode := resp.StatusCode
+	if statusCode/100 == 2 {
+		_ = resp.Body.Close()
+		c.requestsOKCount.Inc()
+		return
 	}
-	if err != fasthttp.ErrConnectionClosed {
-		return err
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
+	if statusCode == 409 {
+		// Just drop block on 409 status code like Prometheus does.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+		body, _ := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		logger.Errorf("unexpected status code received when sending a block with size %d bytes to %q: #%d; dropping the block like Prometheus does; "+
+			"response body=%q", len(block), c.sanitizedURL, statusCode, body)
+		c.packetsDropped.Inc()
+		return
 	}
-	// Retry request if the server closed the keep-alive connection during the first attempt.
-	return hc.Do(req, resp)
+
+	// Unexpected status code returned
+	retriesCount++
+	retryDuration *= 2
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
+	} else {
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
+			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
+	}
+	t := time.NewTimer(retryDuration)
+	select {
+	case <-c.stopCh:
+		t.Stop()
+		return
+	case <-t.C:
+	}
+	c.retriesCount.Inc()
+	goto again
 }

app/vmagent/remotewrite/pendingseries.go

@@ -3,20 +3,24 @@ package remotewrite
 import (
 	"flag"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
 
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
-		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage")
-	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage. "+
+		"Minimum supported interval is 1 second")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
 		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
 )
 
@@ -55,6 +59,10 @@ func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
 }
 
 func (ps *pendingSeries) periodicFlusher() {
+	flushSeconds := int64(flushInterval.Seconds())
+	if flushSeconds <= 0 {
+		flushSeconds = 1
+	}
 	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
 	mustStop := false
@@ -63,7 +71,7 @@ func (ps *pendingSeries) periodicFlusher() {
 		case <-ps.stopCh:
 			mustStop = true
 		case <-ticker.C:
-			if time.Since(ps.wr.lastFlushTime) < *flushInterval/2 {
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
@@ -74,9 +82,11 @@ func (ps *pendingSeries) periodicFlusher() {
 }
 
 type writeRequest struct {
-	wr            prompbmarshal.WriteRequest
-	pushBlock     func(block []byte)
-	lastFlushTime time.Time
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
+	lastFlushTime uint64
+
+	wr        prompbmarshal.WriteRequest
+	pushBlock func(block []byte)
 
 	tss []prompbmarshal.TimeSeries
 
@@ -95,11 +105,7 @@ func (wr *writeRequest) reset() {
 	}
 	wr.tss = wr.tss[:0]
 
-	for i := range wr.labels {
-		label := &wr.labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(wr.labels)
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
@@ -108,7 +114,7 @@ func (wr *writeRequest) reset() {
 
 func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
-	wr.lastFlushTime = time.Now()
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
 	pushWriteRequest(&wr.wr, wr.pushBlock)
 	wr.reset()
 }
@@ -117,9 +123,9 @@ func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		dst := &tssDst[len(tssDst)-1]
-		wr.copyTimeSeries(dst, &src[i])
-		if len(wr.tss) >= maxRowsPerBlock {
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
+		if len(wr.samples) >= maxRowsPerBlock {
+			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss
 		}
@@ -144,13 +150,8 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	}
 	dst.Labels = labelsDst[labelsLen:]
 
-	samplesDst = append(samplesDst, prompbmarshal.Sample{})
-	dstSample := &samplesDst[len(samplesDst)-1]
-	if len(src.Samples) != 1 {
-		logger.Panicf("BUG: unexpected number of samples in time series; got %d; want 1", len(src.Samples))
-	}
-	*dstSample = src.Samples[0]
-	dst.Samples = samplesDst[len(samplesDst)-1:]
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
 
 	wr.samples = samplesDst
 	wr.labels = labelsDst
@@ -164,7 +165,7 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 	}
 	bb := writeRequestBufPool.Get()
 	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
-	if len(bb.B) <= *maxUnpackedBlockSize {
+	if len(bb.B) <= maxUnpackedBlockSize.N {
 		zb := snappyBufPool.Get()
 		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		writeRequestBufPool.Put(bb)

app/vmagent/remotewrite/relabel.go

@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"flag"
+	"fmt"
 	"strings"
 	"sync"
 
@@ -15,36 +16,64 @@ var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
-		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+		"before sending them to -remoteWrite.url. See https://victoriametrics.github.io/vmagent.html#relabeling for details")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
 )
 
 var labelsGlobal []prompbmarshal.Label
-var prcsGlobal []promrelabel.ParsedRelabelConfig
 
-// initRelabelGlobal must be called after parsing command-line flags.
-func initRelabelGlobal() {
-	// Init labelsGlobal
+// CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
+func CheckRelabelConfigs() error {
+	_, err := loadRelabelConfigs()
+	return err
+}
+
+func loadRelabelConfigs() (*relabelConfigs, error) {
+	var rcs relabelConfigs
+	if *relabelConfigPathGlobal != "" {
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
+		}
+		rcs.global = global
+	}
+	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
+			len(*relabelConfigPaths), len(*remoteWriteURLs))
+	}
+	rcs.perURL = make([][]promrelabel.ParsedRelabelConfig, len(*remoteWriteURLs))
+	for i, path := range *relabelConfigPaths {
+		if len(path) == 0 {
+			// Skip empty relabel config.
+			continue
+		}
+		prc, err := promrelabel.LoadRelabelConfigs(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
+		}
+		rcs.perURL[i] = prc
+	}
+	return &rcs, nil
+}
+
+type relabelConfigs struct {
+	global []promrelabel.ParsedRelabelConfig
+	perURL [][]promrelabel.ParsedRelabelConfig
+}
+
+// initLabelsGlobal must be called after parsing command-line flags.
+func initLabelsGlobal() {
 	labelsGlobal = nil
 	for _, s := range *unparsedLabelsGlobal {
 		n := strings.IndexByte(s, '=')
 		if n < 0 {
-			logger.Panicf("FATAL: missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+			logger.Fatalf("missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
 		}
 		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
 			Name:  s[:n],
 			Value: s[n+1:],
 		})
 	}
-
-	// Init prcsGlobal
-	prcsGlobal = nil
-	if len(*relabelConfigPathGlobal) > 0 {
-		var err error
-		prcsGlobal, err = promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
-		if err != nil {
-			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.relabelConfig=%q: %s", *relabelConfigPathGlobal, err)
-		}
-	}
 }
 
 func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
@@ -88,12 +117,7 @@ type relabelCtx struct {
 }
 
 func (rctx *relabelCtx) reset() {
-	labels := rctx.labels
-	for i := range labels {
-		label := &labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(rctx.labels)
 	rctx.labels = rctx.labels[:0]
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -3,15 +3,17 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"runtime"
+	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/metrics"
 	xxhash "github.com/cespare/xxhash/v2"
 )
@@ -20,20 +22,37 @@ var (
 	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
-	tmpDataPath        = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
-	queues             = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
+	queues      = flag.Int("remoteWrite.queues", 4, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
-		"It is hidden by default, since it can contain sensistive auth info")
-	maxPendingBytesPerURL = flag.Int("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
 		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
 		"Disk usage is unlimited if the value is set to 0")
+	significantFigures = flag.Int("remoteWrite.significantFigures", 0, "The number of significant figures to leave in metric values before writing them to remote storage. "+
+		"See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
+		"This option may be used for increasing on-disk compression level for the stored metrics")
 )
 
 var rwctxs []*remoteWriteCtx
 
+// Contains the current relabelConfigs.
+var allRelabelConfigs atomic.Value
+
+// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
+// since it may lead to high memory usage due to big number of buffers.
+var maxQueues = runtime.GOMAXPROCS(-1) * 4
+
+// InitSecretFlags must be called after flag.Parse and before any logging.
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -41,14 +60,20 @@ var rwctxs []*remoteWriteCtx
 // Stop must be called for graceful shutdown.
 func Init() {
 	if len(*remoteWriteURLs) == 0 {
-		logger.Panicf("FATAL: at least one `-remoteWrite.url` must be set")
+		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
-
-	if !*showRemoteWriteURL {
-		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
-		httpserver.RegisterSecretFlag("remoteWrite.url")
+	if *queues > maxQueues {
+		*queues = maxQueues
 	}
-	initRelabelGlobal()
+	if *queues <= 0 {
+		*queues = 1
+	}
+	initLabelsGlobal()
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot load relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
 
 	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
 	if maxInmemoryBlocks > 200 {
@@ -61,23 +86,47 @@ func Init() {
 		maxInmemoryBlocks = 2
 	}
 	for i, remoteWriteURL := range *remoteWriteURLs {
-		relabelConfigPath := ""
-		if i < len(*relabelConfigPaths) {
-			relabelConfigPath = (*relabelConfigPaths)[i]
-		}
-		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if *showRemoteWriteURL {
-			urlLabelValue = remoteWriteURL
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, relabelConfigPath, maxInmemoryBlocks, urlLabelValue)
+		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 		rwctxs = append(rwctxs, rwctx)
 	}
+
+	// Start config reloader.
+	sighupCh := procutil.NewSighupChan()
+	configReloaderWG.Add(1)
+	go func() {
+		defer configReloaderWG.Done()
+		for {
+			select {
+			case <-sighupCh:
+			case <-stopCh:
+				return
+			}
+			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+			rcs, err := loadRelabelConfigs()
+			if err != nil {
+				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+				continue
+			}
+			allRelabelConfigs.Store(rcs)
+			logger.Infof("Successfully reloaded relabel configs")
+		}
+	}()
 }
 
+var stopCh = make(chan struct{})
+var configReloaderWG sync.WaitGroup
+
 // Stop stops remotewrite.
 //
 // It is expected that nobody calls Push during and after the call to this func.
 func Stop() {
+	close(stopCh)
+	configReloaderWG.Wait()
+
 	for _, rwctx := range rwctxs {
 		rwctx.MustStop()
 	}
@@ -86,21 +135,41 @@ func Stop() {
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// Each timeseries in wr.Timeseries must contain one sample.
-//
-// Note that wr may be modified by Push due to relabeling.
+// Note that wr may be modified by Push due to relabeling and rounding.
 func Push(wr *prompbmarshal.WriteRequest) {
+	if *significantFigures > 0 {
+		// Round values according to significantFigures
+		for i := range wr.Timeseries {
+			samples := wr.Timeseries[i].Samples
+			for j := range samples {
+				s := &samples[j]
+				s.Value = decimal.Round(s.Value, *significantFigures)
+			}
+		}
+	}
+
 	var rctx *relabelCtx
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	prcsGlobal := rcs.global
 	if len(prcsGlobal) > 0 || len(labelsGlobal) > 0 {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
 	for len(tss) > 0 {
-		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		// Process big tss in smaller blocks in order to reduce the maximum memory usage
+		samplesCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			i++
+			if samplesCount > maxRowsPerBlock {
+				break
+			}
+		}
 		tssBlock := tss
-		if len(tssBlock) > maxRowsPerBlock {
-			tssBlock = tss[:maxRowsPerBlock]
-			tss = tss[maxRowsPerBlock:]
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
 		} else {
 			tss = nil
 		}
@@ -124,47 +193,37 @@ func Push(wr *prompbmarshal.WriteRequest) {
 var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
 
 type remoteWriteCtx struct {
+	idx        int
 	fq         *persistentqueue.FastQueue
 	c          *client
-	prcs       []promrelabel.ParsedRelabelConfig
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
-	tss []prompbmarshal.TimeSeries
-
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL, relabelConfigPath string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
-	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, *maxPendingBytesPerURL)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, urlLabelValue, fq, *queues)
-	var prcs []promrelabel.ParsedRelabelConfig
-	if len(relabelConfigPath) > 0 {
-		var err error
-		prcs, err = promrelabel.LoadRelabelConfigs(relabelConfigPath)
-		if err != nil {
-			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %s", relabelConfigPath, err)
-		}
-	}
+	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
 	pss := make([]*pendingSeries, *queues)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq.MustWriteBlock)
 	}
 	return &remoteWriteCtx{
-		fq:   fq,
-		c:    c,
-		prcs: prcs,
-		pss:  pss,
+		idx: argIdx,
+		fq:  fq,
+		c:   c,
+		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
 	}
 }
 
@@ -172,10 +231,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
+	rwctx.idx = 0
 	rwctx.pss = nil
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
-	rwctx.prcs = nil
 	rwctx.c.MustStop()
 	rwctx.c = nil
 
@@ -184,23 +243,34 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	var rctx *relabelCtx
-	if len(rwctx.prcs) > 0 {
+	var v *[]prompbmarshal.TimeSeries
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	prcs := rcs.perURL[rwctx.idx]
+	if len(prcs) > 0 {
+		rctx = getRelabelCtx()
 		// Make a copy of tss before applying relabeling in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467 for details.
-		rwctx.tss = append(rwctx.tss[:0], tss...)
-		tss = rwctx.tss
-		rctx = getRelabelCtx()
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
+		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
+		tss = append(*v, tss...)
 		tssLen := len(tss)
-		tss = rctx.applyRelabeling(tss, nil, rwctx.prcs)
+		tss = rctx.applyRelabeling(tss, nil, prcs)
 		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
 	}
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	pss[idx].Push(tss)
 	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssRelabelPool.Put(v)
 		putRelabelCtx(rctx)
-		// Zero rwctx.tss in order to free up GC references.
-		rwctx.tss = prompbmarshal.ResetTimeSeries(rwctx.tss)
 	}
 }
+
+var tssRelabelPool = &sync.Pool{
+	New: func() interface{} {
+		a := []prompbmarshal.TimeSeries{}
+		return &a
+	},
+}

app/vmagent/remotewrite/statconn.go

@@ -1,7 +1,9 @@
 package remotewrite
 
 import (
+	"fmt"
 	"net"
+	"strings"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -9,7 +11,10 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func statDial(addr string) (conn net.Conn, err error) {
+func statDial(network, addr string) (conn net.Conn, err error) {
+	if !strings.HasPrefix(network, "tcp") {
+		return nil, fmt.Errorf("unexpected network passed to statDial: %q; it must start from `tcp`", network)
+	}
 	if netutil.TCP6Enabled() {
 		conn, err = fasthttp.DialDualStack(addr)
 	} else {

app/vmagent/vmimport/request_handler.go

@@ -6,7 +6,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,12 +23,18 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, insertRows)
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -36,6 +44,7 @@ func insertRows(rows []parser.Row) error {
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Values)
 		labelsLen := len(labels)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
@@ -44,9 +53,12 @@ func insertRows(rows []parser.Row) error {
 				Value: bytesutil.ToUnsafeString(tag.Value),
 			})
 		}
+		labels = append(labels, extraLabels...)
 		values := r.Values
 		timestamps := r.Timestamps
-		_ = timestamps[len(values)-1]
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
 		samplesLen := len(samples)
 		for j, value := range values {
 			samples = append(samples, prompbmarshal.Sample{
@@ -58,7 +70,6 @@ func insertRows(rows []parser.Row) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(values)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmalert/Makefile

@@ -52,30 +52,39 @@ publish-vmalert:
 	APP_NAME=vmalert $(MAKE) publish-via-docker
 
 test-vmalert:
-	go test -race -cover ./app/vmalert
+	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/datasource
+	go test -v -race -cover ./app/vmalert/notifier
+	go test -v -race -cover ./app/vmalert/config
 
 run-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/testdata/rules0-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-notifier.url=http://localhost:9093 \
-		-remotewrite.url=http://localhost:8428 \
-		-remoteread.url=http://localhost:8428 \
+		-notifier.url=http://127.0.0.1:9093 \
+		-remoteWrite.url=http://localhost:8428 \
+		-remoteRead.url=http://localhost:8428 \
+		-external.label=cluster=east-1 \
+		-external.label=replica=a \
 		-evaluationInterval=3s
 
 vmalert-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-amd64 ./app/vmalert
+	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmalert-local-with-goarch
 
 vmalert-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm ./app/vmalert
+	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmalert-local-with-goarch
 
 vmalert-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm64 ./app/vmalert
+	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmalert-local-with-goarch
 
 vmalert-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-ppc64le ./app/vmalert
+	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmalert-local-with-goarch
 
 vmalert-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-386 ./app/vmalert
+	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmalert-local-with-goarch
+
+vmalert-local-with-goarch:
+	APP_NAME=vmalert $(MAKE) app-local-with-goarch
 
 vmalert-pure:
 	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/README.md

@@ -1,19 +1,28 @@
-## VM Alert
+## vmalert
 
-`vmalert` executes a list of given MetricsQL expressions (rules) and
-sends alerts to [Alert Manager](https://github.com/prometheus/alertmanager).   
+`vmalert` executes a list of given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
+or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
+rules against configured address.
 
 ### Features:
 * Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
 * VictoriaMetrics [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
- expressions validation;
+ support and expressions validation;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Keeps the alerts [state on restarts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmalert#alerts-state-on-restarts);
 * Lightweight without extra dependencies.
 
-### TODO:
-* Configuration hot reload.
+### Limitations:
+* `vmalert` execute queries against remote datasource which has reliability risks because of network. 
+It is recommended to configure alerts thresholds and rules expressions with understanding that network request
+may fail;
+* by default, rules execution is sequential within one group, but persisting of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
+recording rule is reused in next one;
+* there is no `query` function support in templates yet;
+* `vmalert` has no UI, just an API for getting groups and rules statuses.
 
 ### QuickStart
 
@@ -26,85 +35,299 @@ make vmalert
 The build binary will be placed to `VictoriaMetrics/bin` folder.
 
 To start using `vmalert` you will need the following things:
-* list of alert rules - PromQL/MetricsQL expressions to execute;
+* list of rules - PromQL/MetricsQL expressions to execute;
 * datasource address - reachable VictoriaMetrics instance for rules execution;
-* notifier address - reachable Alertmanager instance for processing, 
+* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing, 
 aggregating alerts and sending notifications.
+* remote write address - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+compatible storage address for storing recording rules results and alerts state in for of timeseries. This is optional.
 
 Then configure `vmalert` accordingly:
 ```
 ./bin/vmalert -rule=alert.rules \
-		-datasource.url=http://localhost:8428 \
-        -notifier.url=http://localhost:9093
+    -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
+    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
+    -remoteWrite.url=http://localhost:8428 \ # remote write compatible storage to persist rules
+    -remoteRead.url=http://localhost:8428 \  # PromQL compatible datasource to restore alerts state from
+    -external.label=cluster=east-1 \         # External label to be applied for each rule
+    -external.label=replica=a \              # Multiple external labels may be set
+    -evaluationInterval=3s                   # Default evaluation interval if not specified in rules group
 ```
 
-Example for `.rules` file may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/testdata/rules0-good.rules)
+If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
+to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts. 
 
-`vmalert` runs evaluation for every group in a separate goroutine.
-Rules in group evaluated one-by-one sequentially. 
+Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) 
+and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very 
+similar to Prometheus rules and configured using YAML. Configuration examples may be found 
+in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
+Every `rule` belongs to `group` and every configuration file may contain arbitrary number of groups:
+```yaml
+groups:
+  [ - <rule_group> ]
+```
 
-`vmalert` also runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+#### Groups
+
+Each group has following attributes:
+```yaml
+# The name of the group. Must be unique within a file.
+name: <string>
+
+# How often rules in the group are evaluated.
+[ interval: <duration> | default = global.evaluation_interval ]
+
+# How many rules execute at once. Increasing concurrency may speed
+# up round execution speed. 
+[ concurrency: <integer> | default = 1 ]
+
+rules:
+  [ - <rule> ... ]
+```
+
+#### Rules
+
+There are two types of Rules:
+* [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - 
+Alerting rules allows to define alert conditions via [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+and to send notifications about firing alerts to [Alertmanager](https://github.com/prometheus/alertmanager).
+* [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) - 
+Recording rules allow you to precompute frequently needed or computationally expensive expressions 
+and save their result as a new set of time series.
+
+`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+within one group. 
+
+##### Alerting rules
+
+The syntax for alerting rule is following:
+```yaml
+# The name of the alert. Must be a valid metric name.
+alert: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Alerts are considered firing once they have been returned for this long.
+# Alerts which have not yet fired for long enough are considered pending.
+[ for: <duration> | default = 0s ]
+
+# Labels to add or overwrite for each alert.
+labels:
+  [ <labelname>: <tmpl_string> ]
+
+# Annotations to add to each alert.
+annotations:
+  [ <labelname>: <tmpl_string> ]
+``` 
+
+##### Recording rules
+
+The syntax for recording rules is following:
+```yaml
+# The name of the time series to output to. Must be a valid metric name.
+record: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Labels to add or overwrite before storing the result.
+labels:
+  [ <labelname>: <labelvalue> ]
+```
+
+For recording rules to work `-remoteWrite.url` must specified.
+
+
+#### Alerts state on restarts
+
+`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after reloading of `vmalert` 
+the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
+* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or VMInsert (Cluster). `vmalert` will persist alerts state 
+into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol. 
+These are regular time series and may be queried from VM just as any other time series. 
+The state stored to the configured address on every rule evaluation.
+* `-remoteRead.url` - URL to VictoriaMetrics (Single) or VMSelect (Cluster). `vmalert` will try to restore alerts state 
+from configured address by querying time series with name `ALERTS_FOR_STATE`.
+
+Both flags are required for the proper state restoring. Restore process may fail if time series are missing
+in configured `-remoteRead.url`, weren't updated in the last `1h` or received state doesn't match current `vmalert` 
+rules configuration.
+
+
+#### WEB
+
+`vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
 * `http://<vmalert-addr>/api/v1/<groupName>/<alertID>/status" ` - get alert status by ID.
 Used as alert source in AlertManager.
 * `http://<vmalert-addr>/metrics` - application metrics.
 * `http://<vmalert-addr>/-/reload` - hot configuration reload.
 
-`vmalert` may be configured with `-remotewrite` flag to write alerts state in form of timeseries
-via remote write protocol. Alerts state will be written as `ALERTS` timeseries. These timeseries
-may be used to recover alerts state on `vmalert` restarts if `-remoteread` is configured.
-
 
 ### Configuration
 
 The shortlist of configuration flags is the following:
 ```
-Usage of vmalert:
   -datasource.basicAuth.password string
-        Optional basic auth password for -datasource.url
+    	Optional basic auth password for -datasource.url
   -datasource.basicAuth.username string
-        Optional basic auth username for -datasource.url
+    	Optional basic auth username for -datasource.url
+  -datasource.lookback duration
+        Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+  -datasource.maxIdleConnections int
+        Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
+  -datasource.tlsCAFile string
+    	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
+  -datasource.tlsCertFile string
+    	Optional path to client-side TLS certificate file to use when connecting to -datasource.url
+  -datasource.tlsInsecureSkipVerify
+    	Whether to skip tls verification when connecting to -datasource.url
+  -datasource.tlsKeyFile string
+    	Optional path to client-side TLS certificate key to use when connecting to -datasource.url
+  -datasource.tlsServerName string
+    	Optional TLS server name to use for connections to -datasource.url. By default the server name from -datasource.url is used
   -datasource.url string
-        Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
+    	Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
   -enableTCP6
-        Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
   -evaluationInterval duration
-        How often to evaluate the rules. Default 1m (default 1m0s)
+    	How often to evaluate the rules (default 1m0s)
+  -external.alert.source string
+    	External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
+    	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
+  -external.label array
+    	Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
+    	Supports array of values separated by comma or specified via multiple flags.
   -external.url string
-        External URL is used as alert's source for sent alerts to the notifier
+    	External URL is used as alert's source for sent alerts to the notifier
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-        The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-        Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-        Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-        Address to listen for http connections (default ":8880")
-  -notifier.url string
-        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
-  -remoteread.basicAuth.password string
-        Optional basic auth password for -remoteread.url
-  -remoteread.basicAuth.username string
-        Optional basic auth username for -remoteread.url
-  -remoteread.lookback duration
-        Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
-  -remoteread.url vmalert
-        Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remotewrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
-  -remotewrite.basicAuth.password string
-        Optional basic auth password for -remotewrite.url
-  -remotewrite.basicAuth.username string
-        Optional basic auth username for -remotewrite.url
-  -remotewrite.url string
-        Optional URL to Victoria Metrics or VMInsert where to persist alerts state in form of timeseries. E.g. http://127.0.0.1:8428
-  -rule value
-        Path to the file with alert rules. 
-        Supports patterns. Flag can be specified multiple times. 
-        Examples:
-         -rule /path/to/file. Path to a single file with alerting rules
-         -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
-        absolute path to all .yaml files in root.
+    	Address to listen for http connections (default ":8880")
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit (default 10)
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -memory.allowedBytes value
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It overrides httpAuth settings
+  -notifier.basicAuth.password array
+    	Optional basic auth password for -datasource.url
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.basicAuth.username array
+    	Optional basic auth username for -datasource.url
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.tlsCAFile array
+    	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.tlsCertFile array
+    	Optional path to client-side TLS certificate file to use when connecting to -notifier.url
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.tlsInsecureSkipVerify
+    	Whether to skip tls verification when connecting to -notifier.url
+  -notifier.tlsKeyFile array
+    	Optional path to client-side TLS certificate key to use when connecting to -notifier.url
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.tlsServerName array
+    	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
+    	Supports array of values separated by comma or specified via multiple flags.
+  -notifier.url array
+    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+    	Supports array of values separated by comma or specified via multiple flags.
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It overrides httpAuth settings
+  -remoteRead.basicAuth.password string
+    	Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.username string
+    	Optional basic auth username for -remoteRead.url
+  -remoteRead.lookback duration
+    	Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteRead.tlsCAFile string
+    	Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
+  -remoteRead.tlsCertFile string
+    	Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url
+  -remoteRead.tlsInsecureSkipVerify
+    	Whether to skip tls verification when connecting to -remoteRead.url
+  -remoteRead.tlsKeyFile string
+    	Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url
+  -remoteRead.tlsServerName string
+    	Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
+  -remoteRead.url vmalert
+    	Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+  -remoteWrite.basicAuth.password string
+    	Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.username string
+    	Optional basic auth username for -remoteWrite.url
+  -remoteWrite.concurrency int
+    	Defines number of writers for concurrent writing into remote querier (default 1)
+  -remoteWrite.flushInterval duration
+    	Defines interval of flushes to remote write endpoint (default 5s)
+  -remoteWrite.maxBatchSize int
+    	Defines defines max number of timeseries to be flushed at once (default 1000)
+  -remoteWrite.maxQueueSize int
+    	Defines the max number of pending datapoints to remote write endpoint (default 100000)
+  -remoteWrite.tlsCAFile string
+    	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
+  -remoteWrite.tlsCertFile string
+    	Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url
+  -remoteWrite.tlsInsecureSkipVerify
+    	Whether to skip tls verification when connecting to -remoteWrite.url
+  -remoteWrite.tlsKeyFile string
+    	Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url
+  -remoteWrite.tlsServerName string
+    	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
+  -remoteWrite.url string
+    	Optional URL to Victoria Metrics or VMInsert where to persist alerts state and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428
+  -rule array
+    	Path to the file with alert rules. 
+    	Supports patterns. Flag can be specified multiple times. 
+    	Examples:
+    	 -rule="/path/to/file". Path to a single file with alerting rules
+    	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, 
+    	absolute path to all .yaml files in root.
+    	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+    	Supports array of values separated by comma or specified via multiple flags.
+  -rule.validateExpressions
+    	Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
-        Indicates to validate annotation and label templates (default true)
+    	Whether to validate annotation and label templates (default true)
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+  -version
+    	Show VictoriaMetrics version
 ```
 
 Pass `-help` to `vmalert` in order to see the full list of supported 
@@ -118,3 +341,39 @@ or send GET request to `/-/reload` endpoint.
 `vmalert` is mostly designed and built by VictoriaMetrics community.
 Feel free to share your experience and ideas for improving this 
 software. Please keep simplicity as the main priority.
+
+### How to build from sources
+
+It is recommended using 
+[binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) 
+- `vmalert` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmalert` from the root folder of the repository.
+   It builds `vmalert` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmalert-prod` from the root folder of the repository.
+   It builds `vmalert-prod` binary and puts it into the `bin` folder.
+
+
+#### ARM build
+
+ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
+
+#### Development ARM build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of the repository.
+   It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
+
+#### Production ARM build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmalert-arm-prod` or `make vmalert-arm64-prod` from the root folder of the repository.
+   It builds `vmalert-arm-prod` or `vmalert-arm64-prod` binary respectively and puts it into the `bin` folder.

app/vmalert/alerting.go

@@ -0,0 +1,434 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// AlertingRule is basic alert entity
+type AlertingRule struct {
+	RuleID      uint64
+	Name        string
+	Expr        string
+	For         time.Duration
+	Labels      map[string]string
+	Annotations map[string]string
+	GroupID     uint64
+	GroupName   string
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+
+	metrics *alertingRuleMetrics
+}
+
+type alertingRuleMetrics struct {
+	errors  *gauge
+	pending *gauge
+	active  *gauge
+}
+
+func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
+	ar := &AlertingRule{
+		RuleID:      cfg.ID,
+		Name:        cfg.Alert,
+		Expr:        cfg.Expr,
+		For:         cfg.For.Duration(),
+		Labels:      cfg.Labels,
+		Annotations: cfg.Annotations,
+		GroupID:     group.ID(),
+		GroupName:   group.Name,
+		alerts:      make(map[uint64]*notifier.Alert),
+		metrics:     &alertingRuleMetrics{},
+	}
+
+	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
+	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+		func() float64 {
+			ar.mu.Lock()
+			defer ar.mu.Unlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StatePending {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+		func() float64 {
+			ar.mu.Lock()
+			defer ar.mu.Unlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StateFiring {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_error{%s}`, labels),
+		func() float64 {
+			ar.mu.Lock()
+			defer ar.mu.Unlock()
+			if ar.lastExecError == nil {
+				return 0
+			}
+			return 1
+		})
+	return ar
+}
+
+// Close unregisters rule metrics
+func (ar *AlertingRule) Close() {
+	metrics.UnregisterMetric(ar.metrics.active.name)
+	metrics.UnregisterMetric(ar.metrics.pending.name)
+	metrics.UnregisterMetric(ar.metrics.errors.name)
+}
+
+// String implements Stringer interface
+func (ar *AlertingRule) String() string {
+	return ar.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (ar *AlertingRule) ID() uint64 {
+	return ar.RuleID
+}
+
+// Exec executes AlertingRule expression via the given Querier.
+// Based on the Querier results AlertingRule maintains notifier.Alerts
+func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := q.Query(ctx, ar.Expr)
+	ar.mu.Lock()
+	defer ar.mu.Unlock()
+
+	ar.lastExecError = err
+	ar.lastExecTime = time.Now()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
+	}
+
+	for h, a := range ar.alerts {
+		// cleanup inactive alerts from previous Exec
+		if a.State == notifier.StateInactive {
+			delete(ar.alerts, h)
+		}
+	}
+
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		for k, v := range ar.Labels {
+			// apply extra labels
+			m.SetLabel(k, v)
+		}
+
+		h := hash(m)
+		if _, ok := updated[h]; ok {
+			// duplicate may be caused by extra labels
+			// conflicting with the metric labels
+			return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
+		}
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
+			if a.Value != m.Value {
+				// update Value field with latest value
+				a.Value = m.Value
+				// and re-exec template since Value can be used
+				// in templates
+				err = ar.template(a)
+				if err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+		a, err := ar.newAlert(m, ar.lastExecTime)
+		if err != nil {
+			ar.lastExecError = err
+			return nil, fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		ar.alerts[h] = a
+	}
+
+	for h, a := range ar.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(ar.alerts, h)
+				continue
+			}
+			a.State = notifier.StateInactive
+			continue
+		}
+		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+			a.State = notifier.StateFiring
+			alertsFired.Inc()
+		}
+	}
+	if series {
+		return ar.toTimeSeries(ar.lastExecTime), nil
+	}
+	return nil, nil
+}
+
+func (ar *AlertingRule) toTimeSeries(timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		ts := ar.alertToTimeSeries(a, timestamp)
+		tss = append(tss, ts...)
+	}
+	return tss
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(m datasource.Metric) uint64 {
+	hash := fnv.New64a()
+	labels := m.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifier.Alert, error) {
+	a := &notifier.Alert{
+		GroupID: ar.GroupID,
+		Name:    ar.Name,
+		Labels:  map[string]string{},
+		Value:   m.Value,
+		Start:   start,
+		Expr:    ar.Expr,
+	}
+	// label defined here to make override possible by
+	// time series labels.
+	a.Labels[alertGroupNameLabel] = ar.GroupName
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		a.Labels[l.Name] = l.Value
+	}
+	return a, ar.template(a)
+}
+
+func (ar *AlertingRule) template(a *notifier.Alert) error {
+	var err error
+	a.Labels, err = a.ExecTemplate(a.Labels)
+	if err != nil {
+		return err
+	}
+	a.Annotations, err = a.ExecTemplate(ar.Annotations)
+	return err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.mu.RLock()
+	defer ar.mu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// RuleAPI returns Rule representation in form
+// of APIAlertingRule
+func (ar *AlertingRule) RuleAPI() APIAlertingRule {
+	var lastErr string
+	if ar.lastExecError != nil {
+		lastErr = ar.lastExecError.Error()
+	}
+	return APIAlertingRule{
+		// encode as strings to avoid rounding
+		ID:          fmt.Sprintf("%d", ar.ID()),
+		GroupID:     fmt.Sprintf("%d", ar.GroupID),
+		Name:        ar.Name,
+		Expression:  ar.Expr,
+		For:         ar.For.String(),
+		LastError:   lastErr,
+		LastExec:    ar.lastExecTime,
+		Labels:      ar.Labels,
+		Annotations: ar.Annotations,
+	}
+}
+
+// AlertsAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.mu.RLock()
+	for _, a := range ar.alerts {
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.mu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	return &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.Start,
+		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+	}
+}
+
+const (
+	// alertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// alertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// alertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// alertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+
+	// alertGroupNameLabel defines the label name attached for generated time series.
+	alertGroupNameLabel = "alertgroup"
+)
+
+// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores only Start field. Field State will be always Pending and supposed
+// to be updated on next Exec, as well as Value field.
+// Only rules with For > 0 will be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
+	if q == nil {
+		return fmt.Errorf("querier is nil")
+	}
+
+	// account for external labels in filter
+	var labelsFilter string
+	for k, v := range labels {
+		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
+	}
+
+	// Get the last data point in range via MetricsQL `last_over_time`.
+	// We don't use plain PromQL since Prometheus doesn't support
+	// remote write protocol which is used for state persistence in vmalert.
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
+		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
+	qMetrics, err := q.Query(ctx, expr)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range qMetrics {
+		labels := m.Labels
+		m.Labels = make([]datasource.Label, 0)
+		// drop all extra labels, so hash key will
+		// be identical to time series received in Exec
+		for _, l := range labels {
+			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
+				continue
+			}
+			m.Labels = append(m.Labels, l)
+		}
+
+		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0))
+		if err != nil {
+			return fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = hash(m)
+		a.State = notifier.StatePending
+		ar.alerts[a.ID] = a
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
+	}
+	return nil
+}

app/vmalert/alerting_test.go

@@ -2,7 +2,8 @@ package main
 
 import (
 	"context"
-	"sync"
+	"errors"
+	"strings"
 	"testing"
 	"time"
 
@@ -11,30 +12,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestRule_Validate(t *testing.T) {
-	if err := (&Rule{}).Validate(); err == nil {
-		t.Errorf("exptected empty name error")
-	}
-	if err := (&Rule{Name: "alert"}).Validate(); err == nil {
-		t.Errorf("exptected empty expr error")
-	}
-	if err := (&Rule{Name: "alert", Expr: "test{"}).Validate(); err == nil {
-		t.Errorf("exptected invalid expr error")
-	}
-	if err := (&Rule{Name: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Errorf("exptected valid rule got %s", err)
-	}
-}
-
-func TestRule_AlertToTimeSeries(t *testing.T) {
+func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
-		rule  *Rule
+		rule  *AlertingRule
 		alert *notifier.Alert
 		expTS []prompbmarshal.TimeSeries
 	}{
 		{
-			newTestRule("instant", 0),
+			newTestAlertingRule("instant", 0),
 			&notifier.Alert{State: notifier.StateFiring},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries(1, map[string]string{
@@ -45,7 +31,7 @@ func TestRule_AlertToTimeSeries(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("instant extra labels", 0),
+			newTestAlertingRule("instant extra labels", 0),
 			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
 				"job":      "foo",
 				"instance": "bar",
@@ -61,7 +47,7 @@ func TestRule_AlertToTimeSeries(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("instant labels override", 0),
+			newTestAlertingRule("instant labels override", 0),
 			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
 				alertStateLabel: "foo",
 				"__name__":      "bar",
@@ -75,7 +61,7 @@ func TestRule_AlertToTimeSeries(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for", time.Second),
+			newTestAlertingRule("for", time.Second),
 			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries(1, map[string]string{
@@ -90,7 +76,7 @@ func TestRule_AlertToTimeSeries(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for pending", 10*time.Second),
+			newTestAlertingRule("for pending", 10*time.Second),
 			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries(1, map[string]string{
@@ -107,58 +93,29 @@ func TestRule_AlertToTimeSeries(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
-			tss := tc.rule.AlertToTimeSeries(tc.alert, timestamp)
-			if len(tc.expTS) != len(tss) {
-				t.Fatalf("expected number of timeseries %d; got %d", len(tc.expTS), len(tss))
-			}
-			for i := range tc.expTS {
-				expTS, gotTS := tc.expTS[i], tss[i]
-				if len(expTS.Samples) != len(gotTS.Samples) {
-					t.Fatalf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
-				}
-				for i, exp := range expTS.Samples {
-					got := gotTS.Samples[i]
-					if got.Value != exp.Value {
-						t.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
-					}
-					if got.Timestamp != exp.Timestamp {
-						t.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
-					}
-				}
-				if len(expTS.Labels) != len(gotTS.Labels) {
-					t.Fatalf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
-				}
-				for i, exp := range expTS.Labels {
-					got := gotTS.Labels[i]
-					if got.Name != exp.Name {
-						t.Errorf("expected label name %q; got %q", exp.Name, got.Name)
-					}
-					if got.Value != exp.Value {
-						t.Errorf("expected label value %q; got %q", exp.Value, got.Value)
-					}
-				}
+			tc.rule.alerts[tc.alert.ID] = tc.alert
+			tss := tc.rule.toTimeSeries(timestamp)
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
 			}
 		})
 	}
 }
 
-func newTestRule(name string, waitFor time.Duration) *Rule {
-	return &Rule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
-}
-
-func TestRule_Exec(t *testing.T) {
+func TestAlertingRule_Exec(t *testing.T) {
+	const defaultStep = 5 * time.Millisecond
 	testCases := []struct {
-		rule      *Rule
+		rule      *AlertingRule
 		steps     [][]datasource.Metric
 		expAlerts map[uint64]*notifier.Alert
 	}{
 		{
-			newTestRule("empty", 0),
+			newTestAlertingRule("empty", 0),
 			[][]datasource.Metric{},
 			map[uint64]*notifier.Alert{},
 		},
 		{
-			newTestRule("empty labels", 0),
+			newTestAlertingRule("empty labels", 0),
 			[][]datasource.Metric{
 				{datasource.Metric{}},
 			},
@@ -167,7 +124,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("single-firing", 0),
+			newTestAlertingRule("single-firing", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
@@ -176,7 +133,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("single-firing=>inactive", 0),
+			newTestAlertingRule("single-firing=>inactive", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -186,7 +143,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("single-firing=>inactive=>firing", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -197,7 +154,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("single-firing=>inactive=>firing=>inactive", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -209,7 +166,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -220,7 +177,7 @@ func TestRule_Exec(t *testing.T) {
 			map[uint64]*notifier.Alert{},
 		},
 		{
-			newTestRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -234,7 +191,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("multiple-firing", 0),
+			newTestAlertingRule("multiple-firing", 0),
 			[][]datasource.Metric{
 				{
 					metricWithLabels(t, "name", "foo"),
@@ -249,7 +206,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("multiple-steps-firing", 0),
+			newTestAlertingRule("multiple-steps-firing", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo1")},
@@ -264,20 +221,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("duplicate", 0),
-			[][]datasource.Metric{
-				{
-					// metrics with the same labelset should result in one alert
-					metricWithLabels(t, "name", "foo", "type", "bar"),
-					metricWithLabels(t, "type", "bar", "name", "foo"),
-				},
-			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
-			},
-		},
-		{
-			newTestRule("for-pending", time.Minute),
+			newTestAlertingRule("for-pending", time.Minute),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
@@ -286,7 +230,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for-fired", time.Millisecond),
+			newTestAlertingRule("for-fired", defaultStep),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
@@ -296,7 +240,17 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for-pending=>inactive", time.Millisecond),
+			newTestAlertingRule("for-pending=>empty", time.Second),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset and delete pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
@@ -308,19 +262,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for-pending=>firing=>inactive", time.Millisecond),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset pending alerts
-				{},
-			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
-			},
-		},
-		{
-			newTestRule("for-pending=>firing=>inactive=>pending", time.Millisecond),
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
@@ -333,7 +275,7 @@ func TestRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			newTestRule("for-pending=>firing=>inactive=>pending=>firing", time.Millisecond),
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
@@ -351,15 +293,15 @@ func TestRule_Exec(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
-			tc.rule.group = fakeGroup
+			tc.rule.GroupID = fakeGroup.ID()
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if err := tc.rule.Exec(context.TODO(), fq); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), fq, false); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
-				time.Sleep(time.Millisecond)
+				time.Sleep(defaultStep)
 			}
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
 				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
@@ -377,49 +319,9 @@ func TestRule_Exec(t *testing.T) {
 	}
 }
 
-func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
-	t.Helper()
-	if len(labels) == 0 || len(labels)%2 != 0 {
-		t.Fatalf("expected to get even number of labels")
-	}
-	m := datasource.Metric{}
-	for i := 0; i < len(labels); i += 2 {
-		m.Labels = append(m.Labels, datasource.Label{
-			Name:  labels[i],
-			Value: labels[i+1],
-		})
-	}
-	return m
-}
-
-type fakeQuerier struct {
-	sync.Mutex
-	metrics []datasource.Metric
-}
-
-func (fq *fakeQuerier) reset() {
-	fq.Lock()
-	fq.metrics = fq.metrics[:0]
-	fq.Unlock()
-}
-
-func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
-	fq.Lock()
-	fq.metrics = append(fq.metrics, metrics...)
-	fq.Unlock()
-}
-
-func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
-	fq.Lock()
-	cpy := make([]datasource.Metric, len(fq.metrics))
-	copy(cpy, fq.metrics)
-	fq.Unlock()
-	return cpy, nil
-}
-
-func TestRule_Restore(t *testing.T) {
+func TestAlertingRule_Restore(t *testing.T) {
 	testCases := []struct {
-		rule      *Rule
+		rule      *AlertingRule
 		metrics   []datasource.Metric
 		expAlerts map[uint64]*notifier.Alert
 	}{
@@ -442,6 +344,7 @@ func TestRule_Restore(t *testing.T) {
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
 					alertNameLabel, "",
+					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				),
@@ -462,7 +365,7 @@ func TestRule_Restore(t *testing.T) {
 					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
-					// following pair supposed to be dropped
+					// extra labels set by rule
 					"source", "vm",
 				),
 			},
@@ -470,6 +373,7 @@ func TestRule_Restore(t *testing.T) {
 				hash(metricWithLabels(t,
 					"foo", "bar",
 					"namespace", "baz",
+					"source", "vm",
 				)): {State: notifier.StatePending,
 					Start: time.Now().Truncate(time.Hour)},
 			},
@@ -504,9 +408,9 @@ func TestRule_Restore(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
 			fq := &fakeQuerier{}
-			tc.rule.group = fakeGroup
+			tc.rule.GroupID = fakeGroup.ID()
 			fq.add(tc.metrics...)
-			if err := tc.rule.Restore(context.TODO(), fq, time.Hour); err != nil {
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
@@ -528,8 +432,40 @@ func TestRule_Restore(t *testing.T) {
 	}
 }
 
-func newTestRuleWithLabels(name string, labels ...string) *Rule {
-	r := newTestRule(name, 0)
+func TestAlertingRule_Exec_Negative(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+
+	// successful attempt
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	_, err := ar.Exec(context.TODO(), fq, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// label `job` will collide with rule extra label and will make both time series equal
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	_, err = ar.Exec(context.TODO(), fq, false)
+	if !errors.Is(err, errDuplicate) {
+		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
+	}
+
+	fq.reset()
+
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+	_, err = ar.Exec(context.TODO(), fq, false)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
+	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)
 	for i := 0; i < len(labels); i += 2 {
 		r.Labels[labels[i]] = labels[i+1]
@@ -537,9 +473,6 @@ func newTestRuleWithLabels(name string, labels ...string) *Rule {
 	return r
 }
 
-func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
-	t.Helper()
-	m := metricWithLabels(t, labels...)
-	m.Value = value
-	return m
+func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
+	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
 }

app/vmalert/config.go

@@ -1,73 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"gopkg.in/yaml.v2"
-	"io/ioutil"
-	"path/filepath"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-)
-
-// Parse parses rule configs from given file patterns
-func Parse(pathPatterns []string, validateAnnotations bool) ([]Group, error) {
-	var fp []string
-	for _, pattern := range pathPatterns {
-		matches, err := filepath.Glob(pattern)
-		if err != nil {
-			return nil, fmt.Errorf("error reading file patther %s:%v", pattern, err)
-		}
-		fp = append(fp, matches...)
-	}
-	var groups []Group
-	for _, file := range fp {
-		groupsNames := map[string]struct{}{}
-		gr, err := parseFile(file)
-		if err != nil {
-			return nil, fmt.Errorf("file %s: %w", file, err)
-		}
-		for _, g := range gr {
-			if _, ok := groupsNames[g.Name]; ok {
-				return nil, fmt.Errorf("one file can not contain groups with the same name %s, filepath:%s", g.Name, file)
-			}
-			g.File = file
-			g.done = make(chan struct{})
-			g.finished = make(chan struct{})
-
-			groupsNames[g.Name] = struct{}{}
-			for _, rule := range g.Rules {
-				if err = rule.Validate(); err != nil {
-					return nil, fmt.Errorf("invalid rule filepath: %s, group %s: %w", file, g.Name, err)
-				}
-				if validateAnnotations {
-					if err = notifier.ValidateTemplates(rule.Annotations); err != nil {
-						return nil, fmt.Errorf("invalid annotations filepath: %s, group %s: %w", file, g.Name, err)
-					}
-					if err = notifier.ValidateTemplates(rule.Labels); err != nil {
-						return nil, fmt.Errorf("invalid labels filepath: %s, group %s: %w", file, g.Name, err)
-					}
-				}
-				rule.group = g
-				rule.alerts = make(map[uint64]*notifier.Alert)
-			}
-			groups = append(groups, g)
-		}
-	}
-	if len(groups) < 1 {
-		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
-	}
-	return groups, nil
-}
-
-func parseFile(path string) ([]Group, error) {
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("error reading alert rule file: %w", err)
-	}
-	g := struct {
-		Groups []Group `yaml:"groups"`
-	}{}
-	err = yaml.Unmarshal(data, &g)
-	return g.Groups, err
-}

app/vmalert/config/config.go

@@ -0,0 +1,276 @@
+package config
+
+import (
+	"crypto/md5"
+	"fmt"
+	"hash/fnv"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metricsql"
+	"gopkg.in/yaml.v2"
+)
+
+// Group contains list of Rules grouped into
+// entity with one name and evaluation interval
+type Group struct {
+	File        string
+	Name        string        `yaml:"name"`
+	Interval    time.Duration `yaml:"interval,omitempty"`
+	Rules       []Rule        `yaml:"rules"`
+	Concurrency int           `yaml:"concurrency"`
+	// Checksum stores the hash of yaml definition for this group.
+	// May be used to detect any changes like rules re-ordering etc.
+	Checksum string
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type group Group
+	if err := unmarshal((*group)(g)); err != nil {
+		return err
+	}
+	b, err := yaml.Marshal(g)
+	if err != nil {
+		return fmt.Errorf("failed to marshal group configuration for checksum: %w", err)
+	}
+	h := md5.New()
+	h.Write(b)
+	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
+// Validate check for internal Group or Rule configuration errors
+func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
+	if g.Name == "" {
+		return fmt.Errorf("group name must be set")
+	}
+	if len(g.Rules) == 0 {
+		return fmt.Errorf("group %q can't contain no rules", g.Name)
+	}
+	uniqueRules := map[uint64]struct{}{}
+	for _, r := range g.Rules {
+		ruleName := r.Record
+		if r.Alert != "" {
+			ruleName = r.Alert
+		}
+		if _, ok := uniqueRules[r.ID]; ok {
+			return fmt.Errorf("rule %q duplicate", ruleName)
+		}
+		uniqueRules[r.ID] = struct{}{}
+		if err := r.Validate(); err != nil {
+			return fmt.Errorf("invalid rule %q.%q: %w", g.Name, ruleName, err)
+		}
+		if validateExpressions {
+			if _, err := metricsql.Parse(r.Expr); err != nil {
+				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+		}
+		if validateAnnotations {
+			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+		}
+	}
+	return checkOverflow(g.XXX, fmt.Sprintf("group %q", g.Name))
+}
+
+// Rule describes entity that represent either
+// recording rule or alerting rule.
+type Rule struct {
+	ID          uint64
+	Record      string            `yaml:"record,omitempty"`
+	Alert       string            `yaml:"alert,omitempty"`
+	Expr        string            `yaml:"expr"`
+	For         PromDuration      `yaml:"for"`
+	Labels      map[string]string `yaml:"labels,omitempty"`
+	Annotations map[string]string `yaml:"annotations,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// PromDuration is Prometheus duration.
+type PromDuration struct {
+	milliseconds int64
+}
+
+// NewPromDuration returns PromDuration for given d.
+func NewPromDuration(d time.Duration) PromDuration {
+	return PromDuration{
+		milliseconds: d.Milliseconds(),
+	}
+}
+
+// MarshalYAML implements yaml.Marshaler interface.
+func (pd PromDuration) MarshalYAML() (interface{}, error) {
+	return pd.Duration().String(), nil
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler interface.
+func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	ms, err := metricsql.DurationValue(s, 0)
+	if err != nil {
+		return err
+	}
+	pd.milliseconds = ms
+	return nil
+}
+
+// Duration returns duration for pd.
+func (pd *PromDuration) Duration() time.Duration {
+	return time.Duration(pd.milliseconds) * time.Millisecond
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rule Rule
+	if err := unmarshal((*rule)(r)); err != nil {
+		return err
+	}
+	r.ID = HashRule(*r)
+	return nil
+}
+
+// Name returns Rule name according to its type
+func (r *Rule) Name() string {
+	if r.Record != "" {
+		return r.Record
+	}
+	return r.Alert
+}
+
+// HashRule hashes significant Rule fields into
+// unique hash that supposed to define Rule uniqueness
+func HashRule(r Rule) uint64 {
+	h := fnv.New64a()
+	h.Write([]byte(r.Expr))
+	if r.Record != "" {
+		h.Write([]byte("recording"))
+		h.Write([]byte(r.Record))
+	} else {
+		h.Write([]byte("alerting"))
+		h.Write([]byte(r.Alert))
+	}
+	kv := sortMap(r.Labels)
+	for _, i := range kv {
+		h.Write([]byte(i.key))
+		h.Write([]byte(i.value))
+		h.Write([]byte("\xff"))
+	}
+	return h.Sum64()
+}
+
+// Validate check for Rule configuration errors
+func (r *Rule) Validate() error {
+	if (r.Record == "" && r.Alert == "") || (r.Record != "" && r.Alert != "") {
+		return fmt.Errorf("either `record` or `alert` must be set")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("expression can't be empty")
+	}
+	return checkOverflow(r.XXX, "rule")
+}
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file pattern %s: %w", pattern, err)
+		}
+		fp = append(fp, matches...)
+	}
+	errGroup := new(utils.ErrGroup)
+	var groups []Group
+	for _, file := range fp {
+		uniqueGroups := map[string]struct{}{}
+		gr, err := parseFile(file)
+		if err != nil {
+			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
+			continue
+		}
+		for _, g := range gr {
+			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
+				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
+				continue
+			}
+			if _, ok := uniqueGroups[g.Name]; ok {
+				errGroup.Add(fmt.Errorf("group name %q duplicate in file %q", g.Name, file))
+				continue
+			}
+			uniqueGroups[g.Name] = struct{}{}
+			g.File = file
+			groups = append(groups, g)
+		}
+	}
+	if err := errGroup.Err(); err != nil {
+		return nil, err
+	}
+	if len(groups) < 1 {
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parseFile(path string) ([]Group, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
+	}
+	data = envtemplate.Replace(data)
+	g := struct {
+		Groups []Group `yaml:"groups"`
+		// Catches all undefined fields and must be empty after parsing.
+		XXX map[string]interface{} `yaml:",inline"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	if err != nil {
+		return nil, err
+	}
+	return g.Groups, checkOverflow(g.XXX, "config")
+}
+
+func checkOverflow(m map[string]interface{}, ctx string) error {
+	if len(m) > 0 {
+		var keys []string
+		for k := range m {
+			keys = append(keys, k)
+		}
+		return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", "))
+	}
+	return nil
+}
+
+type item struct {
+	key, value string
+}
+
+func sortMap(m map[string]string) []item {
+	var kv []item
+	for k, v := range m {
+		kv = append(kv, item{key: k, value: v})
+	}
+	sort.Slice(kv, func(i, j int) bool {
+		return kv[i].key < kv[j].key
+	})
+	return kv
+}

app/vmalert/config/config_test.go

@@ -0,0 +1,377 @@
+package config
+
+import (
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"gopkg.in/yaml.v2"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"value\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+	}
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, true, true)
+		if err == nil {
+			t.Errorf("expected to get error")
+			return
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestRule_Validate(t *testing.T) {
+	if err := (&Rule{}).Validate(); err == nil {
+		t.Errorf("expected empty name error")
+	}
+	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
+		t.Errorf("expected empty expr error")
+	}
+	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("expected valid rule; got %s", err)
+	}
+}
+
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group:  &Group{Name: "test"},
+			expErr: "contain no rules",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr:              "error parsing annotation",
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+	}
+	for _, tc := range testCases {
+		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: NewPromDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
+}
+
+func TestGroupChecksum(t *testing.T) {
+	f := func(t *testing.T, data, newData string) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		if g.Checksum == "" {
+			t.Fatalf("expected to get non-empty checksum")
+		}
+
+		var ng Group
+		if err := yaml.Unmarshal([]byte(newData), &ng); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		if g.Checksum == ng.Checksum {
+			t.Fatalf("expected to get different checksums")
+		}
+	}
+	t.Run("Ok", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`, `
+name: TestGroup
+rules:
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("Ok, `for` must change cs", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+    for: 5m
+`, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+}

app/vmalert/config/testdata/dir/rules0-good.rules

@@ -6,6 +6,7 @@ groups:
         expr: vm_rows > 0
         labels:
           label: bar
+          expr: "{{ $expr|queryEscape }}"
         annotations:
           summary: "{{ $value|humanize }}"
           description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -9,5 +9,3 @@ groups:
         annotations:
           summary: "{{ $value }}"
           description: "{{$labels}}"
-
-

app/vmalert/config/testdata/dir/rules3-bad.rules

@@ -0,0 +1,5 @@
+groups:
+  - name: group
+    rules:
+      - for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules4-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        record: record
+        for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules5-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        expr: vm_rows > 0
+      - record: rows
+        expr: sum(vm_rows)

app/vmalert/config/testdata/rules2-good.rules

@@ -0,0 +1,35 @@
+groups:
+  - name: TestGroup
+    interval: 2s
+    concurrency: 2
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        for: 3m
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)
+      - record: handler:requests:rate5m
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+        labels:
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: dev
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: staging
+          recording: true
+      - record: successful_requests:ratio_rate5m
+        labels:
+          recording: true
+        expr: |2
+            sum(code:requests:rate5m{code="200"})
+          /
+            sum(code:requests:rate5m)

app/vmalert/config_test.go

@@ -1,39 +0,0 @@
-package main
-
-import (
-	"net/url"
-	"os"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-)
-
-func TestMain(m *testing.M) {
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	notifier.InitTemplateFunc(u)
-	os.Exit(m.Run())
-}
-
-func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true); err != nil {
-		t.Errorf("error parsing files %s", err)
-	}
-}
-
-func TestParseBad(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules0-bad.rules"}, true); err == nil {
-		t.Errorf("expected syntaxt error")
-	}
-	if _, err := Parse([]string{"testdata/dir/rules0-bad.rules"}, true); err == nil {
-		t.Errorf("expected template annotation error")
-	}
-	if _, err := Parse([]string{"testdata/dir/rules1-bad.rules"}, true); err == nil {
-		t.Errorf("expected same group error")
-	}
-	if _, err := Parse([]string{"testdata/dir/rules2-bad.rules"}, true); err == nil {
-		t.Errorf("expected template label error")
-	}
-	if _, err := Parse([]string{"testdata/*.yaml"}, true); err == nil {
-		t.Errorf("expected empty group")
-	}
-}

app/vmalert/datasource/datasource.go

@@ -17,6 +17,23 @@ type Metric struct {
 	Value     float64
 }
 
+// SetLabel adds or updates existing one label
+// by the given key and label
+func (m *Metric) SetLabel(key, value string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels[i].Value = value
+			return
+		}
+	}
+	m.AddLabel(key, value)
+}
+
+// AddLabel appends the given label to the label set
+func (m *Metric) AddLabel(key, value string) {
+	m.Labels = append(m.Labels, Label{Name: key, Value: value})
+}
+
 // Label represents metric's label
 type Label struct {
 	Name  string

app/vmalert/datasource/init.go

@@ -0,0 +1,43 @@
+package datasource
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+
+	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
+	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
+	tlsKeyFile            = flag.String("datasource.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -datasource.url")
+	tlsCAFile             = flag.String("datasource.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -datasource.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
+		"By default the server name from -datasource.url is used")
+
+	lookBack = flag.Duration("datasource.lookback", 0, "Lookback defines how far to look into past when evaluating queries. "+
+		"For example, if datasource.lookback=5m then param \"time\" with value now()-5m will be added to every query.")
+	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, "Defines the number of idle (keep-alive connections) to configured datasource."+
+		"Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state.")
+)
+
+// Init creates a Querier from provided flag values.
+func Init() (Querier, error) {
+	if *addr == "" {
+		return nil, fmt.Errorf("datasource.url is empty")
+	}
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+	tr.MaxIdleConns = *maxIdleConnections
+	c := &http.Client{Transport: tr}
+	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, *lookBack, c), nil
+}

app/vmalert/datasource/vm.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 )
 
 type response struct {
@@ -32,11 +33,11 @@ func (r response) metrics() ([]Metric, error) {
 	for i, res := range r.Data.Result {
 		f, err = strconv.ParseFloat(res.TV[1].(string), 64)
 		if err != nil {
-			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %s", res, res.TV[1], err)
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
 		}
 		m.Labels = nil
 		for k, v := range r.Data.Result[i].Labels {
-			m.Labels = append(m.Labels, Label{Name: k, Value: v})
+			m.AddLabel(k, v)
 		}
 		m.Timestamp = int64(res.TV[0].(float64))
 		m.Value = f
@@ -45,22 +46,25 @@ func (r response) metrics() ([]Metric, error) {
 	return ms, nil
 }
 
-const queryPath = "/api/v1/query?query="
-
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
-	c                            *http.Client
-	queryURL                     string
-	basicAuthUser, basicAuthPass string
+	c             *http.Client
+	queryURL      string
+	basicAuthUser string
+	basicAuthPass string
+	lookBack      time.Duration
 }
 
+const queryPath = "/api/v1/query?query="
+
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, c *http.Client) *VMStorage {
 	return &VMStorage{
 		c:             c,
 		basicAuthUser: basicAuthUser,
 		basicAuthPass: basicAuthPass,
 		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+		lookBack:      lookBack,
 	}
 }
 
@@ -69,35 +73,40 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	const (
 		statusSuccess, statusError, rtVector = "success", "error", "vector"
 	)
-	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	q := s.queryURL + url.QueryEscape(query)
+	if s.lookBack > 0 {
+		lookBack := time.Now().Add(-s.lookBack)
+		q += fmt.Sprintf("&time=%d", lookBack.Unix())
+	}
+	req, err := http.NewRequest("POST", q, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if s.basicAuthPass != "" {
 		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
 	}
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s:%s", req.URL, err)
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err)
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("datasource returns unxeprected response code %d for %s with err %s. Reponse body %s", resp.StatusCode, req.URL, err, body)
+		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
 	}
 	r := &response{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing metrics for %s:%s", req.URL, err)
+		return nil, fmt.Errorf("error parsing metrics for %s: %w", req.URL, err)
 	}
 	if r.Status == statusError {
 		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
-		return nil, fmt.Errorf("unkown status:%s, Expected success or error ", r.Status)
+		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
 	}
 	if r.Data.ResultType != rtVector {
-		return nil, fmt.Errorf("unkown restul type:%s. Expected vector", r.Data.ResultType)
+		return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
 	}
 	return r.metrics()
 }

app/vmalert/datasource/vm_test.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
+	"time"
 )
 
 var (
@@ -29,7 +31,14 @@ func TestVMSelectQuery(t *testing.T) {
 			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("exptected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		timeParam := r.URL.Query().Get("time")
+		if timeParam == "" {
+			t.Errorf("expected 'time' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(timeParam, 10, 64); err != nil {
+			t.Errorf("failed to parse 'time' query param: %s", err)
 		}
 		switch c {
 		case 0:
@@ -52,7 +61,7 @@ func TestVMSelectQuery(t *testing.T) {
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, srv.Client())
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
@@ -66,7 +75,7 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("expected error status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
-		t.Fatalf("expected unkown status got nil")
+		t.Fatalf("expected unknown status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
@@ -76,7 +85,7 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("unexpected %s", err)
 	}
 	if len(m) != 1 {
-		t.Fatalf("exptected 1 metric  got %d in %+v", len(m), m)
+		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
 		Labels:    []Label{{Value: "vm_rows", Name: "__name__"}},
@@ -89,5 +98,4 @@ func TestVMSelectQuery(t *testing.T) {
 		m[0].Labels[0].Name != expected.Labels[0].Name {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
-
 }

app/vmalert/group.go

@@ -4,28 +4,97 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 // Group is an entity for grouping rules
 type Group struct {
-	Name  string
-	File  string
-	Rules []*Rule
+	mu          sync.RWMutex
+	Name        string
+	File        string
+	Rules       []Rule
+	Interval    time.Duration
+	Concurrency int
+	Checksum    string
 
-	done     chan struct{}
-	finished chan struct{}
+	doneCh     chan struct{}
+	finishedCh chan struct{}
+	// channel accepts new Group obj
+	// which supposed to update current group
+	updateCh chan *Group
+
+	metrics *groupMetrics
+}
+
+type groupMetrics struct {
+	iterationTotal    *counter
+	iterationDuration *summary
+}
+
+func newGroupMetrics(name, file string) *groupMetrics {
+	m := &groupMetrics{}
+	labels := fmt.Sprintf(`group=%q, file=%q`, name, file)
+	m.iterationTotal = getOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = getOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	return m
+}
+
+func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string]string) *Group {
+	g := &Group{
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval,
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		doneCh:      make(chan struct{}),
+		finishedCh:  make(chan struct{}),
+		updateCh:    make(chan *Group),
+	}
+	g.metrics = newGroupMetrics(g.Name, g.File)
+	if g.Interval == 0 {
+		g.Interval = defaultInterval
+	}
+	if g.Concurrency < 1 {
+		g.Concurrency = 1
+	}
+	rules := make([]Rule, len(cfg.Rules))
+	for i, r := range cfg.Rules {
+		// override rule labels with external labels
+		for k, v := range labels {
+			if prevV, ok := r.Labels[k]; ok {
+				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+					k, prevV, g.Name, r.Name(), k, v)
+			}
+			if r.Labels == nil {
+				r.Labels = map[string]string{}
+			}
+			r.Labels[k] = v
+		}
+		rules[i] = g.newRule(r)
+	}
+	g.Rules = rules
+	return g
+}
+
+func (g *Group) newRule(rule config.Rule) Rule {
+	if rule.Alert != "" {
+		return newAlertingRule(g, rule)
+	}
+	return newRecordingRule(g, rule)
 }
 
 // ID return unique group ID that consists of
 // rules file and group name
-func (g Group) ID() uint64 {
+func (g *Group) ID() uint64 {
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -33,133 +102,256 @@ func (g Group) ID() uint64 {
 	return hash.Sum64()
 }
 
-// Restore restores alerts state for all group rules with For > 0
-func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+// Restore restores alerts state for group rules
+func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
 	for _, rule := range g.Rules {
-		if rule.For == 0 {
-			return nil
+		rr, ok := rule.(*AlertingRule)
+		if !ok {
+			continue
 		}
-		if err := rule.Restore(ctx, q, lookback); err != nil {
-			return fmt.Errorf("error while restoring rule %q: %s", rule.Name, err)
+		if rr.For < 1 {
+			continue
+		}
+		if err := rr.Restore(ctx, q, lookback, labels); err != nil {
+			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
 		}
 	}
 	return nil
 }
 
 // updateWith updates existing group with
-// passed group object.
-func (g *Group) updateWith(newGroup Group) {
-	rulesRegistry := make(map[string]*Rule)
+// passed group object. This function ignores group
+// evaluation interval change. It supposed to be updated
+// in group.start function.
+// Not thread-safe.
+func (g *Group) updateWith(newGroup *Group) error {
+	rulesRegistry := make(map[uint64]Rule)
 	for _, nr := range newGroup.Rules {
-		rulesRegistry[nr.id()] = nr
+		rulesRegistry[nr.ID()] = nr
 	}
 
 	for i, or := range g.Rules {
-		nr, ok := rulesRegistry[or.id()]
+		nr, ok := rulesRegistry[or.ID()]
 		if !ok {
 			// old rule is not present in the new list
-			// and must be removed
-			or = nil
-			g.Rules = append(g.Rules[:i], g.Rules[i+1:]...)
+			// so we mark it for removing
+			g.Rules[i].Close()
+			g.Rules[i] = nil
 			continue
 		}
-
-		// copy all significant fields.
-		// alerts state isn't copied since
-		// it should be updated in next 2 Evals
-		or.For = nr.For
-		or.Expr = nr.Expr
-		or.Labels = nr.Labels
-		or.Annotations = nr.Annotations
-		delete(rulesRegistry, nr.id())
+		if err := or.UpdateWith(nr); err != nil {
+			return err
+		}
+		delete(rulesRegistry, nr.ID())
 	}
 
+	var newRules []Rule
+	for _, r := range g.Rules {
+		if r == nil {
+			// skip nil rules
+			continue
+		}
+		newRules = append(newRules, r)
+	}
+	// add the rest of rules from registry
 	for _, nr := range rulesRegistry {
-		g.Rules = append(g.Rules, nr)
+		newRules = append(newRules, nr)
 	}
+	g.Concurrency = newGroup.Concurrency
+	g.Checksum = newGroup.Checksum
+	g.Rules = newRules
+	return nil
 }
 
 var (
-	iterationTotal    = metrics.NewCounter(`vmalert_iteration_total`)
-	iterationDuration = metrics.NewSummary(`vmalert_iteration_duration_seconds`)
-
-	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
-	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
-	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
-
 	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
 	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
 	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
-
-	remoteWriteSent   = metrics.NewCounter(`vmalert_remotewrite_sent_total`)
-	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 )
 
 func (g *Group) close() {
-	if g.done == nil {
+	if g.doneCh == nil {
 		return
 	}
-	close(g.done)
-	<-g.finished
+	close(g.doneCh)
+	<-g.finishedCh
+
+	metrics.UnregisterMetric(g.metrics.iterationDuration.name)
+	metrics.UnregisterMetric(g.metrics.iterationTotal.name)
+	for _, rule := range g.Rules {
+		rule.Close()
+	}
 }
 
-func (g *Group) start(ctx context.Context, interval time.Duration,
-	querier datasource.Querier, nr notifier.Notifier, rw *remotewrite.Client) {
-	logger.Infof("group %q started", g.Name)
-	t := time.NewTicker(interval)
+var skipRandSleepOnGroupStart bool
+
+func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []notifier.Notifier, rw *remotewrite.Client) {
+	defer func() { close(g.finishedCh) }()
+
+	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	if !skipRandSleepOnGroupStart {
+		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(g.Interval)
+		}
+		randSleep -= sleepOffset
+		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		select {
+		case <-ctx.Done():
+			sleepTimer.Stop()
+			return
+		case <-g.doneCh:
+			sleepTimer.Stop()
+			return
+		case <-sleepTimer.C:
+		}
+	}
+
+	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+	e := &executor{querier, nts, rw}
+	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
 		select {
 		case <-ctx.Done():
 			logger.Infof("group %q: context cancelled", g.Name)
-			close(g.finished)
 			return
-		case <-g.done:
+		case <-g.doneCh:
 			logger.Infof("group %q: received stop signal", g.Name)
-			close(g.finished)
 			return
+		case ng := <-g.updateCh:
+			g.mu.Lock()
+			err := g.updateWith(ng)
+			if err != nil {
+				logger.Errorf("group %q: failed to update: %s", g.Name, err)
+				g.mu.Unlock()
+				continue
+			}
+			if g.Interval != ng.Interval {
+				g.Interval = ng.Interval
+				t.Stop()
+				t = time.NewTicker(g.Interval)
+			}
+			g.mu.Unlock()
+			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
-			iterationTotal.Inc()
+			g.metrics.iterationTotal.Inc()
 			iterationStart := time.Now()
-			for _, rule := range g.Rules {
-				execTotal.Inc()
-
-				execStart := time.Now()
-				err := rule.Exec(ctx, querier)
-				execDuration.UpdateDuration(execStart)
 
+			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
+			for err := range errs {
 				if err != nil {
-					execErrors.Inc()
-					logger.Errorf("failed to execute rule %q.%q: %s", g.Name, rule.Name, err)
-					continue
-				}
-
-				var alertsToSend []notifier.Alert
-				for _, a := range rule.alerts {
-					if a.State != notifier.StatePending {
-						alertsToSend = append(alertsToSend, *a)
-					}
-					if a.State == notifier.StateInactive || rw == nil {
-						continue
-					}
-					tss := rule.AlertToTimeSeries(a, execStart)
-					for _, ts := range tss {
-						remoteWriteSent.Inc()
-						if err := rw.Push(ts); err != nil {
-							remoteWriteErrors.Inc()
-							logger.Errorf("failed to push timeseries to remotewrite: %s", err)
-						}
-					}
-				}
-				if len(alertsToSend) > 0 {
-					alertsSent.Add(len(alertsToSend))
-					if err := nr.Send(ctx, alertsToSend); err != nil {
-						alertsSendErrors.Inc()
-						logger.Errorf("failed to send alert for rule %q.%q: %s", g.Name, rule.Name, err)
-					}
+					logger.Errorf("group %q: %s", g.Name, err)
 				}
 			}
-			iterationDuration.UpdateDuration(iterationStart)
+
+			g.metrics.iterationDuration.UpdateDuration(iterationStart)
 		}
 	}
 }
+
+type executor struct {
+	querier   datasource.Querier
+	notifiers []notifier.Notifier
+	rw        *remotewrite.Client
+}
+
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
+	res := make(chan error, len(rules))
+	var returnSeries bool
+	if e.rw != nil {
+		returnSeries = true
+	}
+
+	if concurrency == 1 {
+		// fast path
+		for _, rule := range rules {
+			res <- e.exec(ctx, rule, returnSeries, interval)
+		}
+		close(res)
+		return res
+	}
+
+	sem := make(chan struct{}, concurrency)
+	go func() {
+		wg := sync.WaitGroup{}
+		for _, rule := range rules {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func(r Rule) {
+				res <- e.exec(ctx, r, returnSeries, interval)
+				<-sem
+				wg.Done()
+			}(rule)
+		}
+		wg.Wait()
+		close(res)
+	}()
+	return res
+}
+
+var (
+	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
+	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+)
+
+func (e *executor) exec(ctx context.Context, rule Rule, returnSeries bool, interval time.Duration) error {
+	execTotal.Inc()
+	execStart := time.Now()
+	defer func() {
+		execDuration.UpdateDuration(execStart)
+	}()
+
+	tss, err := rule.Exec(ctx, e.querier, returnSeries)
+	if err != nil {
+		execErrors.Inc()
+		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
+	}
+
+	if len(tss) > 0 && e.rw != nil {
+		for _, ts := range tss {
+			if err := e.rw.Push(ts); err != nil {
+				remoteWriteErrors.Inc()
+				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)
+			}
+		}
+	}
+
+	ar, ok := rule.(*AlertingRule)
+	if !ok {
+		return nil
+	}
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		switch a.State {
+		case notifier.StateFiring:
+			// set End to execStart + 3 intervals
+			// so notifier can resolve it automatically if `vmalert`
+			// won't be able to send resolve for some reason
+			a.End = time.Now().Add(3 * interval)
+			alerts = append(alerts, *a)
+		case notifier.StateInactive:
+			// set End to execStart to notify
+			// that it was just resolved
+			a.End = time.Now()
+			alerts = append(alerts, *a)
+		}
+	}
+	if len(alerts) < 1 {
+		return nil
+	}
+
+	alertsSent.Add(len(alerts))
+	errGr := new(utils.ErrGroup)
+	for _, nt := range e.notifiers {
+		if err := nt.Send(ctx, alerts); err != nil {
+			alertsSendErrors.Inc()
+			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
+		}
+	}
+	return errGr.Err()
+}

app/vmalert/group_test.go

@@ -2,32 +2,37 @@ package main
 
 import (
 	"context"
-	"reflect"
 	"sort"
-	"sync"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
 
+func init() {
+	// Disable rand sleep on group start during tests in order to speed up test execution.
+	// Rand sleep is needed only in prod code.
+	skipRandSleepOnGroupStart = true
+}
+
 func TestUpdateWith(t *testing.T) {
 	testCases := []struct {
 		name         string
-		currentRules []*Rule
-		newRules     []*Rule
+		currentRules []config.Rule
+		newRules     []config.Rule
 	}{
 		{
 			"new rule",
-			[]*Rule{},
-			[]*Rule{{Name: "bar"}},
+			nil,
+			[]config.Rule{{Alert: "bar"}},
 		},
 		{
-			"update rule",
-			[]*Rule{{
-				Name: "foo",
-				Expr: "up > 0",
-				For:  time.Second,
+			"update alerting rule",
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 0",
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -36,10 +41,10 @@ func TestUpdateWith(t *testing.T) {
 					"description": "{{$labels}}",
 				},
 			}},
-			[]*Rule{{
-				Name: "bar",
-				Expr: "up > 10",
-				For:  time.Second,
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 10",
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -48,43 +53,96 @@ func TestUpdateWith(t *testing.T) {
 				},
 			}},
 		},
+		{
+			"update recording rule",
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "max(up)",
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			}},
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "min(up)",
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+			}},
+		},
 		{
 			"empty rule",
-			[]*Rule{{Name: "foo"}},
-			[]*Rule{},
+			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
+			nil,
 		},
 		{
 			"multiple rules",
-			[]*Rule{{Name: "foo"}, {Name: "bar"}, {Name: "baz"}},
-			[]*Rule{{Name: "foo"}, {Name: "baz"}},
+			[]config.Rule{
+				{Alert: "bar"},
+				{Alert: "baz"},
+				{Alert: "foo"},
+			},
+			[]config.Rule{
+				{Alert: "baz"},
+				{Record: "foo"},
+			},
+		},
+		{
+			"replace rule",
+			[]config.Rule{{Alert: "foo1"}},
+			[]config.Rule{{Alert: "foo2"}},
+		},
+		{
+			"replace multiple rules",
+			[]config.Rule{
+				{Alert: "foo1"},
+				{Record: "foo2"},
+				{Alert: "foo3"},
+			},
+			[]config.Rule{
+				{Alert: "foo3"},
+				{Alert: "foo4"},
+				{Record: "foo5"},
+			},
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			g := &Group{Rules: tc.currentRules}
-			g.updateWith(Group{Rules: tc.newRules})
+			g := &Group{Name: "test"}
+			for _, r := range tc.currentRules {
+				r.ID = config.HashRule(r)
+				g.Rules = append(g.Rules, g.newRule(r))
+			}
+
+			ng := &Group{Name: "test"}
+			for _, r := range tc.newRules {
+				r.ID = config.HashRule(r)
+				ng.Rules = append(ng.Rules, ng.newRule(r))
+			}
+
+			err := g.updateWith(ng)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			if len(g.Rules) != len(tc.newRules) {
 				t.Fatalf("expected to have %d rules; got: %d",
 					len(g.Rules), len(tc.newRules))
 			}
+			sort.Slice(g.Rules, func(i, j int) bool {
+				return g.Rules[i].ID() < g.Rules[j].ID()
+			})
+			sort.Slice(ng.Rules, func(i, j int) bool {
+				return ng.Rules[i].ID() < ng.Rules[j].ID()
+			})
 			for i, r := range g.Rules {
-				got, want := r, tc.newRules[i]
-				if got.Name != want.Name {
-					t.Fatalf("expected to have rule %q; got %q", want.Name, got.Name)
+				got, want := r, ng.Rules[i]
+				if got.ID() != want.ID() {
+					t.Fatalf("expected to have rule %q; got %q", want, got)
 				}
-				if got.Expr != want.Expr {
-					t.Fatalf("expected to have expression %q; got %q", want.Expr, got.Expr)
-				}
-				if got.For != want.For {
-					t.Fatalf("expected to have for %q; got %q", want.For, got.For)
-				}
-				if !reflect.DeepEqual(got.Annotations, want.Annotations) {
-					t.Fatalf("expected to have annotations %#v; got %#v", want.Annotations, got.Annotations)
-				}
-				if !reflect.DeepEqual(got.Labels, want.Labels) {
-					t.Fatalf("expected to have labels %#v; got %#v", want.Labels, got.Labels)
+				if err := compareRules(t, got, want); err != nil {
+					t.Fatalf("comparsion error: %s", err)
 				}
 			}
 		})
@@ -93,11 +151,13 @@ func TestUpdateWith(t *testing.T) {
 
 func TestGroupStart(t *testing.T) {
 	// TODO: make parsing from string instead of file
-	groups, err := Parse([]string{"testdata/rules1-good.rules"}, true)
+	groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
 	if err != nil {
 		t.Fatalf("failed to parse rules: %s", err)
 	}
-	g := groups[0]
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], evalInterval, map[string]string{"cluster": "east-1"})
+	g.Concurrency = 2
 
 	fn := &fakeNotifier{}
 	fs := &fakeQuerier{}
@@ -106,27 +166,36 @@ func TestGroupStart(t *testing.T) {
 	m1 := metricWithLabels(t, "instance", inst1, "job", job)
 	m2 := metricWithLabels(t, "instance", inst2, "job", job)
 
-	r := g.Rules[0]
-	alert1, err := r.newAlert(m1)
+	r := g.Rules[0].(*AlertingRule)
+	alert1, err := r.newAlert(m1, time.Now())
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
 	alert1.State = notifier.StateFiring
+	// add external label
+	alert1.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert1.Labels["label"] = "bar"
+	alert1.Labels["host"] = inst1
 	alert1.ID = hash(m1)
 
-	alert2, err := r.newAlert(m2)
+	alert2, err := r.newAlert(m2, time.Now())
 	if err != nil {
 		t.Fatalf("faield to create alert: %s", err)
 	}
 	alert2.State = notifier.StateFiring
+	// add external label
+	alert2.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert2.Labels["label"] = "bar"
+	alert2.Labels["host"] = inst2
 	alert2.ID = hash(m2)
 
-	const evalInterval = time.Millisecond
 	finished := make(chan struct{})
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), evalInterval, fs, fn, nil)
+		g.start(context.Background(), fs, []notifier.Notifier{fn}, nil)
 		close(finished)
 	}()
 
@@ -152,52 +221,3 @@ func TestGroupStart(t *testing.T) {
 	g.close()
 	<-finished
 }
-
-func compareAlerts(t *testing.T, as, bs []notifier.Alert) {
-	t.Helper()
-	if len(as) != len(bs) {
-		t.Fatalf("expected to have length %d; got %d", len(as), len(bs))
-	}
-	sort.Slice(as, func(i, j int) bool {
-		return as[i].ID < as[j].ID
-	})
-	sort.Slice(bs, func(i, j int) bool {
-		return bs[i].ID < bs[j].ID
-	})
-	for i := range as {
-		a, b := as[i], bs[i]
-		if a.Name != b.Name {
-			t.Fatalf("expected t have Name %q; got %q", a.Name, b.Name)
-		}
-		if a.State != b.State {
-			t.Fatalf("expected t have State %q; got %q", a.State, b.State)
-		}
-		if a.Value != b.Value {
-			t.Fatalf("expected t have Value %f; got %f", a.Value, b.Value)
-		}
-		if !reflect.DeepEqual(a.Annotations, b.Annotations) {
-			t.Fatalf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
-		}
-		if !reflect.DeepEqual(a.Labels, b.Labels) {
-			t.Fatalf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
-		}
-	}
-}
-
-type fakeNotifier struct {
-	sync.Mutex
-	alerts []notifier.Alert
-}
-
-func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
-	fn.Lock()
-	defer fn.Unlock()
-	fn.alerts = alerts
-	return nil
-}
-
-func (fn *fakeNotifier) getAlerts() []notifier.Alert {
-	fn.Lock()
-	defer fn.Unlock()
-	return fn.alerts
-}

app/vmalert/helpers_test.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+type fakeQuerier struct {
+	sync.Mutex
+	metrics []datasource.Metric
+	err     error
+}
+
+func (fq *fakeQuerier) setErr(err error) {
+	fq.Lock()
+	fq.err = err
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) reset() {
+	fq.Lock()
+	fq.err = nil
+	fq.metrics = fq.metrics[:0]
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
+	fq.Lock()
+	fq.metrics = append(fq.metrics, metrics...)
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
+	fq.Lock()
+	defer fq.Unlock()
+	if fq.err != nil {
+		return nil, fq.err
+	}
+	cp := make([]datasource.Metric, len(fq.metrics))
+	copy(cp, fq.metrics)
+	return cp, nil
+}
+
+type fakeNotifier struct {
+	sync.Mutex
+	alerts []notifier.Alert
+}
+
+func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
+	fn.Lock()
+	defer fn.Unlock()
+	fn.alerts = alerts
+	return nil
+}
+
+func (fn *fakeNotifier) getAlerts() []notifier.Alert {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.alerts
+}
+
+func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
+	t.Helper()
+	m := metricWithLabels(t, labels...)
+	m.Value = value
+	return m
+}
+
+func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	m := datasource.Metric{}
+	for i := 0; i < len(labels); i += 2 {
+		m.Labels = append(m.Labels, datasource.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return m
+}
+
+func compareGroups(t *testing.T, a, b *Group) {
+	t.Helper()
+	if a.Name != b.Name {
+		t.Fatalf("expected group name %q; got %q", a.Name, b.Name)
+	}
+	if a.File != b.File {
+		t.Fatalf("expected group %q file name %q; got %q", a.Name, a.File, b.File)
+	}
+	if a.Interval != b.Interval {
+		t.Fatalf("expected group %q interval %v; got %v", a.Name, a.Interval, b.Interval)
+	}
+	if len(a.Rules) != len(b.Rules) {
+		t.Fatalf("expected group %s to have %d rules; got: %d",
+			a.Name, len(a.Rules), len(b.Rules))
+	}
+	for i, r := range a.Rules {
+		got, want := r, b.Rules[i]
+		if a.ID() != b.ID() {
+			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+		}
+		if err := compareRules(t, want, got); err != nil {
+			t.Fatalf("comparsion error: %s", err)
+		}
+	}
+}
+
+func compareRules(t *testing.T, a, b Rule) error {
+	t.Helper()
+	switch v := a.(type) {
+	case *AlertingRule:
+		br, ok := b.(*AlertingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
+		}
+		return compareAlertingRules(t, v, br)
+	case *RecordingRule:
+		br, ok := b.(*RecordingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
+		}
+		return compareRecordingRules(t, v, br)
+	default:
+		return fmt.Errorf("unexpected rule type received %T", a)
+	}
+}
+
+func compareRecordingRules(t *testing.T, a, b *RecordingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	return nil
+}
+
+func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if a.For != b.For {
+		return fmt.Errorf("expected to have for %q; got %q", a.For, b.For)
+	}
+	if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+		return fmt.Errorf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	return nil
+}
+
+func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
+	t.Helper()
+	if len(a) != len(b) {
+		return fmt.Errorf("expected number of timeseries %d; got %d", len(a), len(b))
+	}
+	for i := range a {
+		expTS, gotTS := a[i], b[i]
+		if len(expTS.Samples) != len(gotTS.Samples) {
+			return fmt.Errorf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+		}
+		for i, exp := range expTS.Samples {
+			got := gotTS.Samples[i]
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+			}
+			// timestamp validation isn't always correct for now.
+			// this must be improved with time mock.
+			/*if got.Timestamp != exp.Timestamp {
+				return fmt.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+			}*/
+		}
+		if len(expTS.Labels) != len(gotTS.Labels) {
+			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+		}
+		for i, exp := range expTS.Labels {
+			got := gotTS.Labels[i]
+			if got.Name != exp.Name {
+				return fmt.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+			}
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+			}
+		}
+	}
+	return nil
+}
+
+func compareAlerts(t *testing.T, as, bs []notifier.Alert) {
+	t.Helper()
+	if len(as) != len(bs) {
+		t.Fatalf("expected to have length %d; got %d", len(as), len(bs))
+	}
+	sort.Slice(as, func(i, j int) bool {
+		return as[i].ID < as[j].ID
+	})
+	sort.Slice(bs, func(i, j int) bool {
+		return bs[i].ID < bs[j].ID
+	})
+	for i := range as {
+		a, b := as[i], bs[i]
+		if a.Name != b.Name {
+			t.Fatalf("expected t have Name %q; got %q", a.Name, b.Name)
+		}
+		if a.State != b.State {
+			t.Fatalf("expected t have State %q; got %q", a.State, b.State)
+		}
+		if a.Value != b.Value {
+			t.Fatalf("expected t have Value %f; got %f", a.Value, b.Value)
+		}
+		if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+			t.Fatalf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+		}
+		if !reflect.DeepEqual(a.Labels, b.Labels) {
+			t.Fatalf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+		}
+	}
+}

app/vmalert/main.go

@@ -4,17 +4,21 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"net/http"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -26,97 +30,81 @@ var (
 	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules. 
 Supports patterns. Flag can be specified multiple times. 
 Examples:
- -rule /path/to/file. Path to a single file with alerting rules
- -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
-absolute path to all .yaml files in root.`)
-	validateTemplates = flag.Bool("rule.validateTemplates", true, "Indicates to validate annotation and label templates")
-	httpListenAddr    = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+ -rule="/path/to/file". Path to a single file with alerting rules
+ -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, 
+absolute path to all .yaml files in root.
+Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 
-	datasourceURL = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
-		" E.g. http://127.0.0.1:8428")
-	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
-	remoteWriteURL = flag.String("remotewrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
-		" in form of timeseries. E.g. http://127.0.0.1:8428")
-	remoteWriteUsername = flag.String("remotewrite.basicAuth.username", "", "Optional basic auth username for -remotewrite.url")
-	remoteWritePassword = flag.String("remotewrite.basicAuth.password", "", "Optional basic auth password for -remotewrite.url")
+	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
+	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
+eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
+	externalLabels = flagutil.NewArray("external.label", "Optional label in the form 'name=value' to add to all generated recording rules and alerts. "+
+		"Pass multiple -label flags in order to add multiple label sets.")
 
-	remoteReadURL = flag.String("remoteread.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
-		" state. This configuration makes sense only if `vmalert` was configured with `remotewrite.url` before and has been successfully persisted its state."+
-		" E.g. http://127.0.0.1:8428")
-	remoteReadUsername = flag.String("remoteread.basicAuth.username", "", "Optional basic auth username for -remoteread.url")
-	remoteReadPassword = flag.String("remoteread.basicAuth.password", "", "Optional basic auth password for -remoteread.url")
-	remoteReadLookBack = flag.Duration("remoteread.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
+	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 
-	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules. Default 1m")
-	notifierURL        = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
-	externalURL        = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
 func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	checkFlags()
-	ctx, cancel := context.WithCancel(context.Background())
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		logger.Fatalf("can not get external url: %s ", err)
-	}
-	notifier.InitTemplateFunc(eu)
+	cgroup.UpdateGOMAXPROCSToCPUQuota()
 
-	manager := &manager{
-		groups:  make(map[uint64]*Group),
-		storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
-		notifier: notifier.NewAlertManager(*notifierURL, func(group, alert string) string {
-			return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, alert)
-		}, &http.Client{}),
-	}
-	if *remoteWriteURL != "" {
-		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
-			Addr:          *remoteWriteURL,
-			FlushInterval: *evaluationInterval,
-			BasicAuthUser: *remoteWriteUsername,
-			BasicAuthPass: *remoteWritePassword,
-		})
+	if *dryRun {
+		u, _ := url.Parse("https://victoriametrics.com/")
+		notifier.InitTemplateFunc(u)
+		groups, err := config.Parse(*rulePath, true, true)
 		if err != nil {
-			logger.Fatalf("failed to init remotewrite client: %s", err)
+			logger.Fatalf(err.Error())
 		}
-		manager.rw = c
+		if len(groups) == 0 {
+			logger.Fatalf("No rules for validation. Please specify path to file(s) with alerting and/or recording rules using `-rule` flag")
+		}
+		return
 	}
-	if *remoteReadURL != "" {
-		manager.rr = datasource.NewVMStorage(*remoteReadURL, *remoteReadUsername, *remoteReadPassword, &http.Client{})
+	ctx, cancel := context.WithCancel(context.Background())
+	manager, err := newManager(ctx)
+	if err != nil {
+		logger.Fatalf("failed to init: %s", err)
 	}
-
-	if err := manager.start(ctx, *rulePath, *validateTemplates); err != nil {
+	if err := manager.start(ctx, *rulePath, *validateTemplates, *validateExpressions); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
 
 	go func() {
 		// init reload metrics with positive values to improve alerting conditions
 		configSuccess.Set(1)
-		configTimestamp.Set(uint64(time.Now().UnixNano()) / 1e9)
+		configTimestamp.Set(fasttime.UnixTimestamp())
 		sigHup := procutil.NewSighupChan()
 		for {
 			<-sigHup
 			configReloads.Inc()
 			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
-			if err := manager.update(ctx, *rulePath, *validateTemplates, false); err != nil {
+			if err := manager.update(ctx, *rulePath, *validateTemplates, *validateExpressions, false); err != nil {
 				configReloadErrors.Inc()
 				configSuccess.Set(0)
 				logger.Errorf("error while reloading rules: %s", err)
 				continue
 			}
 			configSuccess.Set(1)
-			configTimestamp.Set(uint64(time.Now().UnixNano()) / 1e9)
+			configTimestamp.Set(fasttime.UnixTimestamp())
 			logger.Infof("Rules reloaded successfully from %q", *rulePath)
 		}
 	}()
 
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, (rh).handler)
+	go httpserver.Serve(*httpListenAddr, rh.handler)
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)
@@ -134,6 +122,53 @@ var (
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
+func newManager(ctx context.Context) (*manager, error) {
+	q, err := datasource.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to init datasource: %w", err)
+	}
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		return nil, fmt.Errorf("failed to init `external.url`: %w", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init `external.alert.source`: %w", err)
+	}
+	nts, err := notifier.Init(aug)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init notifier: %w", err)
+	}
+
+	manager := &manager{
+		groups:    make(map[uint64]*Group),
+		querier:   q,
+		notifiers: nts,
+		labels:    map[string]string{},
+	}
+	rw, err := remotewrite.Init(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteWrite: %w", err)
+	}
+	manager.rw = rw
+
+	rr, err := remoteread.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteRead: %w", err)
+	}
+	manager.rr = rr
+
+	for _, s := range *externalLabels {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		manager.labels[s[:n]] = s[n+1:]
+	}
+	return manager, nil
+}
+
 func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
 	if externalURL != "" {
 		return url.Parse(externalURL)
@@ -153,13 +188,36 @@ func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL
 	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
 }
 
-func checkFlags() {
-	if *notifierURL == "" {
-		flag.PrintDefaults()
-		logger.Fatalf("notifier.url is empty")
+func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
+	if externalAlertSource == "" {
+		return func(alert notifier.Alert) string {
+			return fmt.Sprintf("%s/api/v1/%s/%s/status", externalURL, strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10))
+		}, nil
 	}
-	if *datasourceURL == "" {
-		flag.PrintDefaults()
-		logger.Fatalf("datasource.url is empty")
+	if validateTemplate {
+		if err := notifier.ValidateTemplates(map[string]string{
+			"tpl": externalAlertSource,
+		}); err != nil {
+			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
+		}
 	}
+	m := map[string]string{
+		"tpl": externalAlertSource,
+	}
+	return func(alert notifier.Alert) string {
+		templated, err := alert.ExecTemplate(m)
+		if err != nil {
+			logger.Errorf("can not exec source template %s", err)
+		}
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
+	}, nil
+}
+
+func usage() {
+	const s = `
+vmalert processes alerts and recording rules.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md .
+`
+	flagutil.Usage(s)
 }

app/vmalert/main_test.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestGetExternalURL(t *testing.T) {
+	expURL := "https://vicotriametrics.com/path"
+	u, err := getExternalURL(expURL, "", false)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if u.String() != expURL {
+		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+	}
+	h, _ := os.Hostname()
+	expURL = fmt.Sprintf("https://%s:4242", h)
+	u, err = getExternalURL("", "0.0.0.0:4242", true)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if u.String() != expURL {
+		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+	}
+}
+
+func TestGetAlertURLGenerator(t *testing.T) {
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4}
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	fn, err := getAlertURLGenerator(u, "", false)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if exp := "https://victoriametrics.com/path/api/v1/42/2/status"; exp != fn(testAlert) {
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
+	if err == nil {
+		t.Errorf("expected tempalte validation error got nil")
+	}
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}", true)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if exp := "https://victoriametrics.com/path/foo?query=4"; exp != fn(testAlert) {
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+}

app/vmalert/manager.go

@@ -6,26 +6,29 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
+// manager controls group states
 type manager struct {
-	storage  datasource.Querier
-	notifier notifier.Notifier
+	querier   datasource.Querier
+	notifiers []notifier.Notifier
 
 	rw *remotewrite.Client
 	rr datasource.Querier
 
-	wg sync.WaitGroup
+	wg     sync.WaitGroup
+	labels map[string]string
 
 	groupsMu sync.RWMutex
 	groups   map[uint64]*Group
 }
 
-// AlertAPI generates APIAlert object from alert by its id(hash)
+// AlertAPI generates APIAlert object from alert by its ID(hash)
 func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
@@ -35,15 +38,19 @@ func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
 		return nil, fmt.Errorf("can't find group with id %q", gID)
 	}
 	for _, rule := range g.Rules {
-		if apiAlert := rule.AlertAPI(aID); apiAlert != nil {
+		ar, ok := rule.(*AlertingRule)
+		if !ok {
+			continue
+		}
+		if apiAlert := ar.AlertAPI(aID); apiAlert != nil {
 			return apiAlert, nil
 		}
 	}
-	return nil, fmt.Errorf("can't func alert with id %q in group %q", aID, g.Name)
+	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
 }
 
-func (m *manager) start(ctx context.Context, path []string, validate bool) error {
-	return m.update(ctx, path, validate, true)
+func (m *manager) start(ctx context.Context, path []string, validateTpl, validateExpr bool) error {
+	return m.update(ctx, path, validateTpl, validateExpr, true)
 }
 
 func (m *manager) close() {
@@ -56,9 +63,9 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, group Group, restore bool) {
-	if restore {
-		err := group.Restore(ctx, m.rr, *remoteReadLookBack)
+func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
+	if restore && m.rr != nil {
+		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
 		if err != nil {
 			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
 		}
@@ -67,43 +74,85 @@ func (m *manager) startGroup(ctx context.Context, group Group, restore bool) {
 	m.wg.Add(1)
 	id := group.ID()
 	go func() {
-		group.start(ctx, *evaluationInterval, m.storage, m.notifier, m.rw)
+		group.start(ctx, m.querier, m.notifiers, m.rw)
 		m.wg.Done()
 	}()
-	m.groups[id] = &group
+	m.groups[id] = group
 }
 
-func (m *manager) update(ctx context.Context, path []string, validate, restore bool) error {
-	logger.Infof("reading alert rules configuration file from %q", strings.Join(path, ";"))
-	newGroups, err := Parse(path, validate)
+func (m *manager) update(ctx context.Context, path []string, validateTpl, validateExpr, restore bool) error {
+	logger.Infof("reading rules configuration file from %q", strings.Join(path, ";"))
+	groupsCfg, err := config.Parse(path, validateTpl, validateExpr)
 	if err != nil {
-		return fmt.Errorf("cannot parse configuration file: %s", err)
+		return fmt.Errorf("cannot parse configuration file: %w", err)
 	}
 
-	groupsRegistry := make(map[uint64]Group)
-	for _, ng := range newGroups {
+	groupsRegistry := make(map[uint64]*Group)
+	for _, cfg := range groupsCfg {
+		ng := newGroup(cfg, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	type updateItem struct {
+		old *Group
+		new *Group
+	}
+	var toUpdate []updateItem
+
 	m.groupsMu.Lock()
 	for _, og := range m.groups {
-		id := og.ID()
-		ng, ok := groupsRegistry[id]
+		ng, ok := groupsRegistry[og.ID()]
 		if !ok {
-			// old group is not present in new list
-			// and must be stopped and deleted
+			// old group is not present in new list,
+			// so must be stopped and deleted
 			og.close()
 			delete(m.groups, og.ID())
 			og = nil
 			continue
 		}
-		og.updateWith(ng)
 		delete(groupsRegistry, ng.ID())
+		if og.Checksum != ng.Checksum {
+			toUpdate = append(toUpdate, updateItem{old: og, new: ng})
+		}
 	}
-
 	for _, ng := range groupsRegistry {
 		m.startGroup(ctx, ng, restore)
 	}
 	m.groupsMu.Unlock()
+
+	if len(toUpdate) > 0 {
+		var wg sync.WaitGroup
+		for _, item := range toUpdate {
+			wg.Add(1)
+			go func(old *Group, new *Group) {
+				old.updateCh <- new
+				wg.Done()
+			}(item.old, item.new)
+		}
+		wg.Wait()
+	}
 	return nil
 }
+
+func (g *Group) toAPI() APIGroup {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	ag := APIGroup{
+		// encode as string to avoid rounding
+		ID:          fmt.Sprintf("%d", g.ID()),
+		Name:        g.Name,
+		File:        g.File,
+		Interval:    g.Interval.String(),
+		Concurrency: g.Concurrency,
+	}
+	for _, r := range g.Rules {
+		switch v := r.(type) {
+		case *AlertingRule:
+			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
+		case *RecordingRule:
+			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
+		}
+	}
+	return ag
+}

app/vmalert/manager_test.go

@@ -3,22 +3,30 @@ package main
 import (
 	"context"
 	"math/rand"
-	"strings"
+	"net/url"
+	"os"
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
 
-func TestManagerUpdateError(t *testing.T) {
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+// TestManagerEmptyRulesDir tests
+// successful cases of
+// starting with empty rules folder
+func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
 	path := []string{"foo/bar"}
-	err := m.update(context.Background(), path, true, false)
-	if err == nil {
-		t.Fatalf("expected to have err; got nil instead")
-	}
-	expErr := "no groups found"
-	if !strings.Contains(err.Error(), expErr) {
-		t.Fatalf("expected to got err %s; got %s", expErr, err)
+	err := m.update(context.Background(), path, true, true, false)
+	if err != nil {
+		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
 	}
 }
 
@@ -26,24 +34,36 @@ func TestManagerUpdateError(t *testing.T) {
 // execution of configuration update.
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
-	m := &manager{groups: make(map[uint64]*Group)}
+	m := &manager{
+		groups:    make(map[uint64]*Group),
+		querier:   &fakeQuerier{},
+		notifiers: []notifier.Notifier{&fakeNotifier{}},
+	}
 	paths := []string{
-		"testdata/dir/rules0-good.rules",
-		"testdata/dir/rules1-good.rules",
-		"testdata/rules0-good.rules",
+		"config/testdata/dir/rules0-good.rules",
+		"config/testdata/dir/rules0-bad.rules",
+		"config/testdata/dir/rules1-good.rules",
+		"config/testdata/dir/rules1-bad.rules",
+		"config/testdata/rules0-good.rules",
+		"config/testdata/rules1-good.rules",
+		"config/testdata/rules2-good.rules",
+	}
+	*evaluationInterval = time.Millisecond
+	if err := m.start(context.Background(), []string{paths[0]}, true, true); err != nil {
+		t.Fatalf("failed to start: %s", err)
 	}
 
-	const n = 500
+	const workers = 500
+	const iterations = 10
 	wg := sync.WaitGroup{}
-	wg.Add(n)
-	for i := 0; i < n; i++ {
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
 		go func() {
 			defer wg.Done()
-			rnd := rand.Intn(len(paths))
-			path := []string{paths[rnd]}
-			err := m.update(context.Background(), path, true, false)
-			if err != nil {
-				t.Errorf("update error: %s", err)
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				path := []string{paths[rnd]}
+				_ = m.update(context.Background(), path, true, true, false)
 			}
 		}()
 	}
@@ -53,6 +73,41 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 // TestManagerUpdate tests sequential configuration
 // updates.
 func TestManagerUpdate(t *testing.T) {
+	const defaultEvalInterval = time.Second * 30
+	currentEvalInterval := *evaluationInterval
+	*evaluationInterval = defaultEvalInterval
+	defer func() {
+		*evaluationInterval = currentEvalInterval
+	}()
+
+	var (
+		VMRows = &AlertingRule{
+			Name: "VMRows",
+			Expr: "vm_rows > 0",
+			For:  10 * time.Second,
+			Labels: map[string]string{
+				"label": "bar",
+				"host":  "{{ $labels.instance }}",
+			},
+			Annotations: map[string]string{
+				"summary":     "{{ $value|humanize }}",
+				"description": "{{$labels}}",
+			},
+		}
+		Conns = &AlertingRule{
+			Name: "Conns",
+			Expr: "sum(vm_tcplistener_conns) by(instance) > 1",
+			Annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}}",
+			},
+		}
+		ExampleAlertAlwaysFiring = &AlertingRule{
+			Name: "ExampleAlertAlwaysFiring",
+			Expr: "sum by(job) (up == 1)",
+		}
+	)
+
 	testCases := []struct {
 		name       string
 		initPath   string
@@ -61,64 +116,101 @@ func TestManagerUpdate(t *testing.T) {
 	}{
 		{
 			name:       "update good rules",
-			initPath:   "testdata/rules0-good.rules",
-			updatePath: "testdata/dir/rules1-good.rules",
+			initPath:   "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules1-good.rules",
 			want: []*Group{
 				{
-					File:  "testdata/dir/rules1-good.rules",
-					Name:  "duplicatedGroupDiffFiles",
-					Rules: []*Rule{newTestRule("VMRows", time.Second*10)},
+					File:     "config/testdata/dir/rules1-good.rules",
+					Name:     "duplicatedGroupDiffFiles",
+					Interval: defaultEvalInterval,
+					Rules: []Rule{
+						&AlertingRule{
+							Name:   "VMRows",
+							Expr:   "vm_rows > 0",
+							For:    5 * time.Minute,
+							Labels: map[string]string{"label": "bar"},
+							Annotations: map[string]string{
+								"summary":     "{{ $value }}",
+								"description": "{{$labels}}",
+							},
+						},
+					},
 				},
 			},
 		},
 		{
 			name:       "update good rules from 1 to 2 groups",
-			initPath:   "testdata/dir/rules1-good.rules",
-			updatePath: "testdata/rules0-good.rules",
+			initPath:   "config/testdata/dir/rules1-good.rules",
+			updatePath: "config/testdata/rules0-good.rules",
 			want: []*Group{
 				{
-					File: "testdata/rules0-good.rules",
-					Name: "groupGorSingleAlert", Rules: []*Rule{
-						newTestRule("VMRows", time.Second*10),
-					}},
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Rules:    []Rule{VMRows},
+					Interval: defaultEvalInterval,
+				},
 				{
-					File: "testdata/rules0-good.rules",
-					Name: "TestGroup", Rules: []*Rule{
-						newTestRule("Conns", time.Duration(0)),
-						newTestRule("ExampleAlertAlwaysFiring", time.Duration(0)),
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
 					}},
 			},
 		},
 		{
 			name:       "update with one bad rule file",
-			initPath:   "testdata/rules0-good.rules",
-			updatePath: "testdata/dir/rules2-bad.rules",
+			initPath:   "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules2-bad.rules",
 			want: []*Group{
 				{
-					File: "testdata/rules0-good.rules",
-					Name: "groupGorSingleAlert", Rules: []*Rule{
-						newTestRule("VMRows", time.Second*10),
-					}},
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Interval: defaultEvalInterval,
+					Rules:    []Rule{VMRows},
+				},
 				{
-					File: "testdata/rules0-good.rules",
-					Name: "TestGroup", Rules: []*Rule{
-						newTestRule("Conns", time.Duration(0)),
-						newTestRule("ExampleAlertAlwaysFiring", time.Duration(0)),
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
 					}},
 			},
 		},
+		{
+			name:       "update empty dir rules from 0 to 2 groups",
+			initPath:   "config/testdata/empty/*",
+			updatePath: "config/testdata/rules0-good.rules",
+			want: []*Group{
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Interval: defaultEvalInterval,
+					Rules:    []Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
+			},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group)}
+			m := &manager{groups: make(map[uint64]*Group), querier: &fakeQuerier{}}
 			path := []string{tc.initPath}
-			if err := m.update(ctx, path, true, false); err != nil {
+			if err := m.update(ctx, path, true, true, false); err != nil {
 				t.Fatalf("failed to complete initial rules update: %s", err)
 			}
 
 			path = []string{tc.updatePath}
-			_ = m.update(ctx, path, true, false)
+			_ = m.update(ctx, path, true, true, false)
 			if len(tc.want) != len(m.groups) {
 				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
 			}
@@ -128,7 +220,7 @@ func TestManagerUpdate(t *testing.T) {
 				if !ok {
 					t.Fatalf("expected to have group %q", wantG.Name)
 				}
-				compareGroups(t, gotG, wantG)
+				compareGroups(t, wantG, gotG)
 			}
 
 			cancel()
@@ -136,17 +228,3 @@ func TestManagerUpdate(t *testing.T) {
 		})
 	}
 }
-
-func compareGroups(t *testing.T, a, b *Group) {
-	t.Helper()
-	if len(a.Rules) != len(b.Rules) {
-		t.Fatalf("expected group %s to have %d rules; got: %d",
-			a.Name, len(a.Rules), len(b.Rules))
-	}
-	for i, r := range a.Rules {
-		got, want := r, b.Rules[i]
-		if got.Name != want.Name {
-			t.Fatalf("expected to have rule %q; got %q", want.Name, got.Name)
-		}
-	}
-}

app/vmalert/metrics.go

@@ -0,0 +1,39 @@
+package main
+
+import "github.com/VictoriaMetrics/metrics"
+
+type gauge struct {
+	name string
+	*metrics.Gauge
+}
+
+func getOrCreateGauge(name string, f func() float64) *gauge {
+	return &gauge{
+		name:  name,
+		Gauge: metrics.GetOrCreateGauge(name, f),
+	}
+}
+
+type counter struct {
+	name string
+	*metrics.Counter
+}
+
+func getOrCreateCounter(name string) *counter {
+	return &counter{
+		name:    name,
+		Counter: metrics.GetOrCreateCounter(name),
+	}
+}
+
+type summary struct {
+	name string
+	*metrics.Summary
+}
+
+func getOrCreateSummary(name string) *summary {
+	return &summary{
+		name:    name,
+		Summary: metrics.GetOrCreateSummary(name),
+	}
+}

app/vmalert/notifier/alert.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"text/template"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
 
 // Alert the triggered alert
@@ -18,6 +20,7 @@ type Alert struct {
 	Annotations map[string]string
 	State       AlertState
 
+	Expr  string
 	Start time.Time
 	End   time.Time
 	Value float64
@@ -52,14 +55,15 @@ func (as AlertState) String() string {
 type alertTplData struct {
 	Labels map[string]string
 	Value  float64
+	Expr   string
 }
 
-const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}`
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
 
 // ExecTemplate executes the Alert template for give
 // map of annotations.
 func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
-	tplData := alertTplData{Value: a.Value, Labels: a.Labels}
+	tplData := alertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
 	return templateAnnotations(annotations, tplHeader, tplData)
 }
 
@@ -75,7 +79,7 @@ func ValidateTemplates(annotations map[string]string) error {
 func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
-	eg := errGroup{}
+	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
 	for key, text := range annotations {
 		r[key] = text
@@ -85,21 +89,21 @@ func templateAnnotations(annotations map[string]string, header string, data aler
 		builder.WriteString(header)
 		builder.WriteString(text)
 		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
-			eg.errs = append(eg.errs, fmt.Sprintf("key %s, template %s:%s", key, text, err))
+			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
 		}
 		r[key] = buf.String()
 	}
-	return r, eg.err()
+	return r, eg.Err()
 }
 
 func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
 	tpl, err := template.New("").Funcs(tmplFunc).Option("missingkey=zero").Parse(text)
 	if err != nil {
-		return fmt.Errorf("error parsing annotation:%w", err)
+		return fmt.Errorf("error parsing annotation: %w", err)
 	}
 	if err = tpl.Execute(dst, data); err != nil {
-		return fmt.Errorf("error evaluating annotation template:%w", err)
+		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}
 	return nil
 }

app/vmalert/notifier/alert_test.go

@@ -1,22 +1,24 @@
 package notifier
 
 import (
-	"fmt"
 	"testing"
 )
 
 func TestAlert_ExecTemplate(t *testing.T) {
 	testCases := []struct {
+		name        string
 		alert       *Alert
 		annotations map[string]string
 		expTpl      map[string]string
 	}{
 		{
+			name:        "empty-alert",
 			alert:       &Alert{},
 			annotations: map[string]string{},
 			expTpl:      map[string]string{},
 		},
 		{
+			name: "no-template",
 			alert: &Alert{
 				Value: 1e4,
 				Labels: map[string]string{
@@ -27,6 +29,7 @@ func TestAlert_ExecTemplate(t *testing.T) {
 			expTpl:      map[string]string{},
 		},
 		{
+			name: "label-template",
 			alert: &Alert{
 				Value: 1e4,
 				Labels: map[string]string{
@@ -43,10 +46,24 @@ func TestAlert_ExecTemplate(t *testing.T) {
 				"description": "It is 10000 connections for localhost",
 			},
 		},
+		{
+			name: "expression-template",
+			alert: &Alert{
+				Expr: `vm_rows{"label"="bar"}>0`,
+			},
+			annotations: map[string]string{
+				"exprEscapedQuery": "{{ $expr|quotesEscape|queryEscape }}",
+				"exprEscapedPath":  "{{ $expr|quotesEscape|pathEscape }}",
+			},
+			expTpl: map[string]string{
+				"exprEscapedQuery": "vm_rows%7B%5C%22label%5C%22%3D%5C%22bar%5C%22%7D%3E0",
+				"exprEscapedPath":  "vm_rows%7B%5C%22label%5C%22=%5C%22bar%5C%22%7D%3E0",
+			},
+		},
 	}
 
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
 			tpl, err := tc.alert.ExecTemplate(tc.annotations)
 			if err != nil {
 				t.Fatal(err)

app/vmalert/notifier/alertmanager.go

@@ -12,9 +12,11 @@ import (
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	alertURL string
-	argFunc  AlertURLGenerator
-	client   *http.Client
+	alertURL      string
+	basicAuthUser string
+	basicAuthPass string
+	argFunc       AlertURLGenerator
+	client        *http.Client
 }
 
 // Send an alert or resolve message
@@ -26,8 +28,11 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	req = req.WithContext(ctx)
+	if am.basicAuthPass != "" {
+		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)
+	}
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -38,7 +43,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return fmt.Errorf("failed to read response from %q: %s", am.alertURL, err)
+			return fmt.Errorf("failed to read response from %q: %w", am.alertURL, err)
 		}
 		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
 	}
@@ -46,15 +51,18 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 }
 
 // AlertURLGenerator returns URL to single alert by given name
-type AlertURLGenerator func(group, alert string) string
+type AlertURLGenerator func(Alert) string
 
 const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
-func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+	addr := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
 	return &AlertManager{
-		alertURL: strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath,
-		argFunc:  fn,
-		client:   c,
+		alertURL:      addr,
+		argFunc:       fn,
+		client:        c,
+		basicAuthUser: user,
+		basicAuthPass: pass,
 	}
 }

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,15 +1,14 @@
 {% import (
-    "strconv"
     "time"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string) %}
 [
 {% for i, alert := range alerts %}
 {
-	"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
-	"generatorURL": {%q= generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)) %},
+    "startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
+    "generatorURL": {%q= generatorURL(alert) %},
     {% if !alert.End.IsZero() %}
     "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
     {% endif %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -1,131 +1,130 @@
 // Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
 // See https://github.com/valyala/quicktemplate for details.
 
-//line notifier/alertmanager_request.qtpl:1
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
 package notifier
 
-//line notifier/alertmanager_request.qtpl:1
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
 import (
-	"strconv"
 	"time"
 )
 
-//line notifier/alertmanager_request.qtpl:7
+//line app/vmalert/notifier/alertmanager_request.qtpl:6
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line notifier/alertmanager_request.qtpl:7
+//line app/vmalert/notifier/alertmanager_request.qtpl:6
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line notifier/alertmanager_request.qtpl:7
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line notifier/alertmanager_request.qtpl:7
+//line app/vmalert/notifier/alertmanager_request.qtpl:6
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:6
 	qw422016.N().S(`[`)
-//line notifier/alertmanager_request.qtpl:9
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 	for i, alert := range alerts {
-//line notifier/alertmanager_request.qtpl:9
+//line app/vmalert/notifier/alertmanager_request.qtpl:8
 		qw422016.N().S(`{"startsAt":`)
-//line notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line notifier/alertmanager_request.qtpl:11
+//line app/vmalert/notifier/alertmanager_request.qtpl:10
 		qw422016.N().S(`,"generatorURL":`)
-//line notifier/alertmanager_request.qtpl:12
-		qw422016.N().Q(generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)))
-//line notifier/alertmanager_request.qtpl:12
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		qw422016.N().Q(generatorURL(alert))
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
 		qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 		if !alert.End.IsZero() {
-//line notifier/alertmanager_request.qtpl:13
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
 			qw422016.N().S(`"endsAt":`)
-//line notifier/alertmanager_request.qtpl:14
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line notifier/alertmanager_request.qtpl:14
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
 			qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:15
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		}
-//line notifier/alertmanager_request.qtpl:15
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
 		qw422016.N().S(`"labels": {"alertname":`)
-//line notifier/alertmanager_request.qtpl:17
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
 		qw422016.N().Q(alert.Name)
-//line notifier/alertmanager_request.qtpl:18
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
 		for k, v := range alert.Labels {
-//line notifier/alertmanager_request.qtpl:18
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
 			qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
 			qw422016.N().Q(k)
-//line notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
 			qw422016.N().S(`:`)
-//line notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
 			qw422016.N().Q(v)
-//line notifier/alertmanager_request.qtpl:20
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		}
-//line notifier/alertmanager_request.qtpl:20
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		qw422016.N().S(`},"annotations": {`)
-//line notifier/alertmanager_request.qtpl:23
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 		c := len(alert.Annotations)
 
-//line notifier/alertmanager_request.qtpl:24
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
 		for k, v := range alert.Annotations {
-//line notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:24
 			c = c - 1
 
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 			qw422016.N().Q(k)
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 			qw422016.N().S(`:`)
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 			qw422016.N().Q(v)
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 			if c > 0 {
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 				qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:26
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
 			}
-//line notifier/alertmanager_request.qtpl:27
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		}
-//line notifier/alertmanager_request.qtpl:27
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		qw422016.N().S(`}}`)
-//line notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		if i != len(alerts)-1 {
-//line notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			qw422016.N().S(`,`)
-//line notifier/alertmanager_request.qtpl:30
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 		}
-//line notifier/alertmanager_request.qtpl:31
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
 	}
-//line notifier/alertmanager_request.qtpl:31
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
 	qw422016.N().S(`]`)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 }
 
-//line notifier/alertmanager_request.qtpl:33
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	streamamRequest(qw422016, alerts, generatorURL)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	qt422016.ReleaseWriter(qw422016)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 }
 
-//line notifier/alertmanager_request.qtpl:33
-func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
+func amRequest(alerts []Alert, generatorURL func(Alert) string) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	qb422016 := qt422016.AcquireByteBuffer()
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	writeamRequest(qb422016, alerts, generatorURL)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	qs422016 := string(qb422016.B)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	qt422016.ReleaseByteBuffer(qb422016)
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 	return qs422016
-//line notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:32
 }

app/vmalert/notifier/alertmanager_test.go

@@ -5,17 +5,27 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 	"time"
 )
 
 func TestAlertManager_Send(t *testing.T) {
+	const baUser, baPass = "foo", "bar"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
 		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
+		user, pass, ok := r.BasicAuth()
+		if !ok {
+			t.Errorf("unauthorized request")
+		}
+		if user != baUser || pass != baPass {
+			t.Errorf("wrong creds %q:%q; expected %q:%q",
+				user, pass, baUser, baPass)
+		}
 		c++
 		if r.Method != http.MethodPost {
 			t.Errorf("expected POST method got %s", r.Method)
@@ -42,23 +52,23 @@ func TestAlertManager_Send(t *testing.T) {
 				t.Errorf("expected 1 alert in array got %d", len(a))
 			}
 			if a[0].GeneratorURL != "0/0" {
-				t.Errorf("exptected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
 			if a[0].Labels["alertname"] != "alert0" {
-				t.Errorf("exptected alert0 as alert name got %s", a[0].Labels["alertname"])
+				t.Errorf("expected alert0 as alert name got %s", a[0].Labels["alertname"])
 			}
 			if a[0].StartsAt.IsZero() {
-				t.Errorf("exptected non-zero start time")
+				t.Errorf("expected non-zero start time")
 			}
 			if a[0].EndAt.IsZero() {
-				t.Errorf("exptected non-zero end time")
+				t.Errorf("expected non-zero end time")
 			}
 		}
 	})
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewAlertManager(srv.URL, func(group, name string) string {
-		return group + "/" + name
+	am := NewAlertManager(srv.URL, baUser, baPass, func(alert Alert) string {
+		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
 	}, srv.Client())
 	if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil {
 		t.Error("expected connection error got nil")

app/vmalert/notifier/init.go

@@ -0,0 +1,46 @@
+package notifier
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -datasource.url")
+	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -datasource.url")
+
+	tlsInsecureSkipVerify = flag.Bool("notifier.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -notifier.url")
+	tlsCertFile           = flagutil.NewArray("notifier.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
+	tlsKeyFile            = flagutil.NewArray("notifier.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
+	tlsCAFile             = flagutil.NewArray("notifier.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArray("notifier.tlsServerName", "Optional TLS server name to use for connections to -notifier.url. "+
+		"By default the server name from -notifier.url is used")
+)
+
+// Init creates a Notifier object based on provided flags.
+func Init(gen AlertURLGenerator) ([]Notifier, error) {
+	if len(*addrs) == 0 {
+		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
+	}
+
+	var notifiers []Notifier
+	for i, addr := range *addrs {
+		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)
+		ca, serverName := tlsCAFile.GetOptionalArg(i), tlsServerName.GetOptionalArg(i)
+		tr, err := utils.Transport(addr, cert, key, ca, serverName, *tlsInsecureSkipVerify)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create transport: %w", err)
+		}
+		user, pass := basicAuthUsername.GetOptionalArg(i), basicAuthPassword.GetOptionalArg(i)
+		am := NewAlertManager(addr, user, pass, gen, &http.Client{Transport: tr})
+		notifiers = append(notifiers, am)
+	}
+
+	return notifiers, nil
+}

app/vmalert/notifier/package_test.go

@@ -0,0 +1,13 @@
+package notifier
+
+import (
+	"net/url"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	InitTemplateFunc(u)
+	os.Exit(m.Run())
+}

app/vmalert/notifier/template_func.go

@@ -142,6 +142,15 @@ func InitTemplateFunc(externalURL *url.URL) {
 		"externalURL": func() string {
 			return externalURL.String()
 		},
+		"pathEscape": func(u string) string {
+			return url.PathEscape(u)
+		},
+		"queryEscape": func(q string) string {
+			return url.QueryEscape(q)
+		},
+		"quotesEscape": func(q string) string {
+			return strings.Replace(q, `"`, `\"`, -1)
+		},
 	}
 }
 

app/vmalert/notifier/utils.go

@@ -1,21 +0,0 @@
-package notifier
-
-import (
-	"fmt"
-	"strings"
-)
-
-type errGroup struct {
-	errs []string
-}
-
-func (eg *errGroup) err() error {
-	if eg == nil || len(eg.errs) == 0 {
-		return nil
-	}
-	return eg
-}
-
-func (eg *errGroup) Error() string {
-	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
-}

app/vmalert/recording.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// RecordingRule is a Rule that supposed
+// to evaluate configured Expression and
+// return TimeSeries as result.
+type RecordingRule struct {
+	RuleID  uint64
+	Name    string
+	Expr    string
+	Labels  map[string]string
+	GroupID uint64
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+
+	metrics *recordingRuleMetrics
+}
+
+type recordingRuleMetrics struct {
+	errors *gauge
+}
+
+// String implements Stringer interface
+func (rr *RecordingRule) String() string {
+	return rr.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (rr *RecordingRule) ID() uint64 {
+	return rr.RuleID
+}
+
+func newRecordingRule(group *Group, cfg config.Rule) *RecordingRule {
+	rr := &RecordingRule{
+		RuleID:  cfg.ID,
+		Name:    cfg.Record,
+		Expr:    cfg.Expr,
+		Labels:  cfg.Labels,
+		GroupID: group.ID(),
+		metrics: &recordingRuleMetrics{},
+	}
+	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
+	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
+		func() float64 {
+			rr.mu.Lock()
+			defer rr.mu.Unlock()
+			if rr.lastExecError == nil {
+				return 0
+			}
+			return 1
+		})
+	return rr
+}
+
+// Close unregisters rule metrics
+func (rr *RecordingRule) Close() {
+	metrics.UnregisterMetric(rr.metrics.errors.name)
+}
+
+// Exec executes RecordingRule expression via the given Querier.
+func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
+	if !series {
+		return nil, nil
+	}
+
+	qMetrics, err := q.Query(ctx, rr.Expr)
+
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+
+	rr.lastExecTime = time.Now()
+	rr.lastExecError = err
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
+	}
+
+	duplicates := make(map[uint64]prompbmarshal.TimeSeries, len(qMetrics))
+	var tss []prompbmarshal.TimeSeries
+	for _, r := range qMetrics {
+		ts := rr.toTimeSeries(r, rr.lastExecTime)
+		h := hashTimeSeries(ts)
+		if _, ok := duplicates[h]; ok {
+			rr.lastExecError = errDuplicate
+			return nil, errDuplicate
+		}
+		duplicates[h] = ts
+		tss = append(tss, ts)
+	}
+	return tss, nil
+}
+
+func hashTimeSeries(ts prompbmarshal.TimeSeries) uint64 {
+	hash := fnv.New64a()
+	labels := ts.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (rr *RecordingRule) toTimeSeries(m datasource.Metric, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for _, l := range m.Labels {
+		labels[l.Name] = l.Value
+	}
+	labels["__name__"] = rr.Name
+	// override existing labels with configured ones
+	for k, v := range rr.Labels {
+		labels[k] = v
+	}
+	return newTimeSeries(m.Value, labels, timestamp)
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (rr *RecordingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*RecordingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update recroding rule with wrong type %#v", r)
+	}
+	rr.Expr = nr.Expr
+	rr.Labels = nr.Labels
+	return nil
+}
+
+// RuleAPI returns Rule representation in form
+// of APIRecordingRule
+func (rr *RecordingRule) RuleAPI() APIRecordingRule {
+	var lastErr string
+	if rr.lastExecError != nil {
+		lastErr = rr.lastExecError.Error()
+	}
+	return APIRecordingRule{
+		// encode as strings to avoid rounding
+		ID:         fmt.Sprintf("%d", rr.ID()),
+		GroupID:    fmt.Sprintf("%d", rr.GroupID),
+		Name:       rr.Name,
+		Expression: rr.Expr,
+		LastError:  lastErr,
+		LastExec:   rr.lastExecTime,
+		Labels:     rr.Labels,
+	}
+}

app/vmalert/recording_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestRecoridngRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValueAndLabels(t, 10,
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(10, map[string]string{
+					"__name__": "foo",
+				}, timestamp),
+			},
+		},
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}, timestamp),
+				newTimeSeries(2, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}, timestamp),
+				newTimeSeries(3, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "baz",
+				}, timestamp),
+			},
+		},
+		{
+			&RecordingRule{Name: "job:foo", Labels: map[string]string{
+				"source": "test",
+			}},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(2, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}, timestamp),
+				newTimeSeries(1, map[string]string{
+					"__name__": "job:foo",
+					"job":      "bar",
+					"source":   "test",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			fq.add(tc.metrics...)
+			tss, err := tc.rule.Exec(context.TODO(), fq, true)
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestRecoridngRule_ToTimeSeriesNegative(t *testing.T) {
+	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
+		"job": "test",
+	}}
+
+	fq := &fakeQuerier{}
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+
+	_, err := rr.Exec(context.TODO(), fq, true)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+
+	fq.reset()
+
+	// add metrics which differs only by `job` label
+	// which will be overridden by rule
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
+	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
+
+	_, err = rr.Exec(context.TODO(), fq, true)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), errDuplicate.Error()) {
+		t.Fatalf("expected to get err %q; got %q insterad", errDuplicate, err)
+	}
+}

app/vmalert/remoteread/init.go

@@ -0,0 +1,39 @@
+package remoteread
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
+		" state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
+	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
+	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
+	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
+	tlsCAFile             = flag.String("remoteRead.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteRead.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
+		"By default the server name from -remoteRead.url is used")
+)
+
+// Init creates a Querier from provided flag values.
+// Returns nil if addr flag wasn't set.
+func Init() (datasource.Querier, error) {
+	if *addr == "" {
+		return nil, nil
+	}
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+	c := &http.Client{Transport: tr}
+	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, c), nil
+}

app/vmalert/remotewrite/init.go

@@ -0,0 +1,54 @@
+package remotewrite
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
+		" and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+
+	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
+	concurrency   = flag.Int("remoteWrite.concurrency", 1, "Defines number of writers for concurrent writing into remote querier")
+	flushInterval = flag.Duration("remoteWrite.flushInterval", 5*time.Second, "Defines interval of flushes to remote write endpoint")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
+	tlsKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
+	tlsCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
+)
+
+// Init creates Client object from given flags.
+// Returns nil if addr flag wasn't set.
+func Init(ctx context.Context) (*Client, error) {
+	if *addr == "" {
+		return nil, nil
+	}
+
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+
+	return NewClient(ctx, Config{
+		Addr:          *addr,
+		Concurrency:   *concurrency,
+		MaxQueueSize:  *maxQueueSize,
+		MaxBatchSize:  *maxBatchSize,
+		FlushInterval: *flushInterval,
+		BasicAuthUser: *basicAuthUsername,
+		BasicAuthPass: *basicAuthPassword,
+		Transport:     t,
+	})
+}

app/vmalert/remotewrite/remotewrite.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
 
@@ -38,22 +39,29 @@ type Config struct {
 	BasicAuthUser string
 	BasicAuthPass string
 
+	// Concurrency defines number of readers that
+	// concurrently read from the queue and flush data
+	Concurrency int
 	// MaxBatchSize defines max number of timeseries
 	// to be flushed at once
 	MaxBatchSize int
 	// MaxQueueSize defines max length of input queue
-	// populated by Push method
+	// populated by Push method.
+	// Push will be rejected once queue is full.
 	MaxQueueSize int
 	// FlushInterval defines time interval for flushing batches
 	FlushInterval time.Duration
 	// WriteTimeout defines timeout for HTTP write request
 	// to remote storage
 	WriteTimeout time.Duration
+	// Transport will be used by the underlying http.Client
+	Transport *http.Transport
 }
 
 const (
+	defaultConcurrency   = 4
 	defaultMaxBatchSize  = 1e3
-	defaultMaxQueueSize  = 100
+	defaultMaxQueueSize  = 1e5
 	defaultFlushInterval = 5 * time.Second
 	defaultWriteTimeout  = 30 * time.Second
 )
@@ -78,19 +86,30 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.WriteTimeout == 0 {
 		cfg.WriteTimeout = defaultWriteTimeout
 	}
+	if cfg.Transport == nil {
+		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
+	}
 	c := &Client{
 		c: &http.Client{
-			Timeout: cfg.WriteTimeout,
+			Timeout:   cfg.WriteTimeout,
+			Transport: cfg.Transport,
 		},
 		addr:          strings.TrimSuffix(cfg.Addr, "/") + writePath,
 		baUser:        cfg.BasicAuthUser,
 		baPass:        cfg.BasicAuthPass,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,
+		maxQueueSize:  cfg.MaxQueueSize,
 		doneCh:        make(chan struct{}),
 		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 	}
-	c.run(ctx)
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
+	for i := 0; i < cc; i++ {
+		c.run(ctx)
+	}
 	return c, nil
 }
 
@@ -103,7 +122,8 @@ func (c *Client) Push(s prompbmarshal.TimeSeries) error {
 	case c.input <- s:
 		return nil
 	default:
-		return fmt.Errorf("failed to push timeseries - queue is full (%d entries)",
+		return fmt.Errorf("failed to push timeseries - queue is full (%d entries). "+
+			"Queue size is controlled by -remoteWrite.maxQueueSize flag",
 			c.maxQueueSize)
 	}
 }
@@ -122,12 +142,12 @@ func (c *Client) Close() error {
 
 func (c *Client) run(ctx context.Context) {
 	ticker := time.NewTicker(c.flushInterval)
-	wr := prompbmarshal.WriteRequest{}
+	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
 		for ts := range c.input {
 			wr.Timeseries = append(wr.Timeseries, ts)
 		}
-		lastCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
 		c.flush(lastCtx, wr)
 		cancel()
 	}
@@ -145,44 +165,82 @@ func (c *Client) run(ctx context.Context) {
 				return
 			case <-ticker.C:
 				c.flush(ctx, wr)
-				wr = prompbmarshal.WriteRequest{}
-			case ts := <-c.input:
+			case ts, ok := <-c.input:
+				if !ok {
+					continue
+				}
 				wr.Timeseries = append(wr.Timeseries, ts)
 				if len(wr.Timeseries) >= c.maxBatchSize {
 					c.flush(ctx, wr)
-					wr = prompbmarshal.WriteRequest{}
 				}
 			}
 		}
 	}()
 }
 
-func (c *Client) flush(ctx context.Context, wr prompbmarshal.WriteRequest) {
+var (
+	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+)
+
+// flush is a blocking function that marshals WriteRequest and sends
+// it to remote write endpoint. Flush performs limited amount of retries
+// if request fails.
+func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 	if len(wr.Timeseries) < 1 {
 		return
 	}
+	defer prompbmarshal.ResetWriteRequest(wr)
+
 	data, err := wr.Marshal()
 	if err != nil {
 		logger.Errorf("failed to marshal WriteRequest: %s", err)
 		return
 	}
-	req, err := http.NewRequest("POST", c.addr, bytes.NewReader(snappy.Encode(nil, data)))
+
+	const attempts = 5
+	b := snappy.Encode(nil, data)
+	for i := 0; i < attempts; i++ {
+		err := c.send(ctx, b)
+		if err == nil {
+			sentRows.Add(len(wr.Timeseries))
+			sentBytes.Add(len(b))
+			return
+		}
+
+		logger.Errorf("attempt %d to send request failed: %s", i+1, err)
+		// sleeping to avoid remote db hammering
+		time.Sleep(time.Second)
+		continue
+	}
+
+	droppedRows.Add(len(wr.Timeseries))
+	droppedBytes.Add(len(b))
+	logger.Errorf("all %d attempts to send request failed - dropping %d timeseries",
+		attempts, len(wr.Timeseries))
+}
+
+func (c *Client) send(ctx context.Context, data []byte) error {
+	r := bytes.NewReader(data)
+	req, err := http.NewRequest("POST", c.addr, r)
 	if err != nil {
-		logger.Errorf("failed to create new HTTP request: %s", err)
-		return
+		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 	if c.baPass != "" {
 		req.SetBasicAuth(c.baUser, c.baPass)
 	}
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
-		logger.Errorf("error getting response from %s:%s", req.URL, err)
-		return
+		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
+			req.URL, err, len(data), r.Size())
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusNoContent {
 		body, _ := ioutil.ReadAll(resp.Body)
-		logger.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
-		return
+		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
+			resp.StatusCode, req.URL, body)
 	}
+	return nil
 }

app/vmalert/remotewrite/remotewrite_test.go

@@ -0,0 +1,102 @@
+package remotewrite
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestClient_Push(t *testing.T) {
+	testSrv := newRWServer()
+	cfg := Config{
+		Addr:         testSrv.URL,
+		MaxBatchSize: 100,
+	}
+	client, err := NewClient(context.Background(), cfg)
+	if err != nil {
+		t.Fatalf("failed to create client: %s", err)
+	}
+	const rowsN = 1e4
+	var sent int
+	for i := 0; i < rowsN; i++ {
+		s := prompbmarshal.TimeSeries{
+			Samples: []prompbmarshal.Sample{{
+				Value:     rand.Float64(),
+				Timestamp: time.Now().Unix(),
+			}},
+		}
+		err := client.Push(s)
+		if err == nil {
+			sent++
+		}
+	}
+	if sent == 0 {
+		t.Fatalf("0 series sent")
+	}
+	if err := client.Close(); err != nil {
+		t.Fatalf("failed to close client: %s", err)
+	}
+	got := testSrv.accepted()
+	if got != sent {
+		t.Fatalf("expected to have %d series; got %d", sent, got)
+	}
+}
+
+func newRWServer() *rwServer {
+	rw := &rwServer{}
+	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
+	return rw
+}
+
+type rwServer struct {
+	// WARN: ordering of fields is important for alignment!
+	// see https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	acceptedRows uint64
+	*httptest.Server
+}
+
+func (rw *rwServer) accepted() int {
+	return int(atomic.LoadUint64(&rw.acceptedRows))
+}
+
+func (rw *rwServer) err(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusBadRequest)
+	w.Write([]byte(err.Error()))
+}
+
+func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		rw.err(w, fmt.Errorf("bad method %q", r.Method))
+		return
+	}
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		rw.err(w, fmt.Errorf("body read err: %w", err))
+		return
+	}
+	defer func() { _ = r.Body.Close() }()
+
+	b, err := snappy.Decode(nil, data)
+	if err != nil {
+		rw.err(w, fmt.Errorf("decode err: %w", err))
+		return
+	}
+	wr := &prompb.WriteRequest{}
+	if err := wr.Unmarshal(b); err != nil {
+		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))
+		return
+	}
+	atomic.AddUint64(&rw.acceptedRows, uint64(len(wr.Timeseries)))
+	w.WriteHeader(http.StatusNoContent)
+}

app/vmalert/rule.go

@@ -3,333 +3,28 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
-	"hash/fnv"
-	"sort"
-	"strconv"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
-// Rule is basic alert entity
-type Rule struct {
-	Name        string            `yaml:"alert"`
-	Expr        string            `yaml:"expr"`
-	For         time.Duration     `yaml:"for"`
-	Labels      map[string]string `yaml:"labels"`
-	Annotations map[string]string `yaml:"annotations"`
-
-	group Group
-
-	// guard status fields
-	mu sync.RWMutex
-	// stores list of active alerts
-	alerts map[uint64]*notifier.Alert
-	// stores last moment of time Exec was called
-	lastExecTime time.Time
-	// stores last error that happened in Exec func
-	// resets on every successful Exec
-	// may be used as Health state
-	lastExecError error
+// Rule represents alerting or recording rule
+// that has unique ID, can be Executed and
+// updated with other Rule.
+type Rule interface {
+	// Returns unique ID that may be used for
+	// identifying this Rule among others.
+	ID() uint64
+	// Exec executes the rule with given context
+	// and Querier. If returnSeries is true, Exec
+	// may return TimeSeries as result of execution
+	Exec(ctx context.Context, q datasource.Querier, returnSeries bool) ([]prompbmarshal.TimeSeries, error)
+	// UpdateWith performs modification of current Rule
+	// with fields of the given Rule.
+	UpdateWith(Rule) error
+	// Close performs the shutdown procedures for rule
+	// such as metrics unregister
+	Close()
 }
 
-func (r *Rule) id() string {
-	return r.Name
-}
-
-// Validate validates rule
-func (r *Rule) Validate() error {
-	if r.Name == "" {
-		return errors.New("rule name can not be empty")
-	}
-	if r.Expr == "" {
-		return fmt.Errorf("expression for rule %q can't be empty", r.Name)
-	}
-	if _, err := metricsql.Parse(r.Expr); err != nil {
-		return fmt.Errorf("invalid expression for rule %q: %w", r.Name, err)
-	}
-	return nil
-}
-
-// Exec executes Rule expression via the given Querier.
-// Based on the Querier results Rule maintains notifier.Alerts
-func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
-	qMetrics, err := q.Query(ctx, r.Expr)
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	r.lastExecError = err
-	r.lastExecTime = time.Now()
-	if err != nil {
-		return fmt.Errorf("failed to execute query %q: %s", r.Expr, err)
-	}
-
-	for h, a := range r.alerts {
-		// cleanup inactive alerts from previous Eval
-		if a.State == notifier.StateInactive {
-			delete(r.alerts, h)
-		}
-	}
-
-	updated := make(map[uint64]struct{})
-	// update list of active alerts
-	for _, m := range qMetrics {
-		h := hash(m)
-		updated[h] = struct{}{}
-		if a, ok := r.alerts[h]; ok {
-			if a.Value != m.Value {
-				// update Value field with latest value
-				a.Value = m.Value
-				// and re-exec template since Value can be used
-				// in templates
-				err = r.template(a)
-				if err != nil {
-					return err
-				}
-			}
-			continue
-		}
-		a, err := r.newAlert(m)
-		if err != nil {
-			r.lastExecError = err
-			return fmt.Errorf("failed to create alert: %s", err)
-		}
-		a.ID = h
-		a.State = notifier.StatePending
-		r.alerts[h] = a
-	}
-
-	for h, a := range r.alerts {
-		// if alert wasn't updated in this iteration
-		// means it is resolved already
-		if _, ok := updated[h]; !ok {
-			a.State = notifier.StateInactive
-			// set endTime to last execution time
-			// so it can be sent by notifier on next step
-			a.End = r.lastExecTime
-			continue
-		}
-		if a.State == notifier.StatePending && time.Since(a.Start) >= r.For {
-			a.State = notifier.StateFiring
-			alertsFired.Inc()
-		}
-		if a.State == notifier.StateFiring {
-			a.End = r.lastExecTime.Add(3 * *evaluationInterval)
-		}
-	}
-	return nil
-}
-
-// TODO: consider hashing algorithm in VM
-func hash(m datasource.Metric) uint64 {
-	hash := fnv.New64a()
-	labels := m.Labels
-	sort.Slice(labels, func(i, j int) bool {
-		return labels[i].Name < labels[j].Name
-	})
-	for _, l := range labels {
-		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
-			continue
-		}
-		hash.Write([]byte(l.Name))
-		hash.Write([]byte(l.Value))
-		hash.Write([]byte("\xff"))
-	}
-	return hash.Sum64()
-}
-
-func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
-	a := &notifier.Alert{
-		GroupID: r.group.ID(),
-		Name:    r.Name,
-		Labels:  map[string]string{},
-		Value:   m.Value,
-		Start:   time.Now(),
-		// TODO: support End time
-	}
-	for _, l := range m.Labels {
-		// drop __name__ to be consistent with Prometheus alerting
-		if l.Name == "__name__" {
-			continue
-		}
-		a.Labels[l.Name] = l.Value
-	}
-	return a, r.template(a)
-}
-
-func (r *Rule) template(a *notifier.Alert) error {
-	// 1. template rule labels with data labels
-	rLabels, err := a.ExecTemplate(r.Labels)
-	if err != nil {
-		return err
-	}
-
-	// 2. merge data labels and rule labels
-	// metric labels may be overridden by
-	// rule labels
-	for k, v := range rLabels {
-		a.Labels[k] = v
-	}
-
-	// 3. template merged labels
-	a.Labels, err = a.ExecTemplate(a.Labels)
-	if err != nil {
-		return err
-	}
-
-	a.Annotations, err = a.ExecTemplate(r.Annotations)
-	return err
-}
-
-// AlertAPI generates APIAlert object from alert by its id(hash)
-func (r *Rule) AlertAPI(id uint64) *APIAlert {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	a, ok := r.alerts[id]
-	if !ok {
-		return nil
-	}
-	return r.newAlertAPI(*a)
-}
-
-// AlertsAPI generates list of APIAlert objects from existing alerts
-func (r *Rule) AlertsAPI() []*APIAlert {
-	var alerts []*APIAlert
-	r.mu.RLock()
-	for _, a := range r.alerts {
-		alerts = append(alerts, r.newAlertAPI(*a))
-	}
-	r.mu.RUnlock()
-	return alerts
-}
-
-func (r *Rule) newAlertAPI(a notifier.Alert) *APIAlert {
-	return &APIAlert{
-		// encode as strings to avoid rounding
-		ID:      fmt.Sprintf("%d", a.ID),
-		GroupID: fmt.Sprintf("%d", a.GroupID),
-
-		Name:        a.Name,
-		Expression:  r.Expr,
-		Labels:      a.Labels,
-		Annotations: a.Annotations,
-		State:       a.State.String(),
-		ActiveAt:    a.Start,
-		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
-	}
-}
-
-const (
-	// AlertMetricName is the metric name for synthetic alert timeseries.
-	alertMetricName = "ALERTS"
-	// AlertForStateMetricName is the metric name for 'for' state of alert.
-	alertForStateMetricName = "ALERTS_FOR_STATE"
-
-	// AlertNameLabel is the label name indicating the name of an alert.
-	alertNameLabel = "alertname"
-	// AlertStateLabel is the label name indicating the state of an alert.
-	alertStateLabel = "alertstate"
-)
-
-// AlertToTimeSeries converts the given alert with the given timestamp to timeseries
-func (r *Rule) AlertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
-	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(r.Name, a, timestamp))
-	if r.For > 0 {
-		tss = append(tss, alertForToTimeSeries(r.Name, a, timestamp))
-	}
-	return tss
-}
-
-func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
-	for k, v := range a.Labels {
-		labels[k] = v
-	}
-	labels["__name__"] = alertMetricName
-	labels[alertNameLabel] = name
-	labels[alertStateLabel] = a.State.String()
-	return newTimeSeries(1, labels, timestamp)
-}
-
-// alertForToTimeSeries returns a timeseries that represents
-// state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
-	for k, v := range a.Labels {
-		labels[k] = v
-	}
-	labels["__name__"] = alertForStateMetricName
-	labels[alertNameLabel] = name
-	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
-}
-
-func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
-	ts := prompbmarshal.TimeSeries{}
-	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
-		Value:     value,
-		Timestamp: timestamp.UnixNano() / 1e6,
-	})
-	keys := make([]string, 0, len(labels))
-	for k := range labels {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	for _, key := range keys {
-		ts.Labels = append(ts.Labels, prompbmarshal.Label{
-			Name:  key,
-			Value: labels[key],
-		})
-	}
-	return ts
-}
-
-// Restore restores the state of active alerts basing on previously written timeseries.
-// Restore restores only Start field. Field State will be always Pending and supposed
-// to be updated on next Eval, as well as Value field.
-func (r *Rule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
-	// Get the last datapoint in range via MetricsQL `last_over_time`.
-	// We don't use plain PromQL since Prometheus doesn't support
-	// remote write protocol which is used for state persistence in vmalert.
-	expr := fmt.Sprintf("last_over_time(%s{alertname=%q}[%ds])",
-		alertForStateMetricName, r.Name, int(lookback.Seconds()))
-	qMetrics, err := q.Query(ctx, expr)
-	if err != nil {
-		return err
-	}
-
-	for _, m := range qMetrics {
-		labels := m.Labels
-		m.Labels = make([]datasource.Label, 0)
-		// drop all extra labels, so hash key will
-		// be identical to timeseries received in Eval
-		for _, l := range labels {
-			if l.Name == alertNameLabel {
-				continue
-			}
-			// drop all overridden labels
-			if _, ok := r.Labels[l.Name]; ok {
-				continue
-			}
-			m.Labels = append(m.Labels, l)
-		}
-
-		a, err := r.newAlert(m)
-		if err != nil {
-			return fmt.Errorf("failed to create alert: %s", err)
-		}
-		a.ID = hash(m)
-		a.State = notifier.StatePending
-		a.Start = time.Unix(int64(m.Value), 0)
-		r.alerts[a.ID] = a
-		logger.Infof("alert %q(%d) restored to state at %v", a.Name, a.ID, a.Start)
-	}
-	return nil
-}
+var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")

app/vmalert/utils.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"sort"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{}
+	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
+		Value:     value,
+		Timestamp: timestamp.UnixNano() / 1e6,
+	})
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}

app/vmalert/utils/err_group.go

@@ -0,0 +1,43 @@
+package utils
+
+import (
+	"fmt"
+	"strings"
+)
+
+// ErrGroup accumulates multiple errors
+// and produces single error message.
+type ErrGroup struct {
+	errs []error
+}
+
+// Add adds a new error to group.
+// Isn't thread-safe.
+func (eg *ErrGroup) Add(err error) {
+	eg.errs = append(eg.errs, err)
+}
+
+// Err checks if group contains at least
+// one error.
+func (eg *ErrGroup) Err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+// Error satisfies Error interface
+func (eg *ErrGroup) Error() string {
+	if len(eg.errs) == 0 {
+		return ""
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "errors(%d): ", len(eg.errs))
+	for i, err := range eg.errs {
+		b.WriteString(err.Error())
+		if i != len(eg.errs)-1 {
+			b.WriteString("\n")
+		}
+	}
+	return b.String()
+}

app/vmalert/utils/err_group_test.go

@@ -0,0 +1,38 @@
+package utils
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestErrGroup(t *testing.T) {
+	testCases := []struct {
+		errs []error
+		exp  string
+	}{
+		{nil, ""},
+		{[]error{errors.New("timeout")}, "errors(1): timeout"},
+		{
+			[]error{errors.New("timeout"), errors.New("deadline")},
+			"errors(2): timeout\ndeadline",
+		},
+	}
+	for _, tc := range testCases {
+		eg := new(ErrGroup)
+		for _, err := range tc.errs {
+			eg.Add(err)
+		}
+		if len(tc.errs) == 0 {
+			if eg.Err() != nil {
+				t.Fatalf("expected to get nil error")
+			}
+			continue
+		}
+		if eg.Err() == nil {
+			t.Fatalf("expected to get non-nil error")
+		}
+		if eg.Error() != tc.exp {
+			t.Fatalf("expected to have: \n%q\ngot:\n%q", tc.exp, eg.Error())
+		}
+	}
+}

app/vmalert/utils/tls.go

@@ -0,0 +1,58 @@
+package utils
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// Transport creates http.Transport object based on provided URL.
+// Returns Transport with TLS configuration if URL contains `https` prefix
+func Transport(URL, certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*http.Transport, error) {
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	if !strings.HasPrefix(URL, "https") {
+		return t, nil
+	}
+	tlsCfg, err := TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		return nil, err
+	}
+	t.TLSClientConfig = tlsCfg
+	return t, nil
+}
+
+// TLSConfig creates tls.Config object from provided arguments
+func TLSConfig(certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*tls.Config, error) {
+	var certs []tls.Certificate
+	if certFile != "" {
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", certFile, keyFile, err)
+		}
+
+		certs = []tls.Certificate{cert}
+	}
+
+	var rootCAs *x509.CertPool
+	if CAFile != "" {
+		pem, err := ioutil.ReadFile(CAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read `ca_file` %q: %w", CAFile, err)
+		}
+
+		rootCAs = x509.NewCertPool()
+		if !rootCAs.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("cannot parse data from `ca_file` %q", CAFile)
+		}
+	}
+
+	return &tls.Config{
+		Certificates:       certs,
+		InsecureSkipVerify: insecureSkipVerify,
+		RootCAs:            rootCAs,
+		ServerName:         serverName,
+	}, nil
+}

app/vmalert/utils/tls_test.go

@@ -0,0 +1,52 @@
+package utils
+
+import "testing"
+
+func TestTLSConfig(t *testing.T) {
+	var certFile, keyFile, CAFile, serverName string
+	var insecureSkipVerify bool
+	serverName = "test"
+	insecureSkipVerify = true
+	tlsCfg, err := TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if tlsCfg == nil {
+		t.Errorf("expected tlsConfig to be set, got nil")
+	}
+	if tlsCfg.ServerName != serverName {
+		t.Errorf("unexpected ServerName, want %s, got %s", serverName, tlsCfg.ServerName)
+	}
+	if tlsCfg.InsecureSkipVerify != insecureSkipVerify {
+		t.Errorf("unexpected InsecureSkipVerify, want %v, got %v", insecureSkipVerify, tlsCfg.InsecureSkipVerify)
+	}
+	certFile = "/path/to/nonexisting/cert/file"
+	_, err = TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err == nil {
+		t.Errorf("expected keypair error, got nil")
+	}
+	certFile = ""
+	CAFile = "/path/to/nonexisting/cert/file"
+	_, err = TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err == nil {
+		t.Errorf("expected read error, got nil")
+	}
+}
+
+func TestTransport(t *testing.T) {
+	var certFile, keyFile, CAFile, serverName string
+	var insecureSkipVerify bool
+	URL := "http://victoriametrics.com"
+	_, err := Transport(URL, certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	URL = "https://victoriametrics.com"
+	tr, err := Transport(URL, certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if tr.TLSClientConfig == nil {
+		t.Errorf("expected TLSClientConfig to be set, got nil")
+	}
+}

app/vmalert/web.go

@@ -7,32 +7,18 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
-// APIAlert represents an notifier.Alert state
-// for WEB view
-type APIAlert struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	GroupID     string            `json:"group_id"`
-	Expression  string            `json:"expression"`
-	State       string            `json:"state"`
-	Value       string            `json:"value"`
-	Labels      map[string]string `json:"labels"`
-	Annotations map[string]string `json:"annotations"`
-	ActiveAt    time.Time         `json:"activeAt"`
-}
-
 type requestHandler struct {
 	m *manager
 }
 
 var pathList = [][]string{
+	{"/api/v1/groups", "list all loaded groups and rules"},
 	{"/api/v1/alerts", "list all active alerts"},
 	{"/api/v1/groupID/alertID/status", "get alert status by ID"},
 	// /metrics is served by httpserver by default
@@ -41,7 +27,6 @@ var pathList = [][]string{
 }
 
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
-	resph := responseHandler{w}
 	switch r.URL.Path {
 	case "/":
 		for _, path := range pathList {
@@ -49,8 +34,23 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
 		}
 		return true
+	case "/api/v1/groups":
+		data, err := rh.listGroups()
+		if err != nil {
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Write(data)
+		return true
 	case "/api/v1/alerts":
-		resph.handle(rh.list())
+		data, err := rh.listAlerts()
+		if err != nil {
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Write(data)
 		return true
 	case "/-/reload":
 		logger.Infof("api config reload was called, sending sighup")
@@ -58,15 +58,52 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	default:
+		if !strings.HasSuffix(r.URL.Path, "/status") {
+			return false
+		}
 		// /api/v1/<groupName>/<alertID>/status
-		if strings.HasSuffix(r.URL.Path, "/status") {
-			resph.handle(rh.alert(r.URL.Path))
+		data, err := rh.alert(r.URL.Path)
+		if err != nil {
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
-		return false
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Write(data)
+		return true
 	}
 }
 
+type listGroupsResponse struct {
+	Data struct {
+		Groups []APIGroup `json:"groups"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	lr := listGroupsResponse{Status: "success"}
+	for _, g := range rh.m.groups {
+		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+	}
+
+	// sort list of alerts for deterministic output
+	sort.Slice(lr.Data.Groups, func(i, j int) bool {
+		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	})
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of active alerts: %w`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
 type listAlertsResponse struct {
 	Data struct {
 		Alerts []*APIAlert `json:"alerts"`
@@ -74,13 +111,18 @@ type listAlertsResponse struct {
 	Status string `json:"status"`
 }
 
-func (rh *requestHandler) list() ([]byte, error) {
+func (rh *requestHandler) listAlerts() ([]byte, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
+
 	lr := listAlertsResponse{Status: "success"}
 	for _, g := range rh.m.groups {
 		for _, r := range g.Rules {
-			lr.Data.Alerts = append(lr.Data.Alerts, r.AlertsAPI()...)
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
 		}
 	}
 
@@ -92,7 +134,7 @@ func (rh *requestHandler) list() ([]byte, error) {
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`error encoding list of active alerts: %s`, err),
+			Err:        fmt.Errorf(`error encoding list of active alerts: %w`, err),
 			StatusCode: http.StatusInternalServerError,
 		}
 	}
@@ -113,11 +155,11 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 
 	groupID, err := uint64FromPath(parts[0])
 	if err != nil {
-		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %s`, err))
+		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
 	}
 	alertID, err := uint64FromPath(parts[1])
 	if err != nil {
-		return nil, badRequest(fmt.Errorf(`cannot parse alertID: %s`, err))
+		return nil, badRequest(fmt.Errorf(`cannot parse alertID: %w`, err))
 	}
 	resp, err := rh.m.AlertAPI(groupID, alertID)
 	if err != nil {
@@ -126,18 +168,6 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 	return json.Marshal(resp)
 }
 
-// responseHandler wrapper on http.ResponseWriter with sugar
-type responseHandler struct{ http.ResponseWriter }
-
-func (w responseHandler) handle(b []byte, err error) {
-	if err != nil {
-		httpserver.Errorf(w, "%s", err)
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(b)
-}
-
 func uint64FromPath(path string) (uint64, error) {
 	s := strings.TrimRight(path, "/")
 	return strconv.ParseUint(s, 10, 0)

app/vmalert/web_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestHandler(t *testing.T) {
-	rule := &Rule{
+	ar := &AlertingRule{
 		Name: "alert",
 		alerts: map[uint64]*notifier.Alert{
 			0: {},
@@ -19,7 +19,7 @@ func TestHandler(t *testing.T) {
 	}
 	g := &Group{
 		Name:  "group",
-		Rules: []*Rule{rule},
+		Rules: []Rule{ar},
 	}
 	m := &manager{groups: make(map[uint64]*Group)}
 	m.groups[0] = g
@@ -54,10 +54,17 @@ func TestHandler(t *testing.T) {
 			t.Errorf("expected 1 alert got %d", length)
 		}
 	})
+	t.Run("/api/v1/groups", func(t *testing.T) {
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/groups", &lr, 200)
+		if length := len(lr.Data.Groups); length != 1 {
+			t.Errorf("expected 1 group got %d", length)
+		}
+	})
 	t.Run("/api/v1/0/0/status", func(t *testing.T) {
 		alert := &APIAlert{}
 		getResp(ts.URL+"/api/v1/0/0/status", alert, 200)
-		expAlert := rule.newAlertAPI(*rule.alerts[0])
+		expAlert := ar.newAlertAPI(*ar.alerts[0])
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}

app/vmalert/web_types.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"time"
+)
+
+// APIAlert represents an notifier.AlertingRule state
+// for WEB view
+type APIAlert struct {
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	State       string            `json:"state"`
+	Value       string            `json:"value"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	ActiveAt    time.Time         `json:"activeAt"`
+}
+
+// APIGroup represents Group for WEB view
+type APIGroup struct {
+	Name           string             `json:"name"`
+	ID             string             `json:"id"`
+	File           string             `json:"file"`
+	Interval       string             `json:"interval"`
+	Concurrency    int                `json:"concurrency"`
+	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
+	RecordingRules []APIRecordingRule `json:"recording_rules"`
+}
+
+// APIAlertingRule represents AlertingRule for WEB view
+type APIAlertingRule struct {
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	For         string            `json:"for"`
+	LastError   string            `json:"last_error"`
+	LastExec    time.Time         `json:"last_exec"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+}
+
+// APIRecordingRule represents RecordingRule for WEB view
+type APIRecordingRule struct {
+	ID         string            `json:"id"`
+	Name       string            `json:"name"`
+	GroupID    string            `json:"group_id"`
+	Expression string            `json:"expression"`
+	LastError  string            `json:"last_error"`
+	LastExec   time.Time         `json:"last_exec"`
+	Labels     map[string]string `json:"labels"`
+}

app/vmauth/Makefile

@@ -53,22 +53,27 @@ publish-vmauth:
 
 run-vmauth:
 	APP_NAME=vmauth \
+	DOCKER_OPTS='-v $(shell pwd)/app/vmauth/:/app/vmauth' \
+	ARGS='-auth.config=app/vmauth/example_config.yml' \
 	$(MAKE) run-via-docker
 
 vmauth-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-amd64 ./app/vmauth
+	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmauth-local-with-goarch
 
 vmauth-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm ./app/vmauth
+	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmauth-local-with-goarch
 
 vmauth-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm64 ./app/vmauth
+	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmauth-local-with-goarch
 
 vmauth-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-ppc64le ./app/vmauth
+	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmauth-local-with-goarch
 
 vmauth-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-386 ./app/vmauth
+	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmauth-local-with-goarch
+
+vmauth-local-with-goarch:
+	APP_NAME=vmauth $(MAKE) app-local-with-goarch
 
 vmauth-pure:
 	APP_NAME=vmauth $(MAKE) app-local-pure

app/vmauth/README.md

@@ -23,7 +23,7 @@ Docker images for `vmauth` are available [here](https://hub.docker.com/r/victori
 
 Pass `-help` to `vmauth` in order to see all the supported command-line flags with their descriptions.
 
-Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, accounting, limits, etc.
+Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML, accounting, limits, etc.
 
 
 ### Auth config
@@ -64,6 +64,9 @@ users:
   url_prefix: "http://vminsert:8480/insert/42/prometheus"
 ```
 
+The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
+This may be useful for passing secrets to the config.
+
 
 ### Security
 
@@ -110,11 +113,11 @@ Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker i
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 
-By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
-by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
-ROOT_IMAGE=alpine:3.11 make package-vmauth
+ROOT_IMAGE=scratch make package-vmauth
 ```
 
 
@@ -137,3 +140,68 @@ curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
 The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+
+
+### Advanced usage
+
+Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
+
+```
+./vmauth -help
+
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md .
+
+  -auth.config string
+    	Path to auth config. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md for details on the format of this auth config
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address to listen for http connections (default ":8427")
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit (default 10)
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -memory.allowedBytes value
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It overrides httpAuth settings
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It overrides httpAuth settings
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+  -version
+    	Show VictoriaMetrics version
+```

app/vmauth/auth_config.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/metrics"
@@ -36,11 +37,11 @@ type UserInfo struct {
 
 func initAuthConfig() {
 	if len(*authConfigPath) == 0 {
-		logger.Panicf("FATAL: missing required `-auth.config` command-line flag")
+		logger.Fatalf("missing required `-auth.config` command-line flag")
 	}
 	m, err := readAuthConfig(*authConfigPath)
 	if err != nil {
-		logger.Panicf("FATAL: cannot load auth config from `-auth.config=%s`: %s", *authConfigPath, err)
+		logger.Fatalf("cannot load auth config from `-auth.config=%s`: %s", *authConfigPath, err)
 	}
 	authConfig.Store(m)
 	stopCh = make(chan struct{})
@@ -63,12 +64,14 @@ func authConfigReloader() {
 		case <-stopCh:
 			return
 		case <-sighupCh:
+			logger.Infof("SIGHUP received; loading -auth.config=%q", *authConfigPath)
 			m, err := readAuthConfig(*authConfigPath)
 			if err != nil {
-				logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
+				logger.Errorf("failed to load -auth.config=%q; using the last successfully loaded config; error: %s", *authConfigPath, err)
 				continue
 			}
 			authConfig.Store(m)
+			logger.Infof("Successfully reloaded -auth.config=%q", *authConfigPath)
 		}
 	}
 }
@@ -80,20 +83,21 @@ var stopCh chan struct{}
 func readAuthConfig(path string) (map[string]*UserInfo, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read %q: %s", path, err)
+		return nil, fmt.Errorf("cannot read %q: %w", path, err)
 	}
 	m, err := parseAuthConfig(data)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse %q: %s", path, err)
+		return nil, fmt.Errorf("cannot parse %q: %w", path, err)
 	}
 	logger.Infof("Loaded information about %d users from %q", len(m), path)
 	return m, nil
 }
 
 func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
+	data = envtemplate.Replace(data)
 	var ac AuthConfig
 	if err := yaml.UnmarshalStrict(data, &ac); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %s", err)
+		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
 	uis := ac.Users
 	if len(uis) == 0 {
@@ -113,7 +117,7 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		// Validate urlPrefix
 		target, err := url.Parse(urlPrefix)
 		if err != nil {
-			return nil, fmt.Errorf("invalid `url_prefix: %q`: %s", urlPrefix, err)
+			return nil, fmt.Errorf("invalid `url_prefix: %q`: %w", urlPrefix, err)
 		}
 		if target.Scheme != "http" && target.Scheme != "https" {
 			return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)

app/vmauth/example_config.yml

@@ -0,0 +1,31 @@
+# Arbitrary number of usernames may be put here.
+# Usernames must be unique.
+
+users:
+
+  # The user for querying local single-node VictoriaMetrics.
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://localhost:8428 .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query
+- username: "local-single-node"
+  password: "***"
+  url_prefix: "http://localhost:8428"
+
+  # The user for querying account 123 in VictoriaMetrics cluster
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vmselect:8481/select/123/prometheus .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8481/select/123/prometheus/api/v1/select
+- username: "cluster-select-account-123"
+  password: "***"
+  url_prefix: "http://vmselect:8481/select/123/prometheus"
+
+  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the reuqests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vminsert:8480/insert/42/prometheus .
+  # For example, http://vmauth:8427/api/v1/write is routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
+- username: "cluster-insert-account-42"
+  password: "***"
+  url_prefix: "http://vminsert:8480/insert/42/prometheus"
+

app/vmauth/main.go

@@ -5,10 +5,13 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"os"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -19,9 +22,13 @@ var (
 )
 
 func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	cgroup.UpdateGOMAXPROCSToCPUQuota()
 	logger.Infof("starting vmauth at %q...", *httpListenAddr)
 	startTime := time.Now()
 	initAuthConfig()
@@ -44,20 +51,21 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	username, password, ok := r.BasicAuth()
 	if !ok {
-		httpserver.Errorf(w, "Missing `Authorization: Basic *` header")
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		http.Error(w, "missing `Authorization: Basic *` header", http.StatusUnauthorized)
 		return true
 	}
 	ac := authConfig.Load().(map[string]*UserInfo)
 	info := ac[username]
 	if info == nil || info.Password != password {
-		httpserver.Errorf(w, "Cannot find the provided username %q or password in config", username)
+		httpserver.Errorf(w, r, "cannot find the provided username %q or password in config", username)
 		return true
 	}
 	info.requests.Inc()
 
 	targetURL := createTargetURL(info.URLPrefix, r.URL)
 	if _, err := url.Parse(targetURL); err != nil {
-		httpserver.Errorf(w, "Invalid targetURL=%q: %s", targetURL, err)
+		httpserver.Errorf(w, r, "invalid targetURL=%q: %s", targetURL, err)
 		return true
 	}
 	r.Header.Set("vm-target-url", targetURL)
@@ -74,6 +82,23 @@ var reverseProxy = &httputil.ReverseProxy{
 		}
 		r.URL = target
 	},
+	Transport: func() *http.Transport {
+		tr := http.DefaultTransport.(*http.Transport).Clone()
+		// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+		tr.DisableCompression = true
+		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+		tr.ForceAttemptHTTP2 = false
+		return tr
+	}(),
 	FlushInterval: time.Second,
 	ErrorLog:      logger.StdErrorLogger(),
 }
+
+func usage() {
+	const s = `
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md .
+`
+	flagutil.Usage(s)
+}

app/vmbackup/Makefile

@@ -51,20 +51,23 @@ package-vmbackup-386:
 publish-vmbackup:
 	APP_NAME=vmbackup $(MAKE) publish-via-docker
 
-vmbackup-pure:
-	APP_NAME=vmbackup $(MAKE) app-local-pure
-
 vmbackup-amd64:
-	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-amd64 ./app/vmbackup
+	CGO_ENABLED=1 GOARCH=amd64 $(MAKE) vmbackup-local-with-goarch
 
 vmbackup-arm:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+	CGO_ENABLED=0 GOARCH=arm $(MAKE) vmbackup-local-with-goarch
 
 vmbackup-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+	CGO_ENABLED=0 GOARCH=arm64 $(MAKE) vmbackup-local-with-goarch
 
 vmbackup-ppc64le:
-	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup
+	CGO_ENABLED=0 GOARCH=ppc64le $(MAKE) vmbackup-local-with-goarch
 
 vmbackup-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup
+	CGO_ENABLED=0 GOARCH=386 $(MAKE) vmbackup-local-with-goarch
+
+vmbackup-local-with-goarch:
+	APP_NAME=vmbackup $(MAKE) app-local-with-goarch
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure

app/vmbackup/README.md

@@ -6,12 +6,12 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. This saves time and costs on data transfer.
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
 
@@ -19,6 +19,9 @@ Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetri
 
 See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
 
+See also [vmbackuper](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) tool built on top of `vmbackup`. This tool simplifies
+creation of hourly, daily, weekly and monthly backups.
+
 
 ### Use cases
 
@@ -32,8 +35,8 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
 
@@ -46,13 +49,13 @@ with the following command:
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
 ```
 
-This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
 
 #### Incremental backups
 
-Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
-This saves time and network bandwidth costs when working with big backups:
+Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+It saves time and network bandwidth costs when working with big backups:
 
 ```
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
@@ -86,6 +89,8 @@ or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when
 
 Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
 
+See also [vmbackuper tool](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for automating smart backups.
+
 
 ### How does it work?
 
@@ -95,16 +100,16 @@ The backup algorithm is the following:
 2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
 3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
 4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
-   This are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+   These are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
+- Old files periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -118,11 +123,51 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 * If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
 * If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
 * If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+* Backups created from [single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md) cannot be restored
+  at [cluster VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md) and vice versa.
 
 
 ### Advanced usage
 
-Run `vmbackup -help` in order to see all the available options:
+
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+  You have to add a custom url endpoint via flag:
+```
+  # for minio
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+* Run `vmbackup -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -131,7 +176,7 @@ Run `vmbackup -help` in order to see all the available options:
     	Path to file with S3 configs. Configs are loaded from default location if not set.
     	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs (default "default")
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
     	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
@@ -145,23 +190,29 @@ Run `vmbackup -help` in order to see all the available options:
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
-    	Whether to use pread() instead of mmap() for reading data files
+    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit (default 10)
   -loggerFormat string
     	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -maxBytesPerSecond int
+  -maxBytesPerSecond value
     	The maximum upload speed. There is no limit if it is set to 0
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedBytes value
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -origin string
     	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
   -snapshot.createURL string
-    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snaphsot/create
   -snapshot.deleteURL string
-    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snaphosts will be automatically deleted. Example: http://victoriametrics:8428/snaphsot/delete
   -snapshotName string
     	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
   -storageDataPath string
@@ -194,9 +245,9 @@ Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` dock
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 
-By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
-by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
-ROOT_IMAGE=alpine:3.11 make package-vmbackup
+ROOT_IMAGE=scratch make package-vmbackup
 ```

app/vmbackup/main.go

@@ -10,33 +10,40 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
 	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
 	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
-	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup."+
+	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+
 		"Example: http://victoriametrics:8428/snaphsot/create")
-	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted."+
-		"Example: http://victoriametrics:8428/snaphsot/delete")
+	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+
+		"All created snaphosts will be automatically deleted. Example: http://victoriametrics:8428/snaphsot/delete")
 	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
 		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
 		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
 	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
 	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
-	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+	maxBytesPerSecond = flagutil.NewBytes("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
 )
 
 func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
 	buildinfo.Init()
+	logger.Init()
+	cgroup.UpdateGOMAXPROCSToCPUQuota()
 
 	if len(*snapshotCreateURL) > 0 {
-		logger.Infof("%s", "Snapshots enabled")
+		logger.Infof("Snapshots enabled")
 		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
@@ -48,17 +55,17 @@ func main() {
 
 		name, err := snapshot.Create(*snapshotCreateURL)
 		if err != nil {
-			logger.Fatalf("%s", err)
+			logger.Fatalf("cannot create snapshot: %s", err)
 		}
 		err = flag.Set("snapshotName", name)
 		if err != nil {
-			logger.Fatalf("Failed to set snapshotName flag: %v", err)
+			logger.Fatalf("cannot set snapshotName flag: %v", err)
 		}
 
 		defer func() {
 			err := snapshot.Delete(*snapshotDeleteURL, name)
 			if err != nil {
-				logger.Fatalf("%s", err)
+				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
 		}()
 	}
@@ -84,6 +91,9 @@ func main() {
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot create backup: %s", err)
 	}
+	srcFS.MustStop()
+	dstFS.MustStop()
+	originFS.MustStop()
 }
 
 func usage() {
@@ -93,10 +103,7 @@ or local filesystem. Backed up data can be restored with vmrestore.
 
 See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
 `
-
-	f := flag.CommandLine.Output()
-	fmt.Fprintf(f, "%s\n", s)
-	flag.PrintDefaults()
+	flagutil.Usage(s)
 }
 
 func newSrcFS() (*fslocal.FS, error) {
@@ -108,12 +115,12 @@ func newSrcFS() (*fslocal.FS, error) {
 	// Verify the snapshot exists.
 	f, err := os.Open(snapshotPath)
 	if err != nil {
-		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+		return nil, fmt.Errorf("cannot open snapshot at %q: %w", snapshotPath, err)
 	}
 	fi, err := f.Stat()
 	_ = f.Close()
 	if err != nil {
-		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+		return nil, fmt.Errorf("cannot stat %q: %w", snapshotPath, err)
 	}
 	if !fi.IsDir() {
 		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
@@ -121,10 +128,10 @@ func newSrcFS() (*fslocal.FS, error) {
 
 	fs := &fslocal.FS{
 		Dir:               snapshotPath,
-		MaxBytesPerSecond: *maxBytesPerSecond,
+		MaxBytesPerSecond: maxBytesPerSecond.N,
 	}
 	if err := fs.Init(); err != nil {
-		return nil, fmt.Errorf("cannot initialize fs: %s", err)
+		return nil, fmt.Errorf("cannot initialize fs: %w", err)
 	}
 	return fs, nil
 }
@@ -132,18 +139,18 @@ func newSrcFS() (*fslocal.FS, error) {
 func newDstFS() (common.RemoteFS, error) {
 	fs, err := actions.NewRemoteFS(*dst)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
 	return fs, nil
 }
 
-func newOriginFS() (common.RemoteFS, error) {
+func newOriginFS() (common.OriginFS, error) {
 	if len(*origin) == 0 {
-		return nil, nil
+		return &fsnil.FS{}, nil
 	}
 	fs, err := actions.NewRemoteFS(*origin)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}
 	return fs, nil
 }

app/vmbackup/snapshot/snapshot.go

@@ -20,26 +20,27 @@ type snapshot struct {
 // Create creates a snapshot and the provided api endpoint and returns
 // the snapshot name
 func Create(createSnapshotURL string) (string, error) {
-	logger.Infof("%s", "Creating snapshot")
+	logger.Infof("Creating snapshot")
 	u, err := url.Parse(createSnapshotURL)
 	if err != nil {
 		return "", err
 	}
-
 	resp, err := http.Get(u.String())
 	if err != nil {
 		return "", err
 	}
-
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
 	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code returned from %q; expecting %d; got %d; response body: %q", createSnapshotURL, resp.StatusCode, http.StatusOK, body)
+	}
 
 	snap := snapshot{}
 	err = json.Unmarshal(body, &snap)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", createSnapshotURL, err, body)
 	}
 
 	if snap.Status == "ok" {
@@ -58,26 +59,26 @@ func Delete(deleteSnapshotURL string, snapshotName string) error {
 	formData := url.Values{
 		"snapshot": {snapshotName},
 	}
-
 	u, err := url.Parse(deleteSnapshotURL)
 	if err != nil {
 		return err
 	}
-
 	resp, err := http.PostForm(u.String(), formData)
 	if err != nil {
 		return err
 	}
-
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected status code returned from %q; expecting %d; got %d; response body: %q", deleteSnapshotURL, resp.StatusCode, http.StatusOK, body)
+	}
 
 	snap := snapshot{}
 	err = json.Unmarshal(body, &snap)
 	if err != nil {
-		return err
+		return fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", deleteSnapshotURL, err, body)
 	}
 
 	if snap.Status == "ok" {

app/vminsert/common/insert_ctx.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -17,11 +18,14 @@ type InsertCtx struct {
 
 	mrs            []storage.MetricRow
 	metricNamesBuf []byte
+
+	relabelCtx relabel.Ctx
 }
 
 // Reset resets ctx for future fill with rowsLen rows.
 func (ctx *InsertCtx) Reset(rowsLen int) {
-	for _, label := range ctx.Labels {
+	for i := range ctx.Labels {
+		label := &ctx.Labels[i]
 		label.Name = nil
 		label.Value = nil
 	}
@@ -37,6 +41,7 @@ func (ctx *InsertCtx) Reset(rowsLen int) {
 	}
 	ctx.mrs = ctx.mrs[:0]
 	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
+	ctx.relabelCtx.Reset()
 }
 
 func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
@@ -48,23 +53,23 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
 }
 
 // WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
-func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) {
+func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) error {
 	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
-	ctx.addRow(metricNameRaw, timestamp, value)
+	return ctx.addRow(metricNameRaw, timestamp, value)
 }
 
 // WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer.
 //
 // It returns metricNameRaw for the given labels if len(metricNameRaw) == 0.
-func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) []byte {
+func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) ([]byte, error) {
 	if len(metricNameRaw) == 0 {
 		metricNameRaw = ctx.marshalMetricNameRaw(nil, labels)
 	}
-	ctx.addRow(metricNameRaw, timestamp, value)
-	return metricNameRaw
+	err := ctx.addRow(metricNameRaw, timestamp, value)
+	return metricNameRaw, err
 }
 
-func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float64) {
+func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float64) error {
 	mrs := ctx.mrs
 	if cap(mrs) > len(mrs) {
 		mrs = mrs[:len(mrs)+1]
@@ -76,55 +81,64 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.MetricNameRaw = metricNameRaw
 	mr.Timestamp = timestamp
 	mr.Value = value
+	if len(ctx.metricNamesBuf) > 16*1024*1024 {
+		if err := ctx.FlushBufs(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // AddLabelBytes adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.
 func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
-	labels := ctx.Labels
-	if cap(labels) > len(labels) {
-		labels = labels[:len(labels)+1]
-	} else {
-		labels = append(labels, prompb.Label{})
+	if len(value) == 0 {
+		// Skip labels without values, since they have no sense.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/600
+		// Do not skip labels with empty name, since they are equal to __name__.
+		return
 	}
-	label := &labels[len(labels)-1]
-
-	// Do not copy name and value contents for performance reasons.
-	// This reduces GC overhead on the number of objects and allocations.
-	label.Name = name
-	label.Value = value
-
-	ctx.Labels = labels
+	ctx.Labels = append(ctx.Labels, prompb.Label{
+		// Do not copy name and value contents for performance reasons.
+		// This reduces GC overhead on the number of objects and allocations.
+		Name:  name,
+		Value: value,
+	})
 }
 
 // AddLabel adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.
 func (ctx *InsertCtx) AddLabel(name, value string) {
-	labels := ctx.Labels
-	if cap(labels) > len(labels) {
-		labels = labels[:len(labels)+1]
-	} else {
-		labels = append(labels, prompb.Label{})
+	if len(value) == 0 {
+		// Skip labels without values, since they have no sense.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/600
+		// Do not skip labels with empty name, since they are equal to __name__.
+		return
 	}
-	label := &labels[len(labels)-1]
+	ctx.Labels = append(ctx.Labels, prompb.Label{
+		// Do not copy name and value contents for performance reasons.
+		// This reduces GC overhead on the number of objects and allocations.
+		Name:  bytesutil.ToUnsafeBytes(name),
+		Value: bytesutil.ToUnsafeBytes(value),
+	})
+}
 
-	// Do not copy name and value contents for performance reasons.
-	// This reduces GC overhead on the number of objects and allocations.
-	label.Name = bytesutil.ToUnsafeBytes(name)
-	label.Value = bytesutil.ToUnsafeBytes(value)
-
-	ctx.Labels = labels
+// ApplyRelabeling applies relabeling to ic.Labels.
+func (ctx *InsertCtx) ApplyRelabeling() {
+	ctx.Labels = ctx.relabelCtx.ApplyRelabeling(ctx.Labels)
 }
 
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
-	if err := vmstorage.AddRows(ctx.mrs); err != nil {
-		return &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("cannot store metrics: %s", err),
-			StatusCode: http.StatusServiceUnavailable,
-		}
+	err := vmstorage.AddRows(ctx.mrs)
+	ctx.Reset(0)
+	if err == nil {
+		return nil
+	}
+	return &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf("cannot store metrics: %w", err),
+		StatusCode: http.StatusServiceUnavailable,
 	}
-	return nil
 }

app/vminsert/csvimport/request_handler.go

@@ -4,6 +4,9 @@ import (
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -16,18 +19,23 @@ var (
 
 // InsertHandler processes /api/v1/import/csv requests.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows)
+			return insertRows(rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
 	ctx.Reset(len(rows))
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
 		ctx.Labels = ctx.Labels[:0]
@@ -36,7 +44,20 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+		for j := range extraLabels {
+			label := &extraLabels[j]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))

app/vminsert/graphite/request_handler.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -28,6 +29,7 @@ func insertRows(rows []parser.Row) error {
 	defer common.PutInsertCtx(ctx)
 
 	ctx.Reset(len(rows))
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
 		ctx.Labels = ctx.Labels[:0]
@@ -36,7 +38,16 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))

app/vminsert/influx/request_handler.go

@@ -8,7 +8,9 @@ import (
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
@@ -18,6 +20,7 @@ import (
 var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
 	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 )
 
 var (
@@ -59,38 +62,70 @@ func insertRows(db string, rows []parser.Row) error {
 	ic := &ctx.Common
 	ic.Reset(rowsLen)
 	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		ic.Labels = ic.Labels[:0]
-		hasDBLabel := false
+		hasDBKey := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
 			if tag.Key == "db" {
-				hasDBLabel = true
+				hasDBKey = true
 			}
 			ic.AddLabel(tag.Key, tag.Value)
 		}
-		if len(db) > 0 && !hasDBLabel {
+		if !hasDBKey {
 			ic.AddLabel("db", db)
 		}
-		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
-		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+		if !*skipMeasurement {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, r.Measurement...)
+		}
 		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
 		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
 			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
 		}
 		metricGroupPrefixLen := len(ctx.metricGroupBuf)
-		for j := range r.Fields {
-			f := &r.Fields[j]
-			if !skipFieldKey {
-				ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+		if hasRelabeling {
+			ctx.originLabels = append(ctx.originLabels[:0], ic.Labels...)
+			for j := range r.Fields {
+				f := &r.Fields[j]
+				if !skipFieldKey {
+					ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+				}
+				metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
+				ic.Labels = append(ic.Labels[:0], ctx.originLabels...)
+				ic.AddLabel("", metricGroup)
+				ic.ApplyRelabeling()
+				if len(ic.Labels) == 0 {
+					// Skip metric without labels.
+					continue
+				}
+				if err := ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, f.Value); err != nil {
+					return err
+				}
+			}
+		} else {
+			ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+			labelsLen := len(ic.Labels)
+			for j := range r.Fields {
+				f := &r.Fields[j]
+				if !skipFieldKey {
+					ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+				}
+				metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
+				ic.Labels = ic.Labels[:labelsLen]
+				ic.AddLabel("", metricGroup)
+				if len(ic.Labels) == 0 {
+					// Skip metric without labels.
+					continue
+				}
+				if err := ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels[len(ic.Labels)-1:], r.Timestamp, f.Value); err != nil {
+					return err
+				}
 			}
-			metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
-			ic.Labels = ic.Labels[:0]
-			ic.AddLabel("", metricGroup)
-			ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels[:1], r.Timestamp, f.Value)
 		}
-		rowsTotal += len(r.Fields)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
@@ -101,12 +136,21 @@ type pushCtx struct {
 	Common         common.InsertCtx
 	metricNameBuf  []byte
 	metricGroupBuf []byte
+	originLabels   []prompb.Label
 }
 
 func (ctx *pushCtx) reset() {
 	ctx.Common.Reset(0)
 	ctx.metricNameBuf = ctx.metricNameBuf[:0]
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+
+	originLabels := ctx.originLabels
+	for i := range originLabels {
+		label := &originLabels[i]
+		label.Name = nil
+		label.Value = nil
+	}
+	ctx.originLabels = ctx.originLabels[:0]
 }
 
 func getPushCtx() *pushCtx {

app/vminsert/main.go

@@ -4,15 +4,20 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prompush"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
@@ -21,6 +26,7 @@ import (
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -28,12 +34,13 @@ import (
 
 var (
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to `http://<victoriametrics>:8428/write`")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
 )
 
 var (
@@ -45,8 +52,9 @@ var (
 
 // Init initializes vminsert.
 func Init() {
+	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
-
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -78,6 +86,7 @@ func Stop() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 }
 
 // RequestHandler is a handler for Prometheus remote storage write API
@@ -88,7 +97,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -97,7 +106,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -106,7 +115,25 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -115,7 +142,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -128,14 +155,31 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		return true
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/ready":
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+			errMsg := fmt.Sprintf("waiting for scrape config to init targets, configs left: %d", rdy)
+			http.Error(w, errMsg, http.StatusTooEarly)
+		} else {
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("OK"))
+		}
+		return true
 	default:
 		// This is not our link
 		return false
@@ -152,12 +196,29 @@ var (
 	csvimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 	csvimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 
+	prometheusimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+	prometheusimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+
+	nativeimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/query", protocol="influx"}`)
 
-	promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
+	promscrapeTargetsRequests      = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
+
+	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+	})
 )

app/vminsert/native/request_handler.go

@@ -0,0 +1,115 @@
+package native
+
+import (
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import/native` request.
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	hasRelabeling := relabel.HasRelabeling()
+	mn := &block.MetricName
+	ic.Labels = ic.Labels[:0]
+	ic.AddLabelBytes(nil, mn.MetricGroup)
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		ic.AddLabelBytes(tag.Key, tag.Value)
+	}
+	for j := range extraLabels {
+		label := &extraLabels[j]
+		ic.AddLabel(label.Name, label.Value)
+	}
+	if hasRelabeling {
+		ic.ApplyRelabeling()
+	}
+	if len(ic.Labels) == 0 {
+		// Skip metric without labels.
+		return nil
+	}
+	ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	for j, value := range values {
+		timestamp := timestamps[j]
+		if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
+			return err
+		}
+	}
+	return ic.FlushBufs()
+}
+
+type pushCtx struct {
+	Common        common.InsertCtx
+	metricNameBuf []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/opentsdb/request_handler.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -28,6 +29,7 @@ func insertRows(rows []parser.Row) error {
 	defer common.PutInsertCtx(ctx)
 
 	ctx.Reset(len(rows))
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
 		ctx.Labels = ctx.Labels[:0]
@@ -36,7 +38,16 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))

app/vminsert/opentsdbhttp/request_handler.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -34,6 +35,7 @@ func insertRows(rows []parser.Row) error {
 	defer common.PutInsertCtx(ctx)
 
 	ctx.Reset(len(rows))
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
 		ctx.Labels = ctx.Labels[:0]
@@ -42,7 +44,16 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))