lib/promscrape/discovery/consul: reduce load on Consul when discovering big number of targets by using background caching

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574

.github/workflows/main.yml

@@ -0,0 +1,42 @@
+name: main
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@master
+        with:
+          go-version: 1.14
+        id: go
+      - name: Dependencies
+        env:
+          GO111MODULE: on
+        run: |
+          go get -u golang.org/x/lint/golint
+          go get -u github.com/kisielk/errcheck
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+            export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+            make check-all
+            git diff --exit-code
+            make test-full
+            make test-pure
+            make test-full-386
+            make vminsert vmselect vmstorage
+            make vminsert-pure vmselect-pure vmstorage-pure
+            make vmutils
+            GOOS=freebsd go build -mod=vendor ./app/vminsert
+            GOOS=freebsd go build -mod=vendor ./app/vmselect
+            GOOS=freebsd go build -mod=vendor ./app/vmstorage
+            GOOS=darwin go build -mod=vendor ./app/vminsert
+            GOOS=darwin go build -mod=vendor ./app/vmselect
+            GOOS=darwin go build -mod=vendor ./app/vmstorage

.gitignore

@@ -7,5 +7,13 @@
 *.swp
 /gocache-for-docker
 /victoria-metrics-data
+/vmagent-remotewrite-data
 /vmstorage-data
 /vmselect-cache
+.DS_Store
+
+
+### terraform
+terraform.tfstate
+terraform.tfstate.*
+.terraform/

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019 VictoriaMetrics, Inc.
+   Copyright 2019-2020 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,7 +1,7 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}')))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -11,7 +11,24 @@ endif
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
 
 all: \
-	victoria-metrics-prod
+	vminsert \
+	vmselect \
+	vmstorage \
+	vmagent \
+	vmalert \
+	vmauth \
+	vmbackup \
+	vmrestore
+
+all-pure: \
+	vminsert-pure \
+	vmselect-pure \
+	vmstorage-pure \
+	vmagent-pure \
+	vmalert-pure \
+	vmauth-pure \
+	vmbackup-pure \
+	vmrestore-pure
 
 include app/*/Makefile
 include deployment/*/Makefile
@@ -19,46 +36,139 @@ include deployment/*/Makefile
 clean:
 	rm -rf bin/*
 
-release: victoria-metrics-prod
-	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod
+publish: \
+	publish-vminsert \
+	publish-vmselect \
+	publish-vmstorage \
+	publish-vmagent \
+	publish-vmalert \
+	publish-vmauth \
+	publish-vmbackup \
+	publish-vmrestore
+
+package: \
+	package-vminsert \
+	package-vmselect \
+	package-vmstorage \
+	package-vmagent \
+	package-vmalert \
+	package-vmauth \
+	package-vmbackup \
+	package-vmrestore
+
+vmutils: \
+	vmagent \
+	vmalert \
+	vmauth \
+	vmbackup \
+	vmrestore
+
+release: \
+	release-vmcluster \
+	release-vmutils
+
+release-vmcluster: \
+	vminsert-prod \
+	vmselect-prod \
+	vmstorage-prod
+	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz vminsert-prod vmselect-prod vmstorage-prod && \
+		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
+
+release-vmutils: \
+	vmagent-prod \
+	vmalert-prod \
+	vmauth-prod \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmauth-prod vmbackup-prod vmrestore-prod && \
+		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	go fmt $(PKG_PREFIX)/lib/...
-	go fmt $(PKG_PREFIX)/app/...
+	GO111MODULE=on gofmt -l -w -s ./lib
+	GO111MODULE=on gofmt -l -w -s ./app
 
 vet:
-	go vet $(PKG_PREFIX)/lib/...
-	go vet $(PKG_PREFIX)/app/...
+	GO111MODULE=on go vet -mod=vendor ./lib/...
+	GO111MODULE=on go vet -mod=vendor ./app/...
 
 lint: install-golint
 	golint lib/...
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get -u github.com/golang/lint/golint
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
 
 install-errcheck:
 	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
 
+check-all: fmt vet lint errcheck golangci-lint
+
 test:
-	go test $(PKG_PREFIX)/lib/...
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+
+test-race:
+	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
+
+test-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+
+test-full:
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
+test-full-386:
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	go test -bench=. $(PKG_PREFIX)/lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+
+benchmark-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
-	go get -u
-	go mod tidy
-	go mod vendor
+	GO111MODULE=on go get -u ./lib/...
+	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+
+app-local:
+	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
 	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
+
+
+golangci-lint: install-golangci-lint
+	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
+
+install-golangci-lint:
+	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+
+docs-sync:
+	cp app/vmagent/README.md docs/vmagent.md
+	cp app/vmalert/README.md docs/vmalert.md
+	cp app/vmauth/README.md docs/vmauth.md
+	cp app/vmbackup/README.md docs/vmbackup.md
+	cp app/vmrestore/README.md docs/vmrestore.md
+	cp README.md docs/Cluster-VictoriaMetrics.md

README.md

@@ -1,386 +1,348 @@
-<img  text-align="center" alt="Victoria Metrics" src="logo.png">
+<img alt="Victoria Metrics" src="logo.png">
 
-## Single-node VictoriaMetrics
+# Cluster version
 
-[![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+VictoriaMetrics is fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
 
-VictoriaMetrics is a long-term remote storage for Prometheus.
-It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
-[docker images](https://hub.docker.com/r/valyala/victoria-metrics/) and
-in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
+It is recommended using [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics) instead of cluster version
+for ingestion rates lower than a million of data points per second.
+Single-node version [scales perfectly](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+with the number of CPU cores, RAM and available storage space.
+Single-node version is easier to configure and operate comparing to cluster version, so think twice before sticking to cluster version.
 
-Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+Join [our Slack](http://slack.victoriametrics.com/) or [contact us](mailto:info@victoriametrics.com) with consulting and support questions.
 
 
 ## Prominent features
 
-* Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
-  Additionally, VictoriaMetrics extends PromQL with opt-in [useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
-* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
-  and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
-  [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* [Uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) when working with millions of unique time series (aka high cardinality).
-* High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
-  may be crammed into a limited storage comparing to TimescaleDB.
-* Optimized for storage with high-latency IO and low iops (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
-  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* Easy operation:
-  * VictoriaMetrics consists of a single executable without external dependencies.
-  * All the configuration is done via explicit command-line flags with reasonable defaults.
-  * All the data is stored in a single directory pointed by `-storageDataPath` flag.
-  * Easy backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-* Storage is protected from corruption on unclean shutdown (i.e. hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-* Supports metrics' ingestion and backfilling via the following protocols:
-  * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-  * [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
-  * [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
-    if `-graphiteListenAddr` is set.
-  * [OpenTSDB put message](http://opentsdb.net/docs/build/html/api_telnet/put.html) if `-opentsdbListenAddr` is set.
-* Ideally works with big amounts of time series data from IoT sensors, connected car sensors and industrial sensors.
-* Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+- Supports all the features of [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- Performance and capacity scales horizontally. See [these docs for details](#cluster-resizing-and-scalability).
+- Supports multiple independent namespaces for time series data (aka multi-tenancy). See [these docs for details](#multitenancy).
+- Supports replication. See [these docs for details](#replication-and-data-safety).
 
 
-## Operation
+## Architecture overview
+
+VictoriaMetrics cluster consists of the following services:
+
+- `vmstorage` - stores the data
+- `vminsert` - proxies the ingested data to `vmstorage` shards using consistent hashing
+- `vmselect` - performs incoming queries using the data from `vmstorage`
+
+Each service may scale independently and may run on the most suitable hardware.
+`vmstorage` nodes don't know about each other, don't communicate with each other and don't share any data.
+This is [shared nothing architecture](https://en.wikipedia.org/wiki/Shared-nothing_architecture).
+It increases cluster availability, simplifies cluster maintenance and cluster scaling.
+
+<img src="https://docs.google.com/drawings/d/e/2PACX-1vTvk2raU9kFgZ84oF-OKolrGwHaePhHRsZEcfQ1I_EC5AB_XPWwB392XshxPramLJ8E4bqptTnFn5LL/pub?w=1104&amp;h=746">
 
 
-### Table of contents
+## Multitenancy
 
-* [How to build from sources](#how-to-build-from-sources)
-* [How to start VictoriaMetrics](#how-to-start-victoriametrics)
-* [Prometheus setup](#prometheus-setup)
-* [Grafana setup](#grafana-setup)
-* [How to send data from InfluxDB-compatible agents such as Telegraf](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
-* [How to send data from Graphite-compatible agents such as StatsD](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
-* [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
-* [How to apply new config / ugrade VictoriaMetrics](#how-to-apply-new-config--upgrade-victoriametrics)
-* [How to work with snapshots](#how-to-work-with-snapshots)
-* [How to delete time series](#how-to-delete-time-series)
-* [How to export time series](#how-to-export-time-series)
-* [Federation](#federation)
-* [Capacity planning](#capacity-planning)
-* [High Availability](#high-availability)
-* [Multiple retentions](#multiple-retentions)
-* [Scalability and cluster version](#scalability-and-cluster-version)
-* [Security](#security)
-* [Tuning](#tuning)
-* [Monitoring](#monitoring)
-* [Troubleshooting](#troubleshooting)
-* [Community and contributions](#community-and-contributions)
-* [Reporting bugs](#reporting-bugs)
+VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces).
+Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request urls.
+See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics:
+
+* Each `accountID` and `projectID` is identified by an arbitrary 32-bit integer in the range `[0 .. 2^32)`.
+If `projectID` is missing, then it is automatically assigned to `0`. It is expected that other information about tenants
+such as auth tokens, tenant names, limits, accounting, etc. is stored in a separate relational database. This database must be managed
+by a separate service sitting in front of VictoriaMetrics cluster such as [vmauth](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md).
+[Contact us](mailto:info@victoriametrics.com) if you need help with creating such a service.
+
+* Tenants are automatically created when the first data point is written into the given tenant.
+
+* Data for all the tenants is evenly spread among available `vmstorage` nodes. This guarantees even load among `vmstorage` nodes
+when different tenants have different amounts of data and different query load.
+
+* VictoriaMetrics doesn't support querying multiple tenants in a single request.
 
 
-### How to build from sources
+## Binaries
 
-We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
-[docker images](https://hub.docker.com/r/valyala/victoria-metrics/) instead of building VictoriaMetrics
-from sources. Building from sources is reasonable when developing an additional features specific
-to your needs.
+Compiled binaries for cluster version are available in the `assets` section of [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+See archives containing `cluster` word.
+
+Docker images for cluster version are available here:
+
+- `vminsert` - https://hub.docker.com/r/victoriametrics/vminsert/tags
+- `vmselect` - https://hub.docker.com/r/victoriametrics/vmselect/tags
+- `vmstorage` - https://hub.docker.com/r/victoriametrics/vmstorage/tags
 
 
-#### Development build
+## Building from sources
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
-2. Run `go build ./app/victoria-metrics` from the root folder of the repository.
-   It will build `victoria-metrics` binary in the root folder of the repository.
+Source code for cluster version is available at [cluster branch](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
-#### Production build
 
-1. [Install docker](https://docs.docker.com/install/).
-2. Run `make victoria-metrics-prod` from the root folder of the respository.
-   It will build `victoria-metrics-prod` binary and put it into the `bin` folder.
+### Production builds
 
-#### Building docker images
+There is no need in installing Go on a host system since binaries are built
+inside [the official docker container for Go](https://hub.docker.com/_/golang).
+This makes reproducible builds.
+So [install docker](https://docs.docker.com/install/) and run the following command:
+
+```
+make vminsert-prod vmselect-prod vmstorage-prod
+```
+
+Production binaries are built into statically linked binaries. They are put into `bin` folder with `-prod` suffixes:
+```
+$ make vminsert-prod vmselect-prod vmstorage-prod
+$ ls -1 bin
+vminsert-prod
+vmselect-prod
+vmstorage-prod
+```
+
+### Development Builds
+
+1. [Install go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make` from the repository root. It should build `vmstorage`, `vmselect`
+   and `vminsert` binaries and put them into the `bin` folder.
+
+
+### Building docker images
+
+Run `make package`. It will build the following docker images locally:
+
+* `victoriametrics/vminsert:<PKG_TAG>`
+* `victoriametrics/vmselect:<PKG_TAG>`
+* `victoriametrics/vmstorage:<PKG_TAG>`
 
-Run `make package-victoria-metrics`. It will build `valyala/victoria-metrics:<PKG_TAG>` docker image locally.
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package`.
 
+By default images are built on top of `alpine` image in order to improve debuggability. It is possible to build an image on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds images on top of `scratch` image:
 
-
-### How to start VictoriaMetrics
-
-Just start VictoriaMetrics executable or docker image with the desired command-line flags.
-
-The following command line flags are used the most:
-
-* `-storageDataPath` - path to data directory. VictoriaMetrics stores all the data in this directory.
-* `-retentionPeriod` - retention period in months for the data. Older data is automatically deleted.
-* `-httpListenAddr` - TCP address to listen to for http requests. By default it listens port `8428` on all the network interfaces.
-* `-graphiteListenAddr` - TCP and UDP address to listen to for Graphite data. By default it is disabled.
-* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data. By default it is disabled.
-
-Pass `-help` to see all the available flags with description and default values.
-
-
-### Prometheus setup
-
-Add the following lines to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`):
-
-```yml
-remote_write:
-  - url: http://<victoriametrics-addr>:8428/api/v1/write
-    queue_config:
-      max_samples_per_send: 10000
+```bash
+ROOT_IMAGE=scratch make package
 ```
 
-Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
-Then apply the new config via the following command:
+## Operation
 
-```
-kill -HUP `pidof prometheus`
-```
+### Cluster setup
 
-Prometheus writes incoming data to local storage and to remote storage in parallel.
-This means the data remains available in local storage for `--storage.tsdb.retention.time` duration
-if remote storage stops working.
+A minimal cluster must contain the following nodes:
 
-If you plan sending data to VictoriaMetrics from multiple Prometheus instances, then add the following lines into `global` section
-of [Prometheus config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file):
+* a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags
+* a single `vminsert` node with `-storageNode=<vmstorage_host>:8400`
+* a single `vmselect` node with `-storageNode=<vmstorage_host>:8401`
 
-```yml
-global:
-  external_labels:
-    datacenter: dc-123
-```
+It is recommended to run at least two nodes for each service
+for high availability purposes.
 
-This instructs Prometheus to add `datacenter=dc-123` label to each time series sent to remote storage.
-The label name may be arbitrary - `datacenter` is just an example. The label value must be unique
-across Prometheus instances, so time series may be filtered and grouped by this label.
+An http load balancer such as `nginx` must be put in front of `vminsert` and `vmselect` nodes:
+- requests starting with `/insert` must be routed to port `8480` on `vminsert` nodes.
+- requests starting with `/select` must be routed to port `8481` on `vmselect` nodes.
 
+Ports may be altered by setting `-httpListenAddr` on the corresponding nodes.
 
-### Grafana setup
+It is recommended setting up [monitoring](#monitoring) for the cluster.
 
-Create [Prometheus datasource](http://docs.grafana.org/features/datasources/prometheus/) in Grafana with the following Url:
+#### Environment variables
 
-```
-http://<victoriametrics-addr>:8428
-```
+Each flag values can be set thru environment variables by following these rules:
 
-Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
-
-Then build graphs with the created datasource using [Prometheus query language](https://prometheus.io/docs/prometheus/latest/querying/basics/).
-VictoriaMetrics supports native PromQL and [extends it with useful features](ExtendedPromQL).
-
-
-### How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)?
-
-Just use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
-For instance, put the following lines into `Telegraf` config, so it sends data to VictoriaMetrics instead of InfluxDB:
-
-```
-[[outputs.influxdb]]
-  urls = ["http://<victoriametrics-addr>:8428"]
-```
-
-Do not forget substituting `<victoriametrics-addr>` with the real address where VictoriaMetrics runs.
-
-VictoriaMetrics maps Influx data using the following rules:
-* [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
-* Field names are mapped to time series names prefixed by `{measurement}.` value
-* Field values are mapped to time series values
-* Tags are mapped to Prometheus labels as-is
-
-
-### How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)?
-
-1) Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
-the following command will enable Graphite receiver in VictoriaMetrics on TCP and UDP port `2003`:
-
-```
-/path/to/victoria-metrics-prod ... -graphiteListenAddr=:2003
-```
-
-2) Use the configured address in Graphite-compatible agents. For instance, set `graphiteHost`
-to the VictoriaMetrics host in `StatsD` configs.
-
-
-### How to send data from OpenTSDB-compatible agents?
-
-1) Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
-the following command will enable OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
-
-```
-/path/to/victoria-metrics-prod ... -opentsdbListenAddr=:4242
-```
-
-2) Send data to the given address from OpenTSDB-compatible agents.
-
-
-### How to apply new config / upgrade VictoriaMetrics?
-
-VictoriaMetrics must be restarted in order to upgrade or apply new config:
-
-1) Send `SIGINT` signal to VictoriaMetrics process in order to gracefully stop it.
-2) Wait until the process stops. This can take a few seconds.
-3) Start the upgraded VictoriaMetrics with new config.
-
-
-### How to work with snapshots?
-
-Navigate to `http://<victoriametrics-addr>:8428/snapshot/create` in order to create an instant snapshot.
-The page will return the following JSON response:
-
-```
-{"status":"ok","snapshot":"<snapshot-name>"}
-```
-
-Snapshots are created under `<-storageDataPath>/snapshots` directory, where `<-storageDataPath>`
-is the command-line flag value. Snapshots can be archived to backup storage via `rsync -L`, `scp -r`
-or any similar tool that follows symlinks during copying.
-
-The `http://<victoriametrics-addr>:8428/snapshot/list` page contains the list of available snapshots.
-
-Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete?snapshot=<snapshot-name>` in order
-to delete `<snapshot-name>` snapshot.
-
-Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete_all` in order to delete all the snapshots.
-
-
-### How to delete time series?
-
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`,
-where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
-for metrics to delete. After that all the time series matching the given selector are deleted. Storage space for
-the deleted time series isn't freed instantly - it is freed during subsequent merges of data files.
-
-
-### How to export time series?
-
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
-where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
-for metrics to export. The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
-Each JSON line would contain data for a single time series. An example output:
-
-```
-{"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
-{"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
-```
-
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
-
-
-### Federation
-
-VictoriaMetrics exports [Prometheus-compatible federation data](https://prometheus.io/docs/prometheus/latest/federation/)
-at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
-
-Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
-`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. By default the last point
-on the interval `[now - max_lookback ... now]` is scraped for each time series. Default value for `max_lookback` is `5m` (5 minutes), but can be overriden.
-For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
-with scrape intervals exceeding `5m`.
-
-
-### Capacity planning
-
-Rough estimation of the required resources:
-
-* RAM size: less than 1KB per active time series. So, ~1GB of RAM is required for 1M active time series.
-  Time series is considered active if new data points have been added to it recently or if it has been recently queried.
-  VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with `-memory.allowedPercent` flag.
-* CPU cores: a CPU core per 300K inserted data points per second. So, ~4 CPU cores are required for processing
-  the insert stream of 1M data points per second.
-  If you see lower numbers per CPU core, then it is likely active time series info doesn't fit caches,
-  so you need more RAM for lowering CPU usage.
-* Storage size: less than a byte per data point on average. So, ~260GB is required for storing a month-long insert stream
-  of 100K data points per second.
-  The actual storage size heavily depends on data randomness (entropy). Higher randomness means higher storage size requirements.
-
-
-### High availability
-
-1) Install multiple VictoriaMetrics instances in distinct datacenters.
-2) Add addresses of these instances to `remote_write` section in Prometheus config:
-
-```yml
-remote_write:
-  - url: http://<victoriametrics-addr-1>:8428/api/v1/write
-    queue_config:
-      max_samples_per_send: 10000
-  # ...
-  - url: http://<victoriametrics-addr-N>:8428/api/v1/write
-    queue_config:
-      max_samples_per_send: 10000
-```
-
-3) Apply the updated config:
-
-```
-kill -HUP `pidof prometheus`
-```
-
-4) Now Prometheus should write data into all the configured `remote_write` urls in parallel.
-5) Set up [Promxy](https://github.com/jacksontj/promxy) in front of all the VictoriaMetrics replicas.
-6) Set up Prometheus datasource in Grafana that points to Promxy.
-
-
-### Multiple retentions
-
-Just start multiple VictoriaMetrics instances with distinct values for the following flags:
-
-* `-retentionPeriod`
-* `-storageDataPath`, so the data for each retention period is saved in a separate directory
-* `-httpListenAddr`, so clients may reach VictoriaMetrics instance with proper retention
-
-
-### Scalability and cluster version
-
-Though single-node VictoriaMetrics cannot scale to multiple nodes, it is optimized for resource usage - storage size / bandwidth / IOPS, RAM, CPU.
-This means that a single-node VictoriaMetrics may scale vertically and substitute moderately sized cluster built with competing solutions
-such as Thanos, Uber M3, InfluxDB or TimescaleDB.
-
-So try single-node VictoriaMetrics at first and then [switch to cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) if you still need
-horizontally scalable long-term remote storage for really large Prometheus deployments.
-[Contact us](mailto:info@victoriametrics.com) for paid support.
-
-
-### Security
-
-Do not forget protecting sensitive endpoints in VictoriaMetrics when exposing it to untrusted networks such as internet.
-Consider setting the following command-line flags:
-
-* `-tls`, `-tlsCertFile` and `-tlsKeyFile` for switching from HTTP to HTTPS.
-* `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
-  with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
-* `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
-* `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
-
-Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
-For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`.
-
-
-### Tuning
-
-* There is no need in VictoriaMetrics tuning, since it uses reasonable defaults for command-line flags,
-  which are automatically adjusted for the available CPU and RAM resources.
-* There is no need in Operating System tuning, since VictoriaMetrics is optimized for default OS settings.
-  The only option is increasing the limit on [the number open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
-  so Prometheus instances could establish more connections to VictoriaMetrics.
+- The `-envflag.enable` flag must be set
+- Each `.` in flag names must be substituted by `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`)
+- For repeating flags, an alternative syntax can be used by joining the different values into one using `,` as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`)
+- It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
 
 
 ### Monitoring
 
-VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page.
-Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics.
-There is [an official Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229).
+All the cluster components expose various metrics in Prometheus-compatible format at `/metrics` page on the TCP port set in `-httpListenAddr` command-line flag.
+By default the following TCP ports are used:
+- `vminsert` - 8480
+- `vmselect` - 8481
+- `vmstorage` - 8482
+
+It is recommended setting up [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md)
+or Prometheus to scrape `/metrics` pages from all the cluster components, so they can be monitored and analyzed
+with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11831).
 
 
-### Troubleshooting
+### URL format
 
-* If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
-  then it is likely you have too many active time series for the current amount of RAM.
-  It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
-  ingestion performance.
-  Another option is to increase `-memory.allowedPercent` command-line flag value. Be careful with this
-  option, since too big value for `-memory.allowedPercent` may result in high I/O usage.
+* URLs for data ingestion: `http://<vminsert>:8480/insert/<accountID>/<suffix>`, where:
+  - `<accountID>` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`,
+    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`.
+  - `<suffix>` may have the following values:
+     - `prometheus` and `prometheus/api/v1/write` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+     - `influx/write` and `influx/api/v2/write` - for inserting data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
+     - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html).
+       This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag.
+       See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#sending-opentsdb-data-via-http-apiput-requests) for details.
+     - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
+     - `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data) for details.
+
+* URLs for querying: `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
+  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
+  - `<suffix>` may have the following values:
+    - `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries).
+    - `api/v1/query_range` - performs [PromQL range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).
+    - `api/v1/series` - performs [series query](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers).
+    - `api/v1/labels` - returns a [list of label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names).
+    - `api/v1/label/<label_name>/values` - returns values for the given `<label_name>` according [to API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values).
+    - `federate` - returns [federated metrics](https://prometheus.io/docs/prometheus/latest/federation/).
+    - `api/v1/export` - exports raw data. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details.
+    - `api/v1/status/tsdb` - for time series stats. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
+
+* URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
+  Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
+  be used on a regular basis, since it carries non-zero overhead.
+
+* `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
+  - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
+    which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
+    command-line flag value.
+  - `/snapshot/list` - list available snasphots.
+  - `/snapshot/delete?snapshot=<id>` - delete the given snapshot.
+  - `/snapshot/delete_all` - delete all the snapshots.
+
+  Snapshots may be created independently on each `vmstorage` node. There is no need in synchronizing snapshots' creation
+  across `vmstorage` nodes.
+
+
+### Cluster resizing and scalability
+
+Cluster performance and capacity scales with adding new nodes.
+
+* `vminsert` and `vmselect` nodes are stateless and may be added / removed at any time.
+  Do not forget updating the list of these nodes on http load balancer.
+  Adding more `vminsert` nodes scales data ingestion rate. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/175#issuecomment-536925841)
+  about ingestion rate scalability.
+  Adding more `vmselect` nodes scales select queries rate.
+* `vmstorage` nodes own the ingested data, so they cannot be removed without data loss.
+  Adding more `vmstorage` nodes scales cluster capacity.
+
+Steps to add `vmstorage` node:
+
+1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster.
+2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8401`.
+3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8400`.
+
+
+### Updating / reconfiguring cluster nodes
+
+All the node types - `vminsert`, `vmselect` and `vmstorage` - may be updated via graceful shutdown.
+Send `SIGINT` signal to the corresponding process, wait until it finishes and then start new version
+with new configs.
+
+Cluster should remain in working state if at least a single node of each type remains available during
+the update process. See [cluster availability](#cluster-availability) section for details.
+
+
+### Cluster availability
+
+* HTTP load balancer must stop routing requests to unavailable `vminsert` and `vmselect` nodes.
+* The cluster remains available if at least a single `vmstorage` node exists:
+
+  - `vminsert` re-routes incoming data from unavailable `vmstorage` nodes to healthy `vmstorage` nodes
+  - `vmselect` continues serving partial responses if at least a single `vmstorage` node is available.
+
+Data replication can be used for increasing storage durability. See [these docs](#replication-and-data-safety) for details.
+
+
+### Capacity planning
+
+Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the most suitable hardware.
+
+#### vminsert
+
+* The recommended total number of vCPU cores for all the `vminsert` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended number of vCPU cores per each `vminsert` instance should equal to the number of `vmstorage` instances in the cluster.
+* The amount of RAM per each `vminsert` instance should be 1GB or more. RAM is used as a buffer for spikes in ingestion rate.
+  The maximum amount of used RAM per `vminsert` node can be tuned with `-memory.allowedPercent` command-line flag. For instance, `-memory.allowedPercent=20`
+  limits the maximum amount of used RAM to 20% of the available RAM on the host system.
+* Sometimes `-rpc.disableCompression` command-line flag on `vminsert` instances could increase ingestion capacity at the cost
+  of higher network bandwidth usage between `vminsert` and `vmstorage`.
+
+#### vmstorage
+
+* The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
+  Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
+* The recommended total amount of storage space for all the `vmstorage` instances can be calculated
+  from the ingestion rate and retention: `storage_space = ingestion_rate * retention_seconds`.
+
+#### vmselect
+
+The recommended hardware for `vmselect` instances highly depends on the type of queries. Lightweight queries over small number of time series usually require
+small number of vCPU cores and small amount of RAM on `vmselect`, while heavy queries over big number of time series (>10K) usually require
+bigger number of vCPU cores and bigger amounts of RAM.
+
+In general it is recommended increasing the number of vCPU cores and RAM per `vmselect` node for higher query performance,
+while adding new `vmselect` nodes only when old nodes are overloaded with incoming query stream.
+
+
+### High availability
+
+It is recommended to run all the components for a single cluster in the same subnetwork with high bandwidth, low latency and low error rates.
+This improves cluster performance and availability.
+It isn't recommended spreading components for a single cluster across multiple availability zones, since cross-AZ network usually has lower bandwidth, higher latency
+and higher error rates comparing the network inside AZ.
+
+If you need multi-AZ setup, then it is recommended running independed clusters in each AZ and setting up
+[vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) in front of these clusters, so it could replicate incoming data
+into all the cluster. Then [promxy](https://github.com/jacksontj/promxy) could be used for querying the data from multiple clusters.
+
+
+### Helm
+
+Helm chart simplifies managing cluster version of VictoriaMetrics in Kubernetes.
+It is available in the [helm-charts](https://github.com/VictoriaMetrics/helm-charts) repository.
+
+Upgrade follows `Cluster resizing procedure` under the hood.
+
+
+### Replication and data safety
+
+In order to enable application-level replication, `-replicationFactor=N` command-line flag must be passed to `vminsert`.
+This guarantees that all the data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
+For example, when `-replicationFactor=3` is passed to `vminsert`, then it replicates all the ingested data to 3 distinct `vmstorage` nodes.
+
+When the replication is enabled, `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect`
+in order to de-duplicate replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms
+when [deduplication](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#deduplication) is used additionally to replication.
+
+Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883),
+so it is recommended performing regular backups. See [these docs](#backups) for details.
+
+By default VictoriaMetrics offloads replication to the underlying storage pointed by `-storageDataPath`.
+It is recommended storing data on [Google Compute Engine persistent disks](https://cloud.google.com/compute/docs/disks/#pdspecs),
+since they are protected from data loss and data corruption. They also provide consistently high performance
+and [may be resized](https://cloud.google.com/compute/docs/disks/add-persistent-disk) without downtime.
+HDD-based persistent disks should be enough for the majority of use cases.
+
+It is recommended using durable replicated persistent volumes in Kubernetes.
+
+
+### Backups
+
+It is recommended performing periodical backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+for protecting from user errors such as accidental data deletion.
+
+The following steps must be performed for each `vmstorage` node for creating a backup:
+
+1. Create an instant snapshot by navigating to `/snapshot/create` HTTP handler. It will create snapshot and return its name.
+2. Archive the created snapshot from `<-storageDataPath>/snapshots/<snapshot_name>` folder using [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmbackup/README.md).
+   The archival process doesn't interfere with `vmstorage` work, so it may be performed at any suitable time.
+3. Delete unused snapshots via `/snapshot/delete?snapshot=<snapshot_name>` or `/snapshot/delete_all` in order to free up occupied storage space.
+
+There is no need in synchronizing backups among all the `vmstorage` nodes.
+
+Restoring from backup:
+
+1. Stop `vmstorage` node with `kill -INT`.
+2. Restore data from backup using [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmrestore/README.md) into `-storageDataPath` directory.
+3. Start `vmstorage` node.
 
 
 ## Community and contributions
 
-Feel free asking any questions regarding VictoriaMetrics [here](https://groups.google.com/forum/#!forum/victorametrics-users).
-
 We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
 
 - Prefer simple code and architecture.
@@ -392,6 +354,16 @@ We are open to third-party pull requests provided they follow [KISS design princ
 
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
 
+Due to `KISS` cluster version of VictoriaMetrics has no the following "features" popular in distributed computing world:
+
+- Fragile gossip protocols. See [failed attempt in Thanos](https://github.com/improbable-eng/thanos/blob/030bc345c12c446962225221795f4973848caab5/docs/proposals/completed/201809_gossip-removal.md).
+- Hard-to-understand-and-implement-properly [Paxos protocols](https://www.quora.com/In-distributed-systems-what-is-a-simple-explanation-of-the-Paxos-algorithm).
+- Complex replication schemes, which may go nuts in unforesseen edge cases. See [replication docs](#replication-and-data-safety) for details.
+- Automatic data reshuffling between storage nodes, which may hurt cluster performance and availability.
+- Automatic cluster resizing, which may cost you a lot of money if improperly configured.
+- Automatic discovering and addition of new nodes in the cluster, which may mix data between dev and prod clusters :)
+- Automatic leader election, which may result in split brain disaster on network errors.
+
 
 ## Reporting bugs
 

app/victoria-metrics/Makefile

@@ -1,21 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-metrics-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker
-
-package-victoria-metrics:
-	APP_NAME=victoria-metrics \
-	$(MAKE) package-via-docker
-
-publish-victoria-metrics:
-	APP_NAME=victoria-metrics $(MAKE) publish-via-docker
-
-run-victoria-metrics:
-	mkdir -p victoria-metrics-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 -p 2003:2003 -p 2003:2003/udp' \
-	APP_NAME=victoria-metrics \
-	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
-	$(MAKE) run-via-docker
-
-victoria-metrics-arm:
-	CC=arm-linux-gnueabi-gcc CGO_ENABLED=1 GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics

app/victoria-metrics/deployment/Dockerfile

@@ -1,5 +0,0 @@
-FROM scratch
-COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY bin/victoria-metrics-prod .
-EXPOSE 8428
-ENTRYPOINT ["/victoria-metrics-prod"]

app/victoria-metrics/main.go

@@ -1,60 +0,0 @@
-package main
-
-import (
-	"flag"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-)
-
-var httpListenAddr = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
-
-func main() {
-	flag.Parse()
-	buildinfo.Init()
-	logger.Init()
-	logger.Infof("starting VictoraMetrics at %q...", *httpListenAddr)
-	startTime := time.Now()
-	vmstorage.Init()
-	vmselect.Init()
-	vminsert.Init()
-
-	go httpserver.Serve(*httpListenAddr, requestHandler)
-	logger.Infof("started VictoriaMetrics in %s", time.Since(startTime))
-
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	vminsert.Stop()
-	logger.Infof("successfully shut down the webservice in %s", time.Since(startTime))
-
-	vmstorage.Stop()
-	vmselect.Stop()
-
-	logger.Infof("the VictoriaMetrics has been stopped in %s", time.Since(startTime))
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if vminsert.RequestHandler(w, r) {
-		return true
-	}
-	if vmselect.RequestHandler(w, r) {
-		return true
-	}
-	if vmstorage.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}

app/vmagent/Makefile

@@ -0,0 +1,77 @@
+# All these commands must run from repository root.
+
+vmagent:
+	APP_NAME=vmagent $(MAKE) app-local
+
+vmagent-race:
+	APP_NAME=vmagent RACE=-race $(MAKE) app-local
+
+vmagent-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker
+
+vmagent-pure-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-pure
+
+vmagent-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+
+vmagent-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+
+vmagent-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+
+vmagent-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+
+vmagent-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-386
+
+package-vmagent:
+	APP_NAME=vmagent $(MAKE) package-via-docker
+
+package-vmagent-pure:
+	APP_NAME=vmagent $(MAKE) package-via-docker-pure
+
+package-vmagent-amd64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-amd64
+
+package-vmagent-arm:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm
+
+package-vmagent-arm64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm64
+
+package-vmagent-ppc64le:
+	APP_NAME=vmagent $(MAKE) package-via-docker-ppc64le
+
+package-vmagent-386:
+	APP_NAME=vmagent $(MAKE) package-via-docker-386
+
+publish-vmagent:
+	APP_NAME=vmagent $(MAKE) publish-via-docker
+
+run-vmagent:
+	mkdir -p vmagent-remotewrite-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-remotewrite-data:/vmagent-remotewrite-data' \
+	ARGS='-remoteWrite.url=http://localhost:8428/api/v1/write' \
+	APP_NAME=vmagent \
+	$(MAKE) run-via-docker
+
+vmagent-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
+
+vmagent-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
+
+vmagent-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
+
+vmagent-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
+
+vmagent-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
+
+vmagent-pure:
+	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -0,0 +1,263 @@
+## vmagent
+
+`vmagent` is a tiny but brave agent, which helps you collect metrics from various sources
+and stores them in [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports the `remote_write` protocol.
+
+<img alt="vmagent" src="vmagent.png">
+
+
+### Motivation
+
+While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
+and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
+Also, we found that users’ infrastructure are snowflakes - no two are alike, and we decided to add more flexibility
+to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
+
+
+### Features
+
+* Can be used as drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
+  See [Quick Start](#quick-start) for details.
+* Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
+* Accepts data via all the ingestion protocols supported by VictoriaMetrics:
+  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
+  * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
+* Can replicate collected metrics simultaneously to multiple remote storage systems.
+* Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
+  are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
+  to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared to Prometheus.
+
+
+### Quick Start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
+
+* `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+
+Example command line:
+
+```
+/path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+If you only need to collect Influx data, then the following is sufficient:
+
+```
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+
+`vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags).
+
+Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
+
+
+### Use cases
+
+
+#### IoT and Edge monitoring
+
+`vmagent` can run and collect metrics in IoT and industrial networks with unreliable or scheduled connections to the remote storage.
+It buffers the collected data in local files until the connection to remote storage becomes available and then sends the buffered
+data to the remote storage. It re-tries sending the data to remote storage on any errors.
+The maximum buffer size can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+
+`vmagent` works on various architectures from IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
+See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+
+#### Drop-in replacement for Prometheus
+
+If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such a setup.
+See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
+
+
+#### Replication and high availability
+
+`vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
+If a single remote storage instance temporarily is out of service, then the collected data remains available in another remote storage instances.
+`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
+Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
+
+
+#### Relabeling and filtering
+
+`vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
+it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
+See [these docs](#relabeling) for details.
+
+
+#### Splitting data streams among multiple systems
+
+`vmagent` supports splitting the collected data between muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
+data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
+Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
+
+
+#### Prometheus remote_write proxy
+
+`vmagent` may be used as a proxy for Prometheus data sent via Prometheus `remote_write` protocol. It can accept data via `remote_write` API
+at `/api/v1/write` endpoint, apply relabeling and filtering and then proxy it to another `remote_write` systems.
+The `vmagent` can be configured to encrypt the incoming `remote_write` requests with `-tls*` command-line flags.
+Additionally, Basic Auth can be enabled for the incoming `remote_write` requests with `-httpAuth.*` command-line flags.
+
+
+
+### How to collect metrics in Prometheus format
+
+Pass the path to `prometheus.yml` to `-promscrape.config` command-line flag. `vmagent` takes into account the following
+sections from [Prometheus config file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/):
+
+* `global`
+* `scrape_configs`
+
+All the other sections are ignored, including [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section.
+Use `-remoteWrite.*` command-line flags instead for configuring remote write settings.
+
+The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
+
+* `static_configs` - for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
+* `file_sd_configs` - for scraping targets defined in external files aka file-based service discover.
+  See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
+* `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
+  See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazon EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
+* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
+  See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` provides the following additional functionality for `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
+* `consul_sd_configs` - for scraping targets registered in Consul.
+  See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
+* `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
+  See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+
+Note that `vmagent` doesn't support `refresh_interval` option these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
+command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
+entries to 60s. Run `vmagent -help` in order to see default values for `-promscrape.*CheckInterval` flags.
+
+
+File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
+
+
+### Adding labels to metrics
+
+Labels can be added to metrics via the following mechanisms:
+
+* Via `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in `-promscrape.config` file.
+* Via `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+
+
+### Relabeling
+
+`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+Additionally it provides the following extra actions:
+
+* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
+* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+
+The relabeling can be defined in the following places:
+
+* At `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels.
+* At `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`.
+* At `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
+* At `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
+
+Read more about relabeling in the following articles:
+
+* [Life of a label](https://www.robustperception.io/life-of-a-label)
+* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+* [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+* [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
+
+
+### Monitoring
+
+`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
+either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+
+
+### Troubleshooting
+
+* It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
+  since `vmagent` establishes at least a single TCP connection per each target.
+
+* When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+
+* It is recommended to increase `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+  and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+
+* `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
+  The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmagent` from the root folder of the repository.
+   It builds `vmagent` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-prod` from the root folder of the repository.
+   It builds `vmagent-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmagent
+```
+
+
+### Profiling
+
+`vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8429/debug/pprof/heap > mem.pprof
+```
+
+* CPU profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8429/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

app/vmagent/common/push_ctx.go

@@ -0,0 +1,70 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// PushCtx is a context used for populating WriteRequest.
+type PushCtx struct {
+	WriteRequest prompbmarshal.WriteRequest
+
+	// Labels contains flat list of all the labels used in WriteRequest.
+	Labels []prompbmarshal.Label
+
+	// Samples contains flat list of all the samples used in WriteRequest.
+	Samples []prompbmarshal.Sample
+}
+
+// Reset resets ctx.
+func (ctx *PushCtx) Reset() {
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
+
+	labels := ctx.Labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	ctx.Labels = ctx.Labels[:0]
+
+	ctx.Samples = ctx.Samples[:0]
+}
+
+// GetPushCtx returns PushCtx from pool.
+//
+// Call PutPushCtx when the ctx is no longer needed.
+func GetPushCtx() *PushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
+	}
+}
+
+// PutPushCtx returns ctx to the pool.
+//
+// ctx mustn't be used after returning to the pool.
+func PutPushCtx(ctx *PushCtx) {
+	ctx.Reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/csvimport/request_handler.go

@@ -0,0 +1,63 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes csv data from req.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8429
+
+ENTRYPOINT ["/vmagent-prod"]
+ARG src_binary
+COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -0,0 +1,65 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -0,0 +1,167 @@
+package influx
+
+import (
+	"flag"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", insertRows)
+	})
+}
+
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, insertRows)
+	})
+}
+
+func insertRows(db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.ctx.Labels[:0]
+	samples := ctx.ctx.Samples[:0]
+	commonLabels := ctx.commonLabels[:0]
+	buf := ctx.buf[:0]
+	for i := range rows {
+		r := &rows[i]
+		commonLabels = commonLabels[:0]
+		hasDBLabel := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		if len(db) > 0 && !hasDBLabel {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  "db",
+				Value: db,
+			})
+		}
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			bufLen := len(buf)
+			buf = append(buf, ctx.metricGroupBuf...)
+			if !skipFieldKey {
+				buf = append(buf, f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: metricGroup,
+			})
+			labels = append(labels, commonLabels...)
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: r.Timestamp,
+				Value:     f.Value,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+		}
+		rowsTotal += len(r.Fields)
+	}
+	ctx.buf = buf
+	ctx.ctx.WriteRequest.Timeseries = tssDst
+	ctx.ctx.Labels = labels
+	ctx.ctx.Samples = samples
+	ctx.commonLabels = commonLabels
+	remotewrite.Push(&ctx.ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+
+	return nil
+}
+
+type pushCtx struct {
+	ctx            common.PushCtx
+	commonLabels   []prompbmarshal.Label
+	metricGroupBuf []byte
+	buf            []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.ctx.Reset()
+
+	commonLabels := ctx.commonLabels
+	for i := range commonLabels {
+		label := &commonLabels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+	ctx.buf = ctx.buf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/main.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . See also -promscrape.config.dryRun")
+)
+
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	if *dryRun {
+		if err := flag.Set("promscrape.config.strictParse", "true"); err != nil {
+			logger.Panicf("BUG: cannot set promscrape.config.strictParse=true: %s", err)
+		}
+		if err := remotewrite.CheckRelabelConfigs(); err != nil {
+			logger.Fatalf("error when checking relabel configs: %s", err)
+		}
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking Prometheus config: %s", err)
+		}
+		logger.Infof("all the configs are ok; exitting with 0 status code")
+		return
+	}
+
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	startTime := time.Now()
+	remotewrite.Init()
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+
+	promscrape.Init(remotewrite.Push)
+
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, requestHandler)
+	}
+	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	}
+
+	promscrape.Stop()
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+	remotewrite.Stop()
+
+	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	switch path {
+	case "/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/write", "/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
+		influxQueryRequests.Inc()
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
+		return true
+	case "/targets":
+		promscrapeTargetsRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
+		return true
+	case "/-/reload":
+		promscrapeConfigReloadRequests.Inc()
+		procutil.SelfSIGHUP()
+		w.WriteHeader(http.StatusOK)
+		return true
+	}
+	return false
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
+
+	vmimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+
+	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+
+	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+)
+
+func usage() {
+	const s = `
+vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}

app/vmagent/opentsdb/request_handler.go

@@ -0,0 +1,65 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/opentsdbhttp/request_handler.go

@@ -0,0 +1,64 @@
+package opentsdbhttp
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -0,0 +1,67 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		labelsLen := len(labels)
+		for i := range ts.Labels {
+			label := &ts.Labels[i]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
+			})
+		}
+		samplesLen := len(samples)
+		for i := range ts.Samples {
+			sample := &ts.Samples[i]
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     sample.Value,
+				Timestamp: sample.Timestamp,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -0,0 +1,274 @@
+package remotewrite
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/fasthttp"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsCAFile = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+
+	basicAuthUsername = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+)
+
+type client struct {
+	urlLabelValue  string
+	remoteWriteURL string
+	host           string
+	requestURI     string
+	authHeader     string
+	fq             *persistentqueue.FastQueue
+	hc             *fasthttp.HostClient
+
+	requestDuration *metrics.Histogram
+	requestsOKCount *metrics.Counter
+	errorsCount     *metrics.Counter
+	retriesCount    *metrics.Counter
+
+	wg     sync.WaitGroup
+	stopCh chan struct{}
+}
+
+func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authHeader := ""
+	username := basicAuthUsername.GetOptionalArg(argIdx)
+	password := basicAuthPassword.GetOptionalArg(argIdx)
+	if len(username) > 0 || len(password) > 0 {
+		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		token := username + ":" + password
+		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		authHeader = "Basic " + token64
+	}
+	token := bearerToken.GetOptionalArg(argIdx)
+	if len(token) > 0 {
+		if authHeader != "" {
+			logger.Fatalf("`-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", token)
+		}
+		authHeader = "Bearer " + token
+	}
+
+	readTimeout := *sendTimeout
+	if readTimeout <= 0 {
+		readTimeout = time.Minute
+	}
+	writeTimeout := readTimeout
+	var u fasthttp.URI
+	u.Update(remoteWriteURL)
+	scheme := string(u.Scheme())
+	switch scheme {
+	case "http", "https":
+	default:
+		logger.Fatalf("unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
+	}
+	host := string(u.Host())
+	if len(host) == 0 {
+		logger.Fatalf("invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
+	}
+	requestURI := string(u.RequestURI())
+	isTLS := scheme == "https"
+	var tlsCfg *tls.Config
+	if isTLS {
+		var err error
+		tlsCfg, err = getTLSConfig(argIdx)
+		if err != nil {
+			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+		}
+	}
+	if !strings.Contains(host, ":") {
+		if isTLS {
+			host += ":443"
+		} else {
+			host += ":80"
+		}
+	}
+	maxConns := 2 * concurrency
+	hc := &fasthttp.HostClient{
+		Addr:                host,
+		Name:                "vmagent",
+		Dial:                statDial,
+		IsTLS:               isTLS,
+		TLSConfig:           tlsCfg,
+		MaxConns:            maxConns,
+		MaxIdleConnDuration: 10 * readTimeout,
+		ReadTimeout:         readTimeout,
+		WriteTimeout:        writeTimeout,
+		MaxResponseBodySize: 1024 * 1024,
+	}
+	c := &client{
+		urlLabelValue:  urlLabelValue,
+		remoteWriteURL: remoteWriteURL,
+		host:           host,
+		requestURI:     requestURI,
+		authHeader:     authHeader,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
+	}
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	for i := 0; i < concurrency; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
+	}
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	return c
+}
+
+func (c *client) MustStop() {
+	close(c.stopCh)
+	c.wg.Wait()
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+}
+
+func getTLSConfig(argIdx int) (*tls.Config, error) {
+	tlsConfig := &promauth.TLSConfig{
+		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
+		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
+		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
+		ServerName:         tlsServerName.GetOptionalArg(argIdx),
+		InsecureSkipVerify: *tlsInsecureSkipVerify,
+	}
+	cfg, err := promauth.NewConfig(".", nil, "", "", tlsConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot populate TLS config: %s", err)
+	}
+	tlsCfg := cfg.NewTLSConfig()
+	return tlsCfg, nil
+}
+
+func (c *client) runWorker() {
+	var ok bool
+	var block []byte
+	ch := make(chan struct{})
+	for {
+		block, ok = c.fq.MustReadBlock(block[:0])
+		if !ok {
+			return
+		}
+		go func() {
+			c.sendBlock(block)
+			ch <- struct{}{}
+		}()
+		select {
+		case <-ch:
+			// The block has been sent successfully
+			continue
+		case <-c.stopCh:
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
+			select {
+			case <-ch:
+				// The block has been sent successfully.
+			case <-time.After(graceDuration):
+				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
+					len(block), c.remoteWriteURL, graceDuration.Seconds())
+			}
+			return
+		}
+	}
+}
+
+func (c *client) sendBlock(block []byte) {
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(c.requestURI)
+	req.SetHost(c.host)
+	req.Header.SetMethod("POST")
+	req.Header.Add("Content-Type", "application/x-protobuf")
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Add("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.authHeader != "" {
+		req.Header.Set("Authorization", c.authHeader)
+	}
+	req.SetBody(block)
+
+	retryDuration := time.Second
+	resp := fasthttp.AcquireResponse()
+
+again:
+	select {
+	case <-c.stopCh:
+		fasthttp.ReleaseRequest(req)
+		fasthttp.ReleaseResponse(resp)
+		return
+	default:
+	}
+
+	startTime := time.Now()
+	err := doRequestWithPossibleRetry(c.hc, req, resp)
+	c.requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		c.errorsCount.Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	statusCode := resp.StatusCode()
+	if statusCode/100 != 2 {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	c.requestsOKCount.Inc()
+
+	// The block has been successfully sent to the remote storage.
+	fasthttp.ReleaseResponse(resp)
+	fasthttp.ReleaseRequest(req)
+}
+
+func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) error {
+	// There is no need in calling DoTimeout, since the timeout must be already set in hc.ReadTimeout.
+	err := hc.Do(req, resp)
+	if err == nil {
+		return nil
+	}
+	if err != fasthttp.ErrConnectionClosed {
+		return err
+	}
+	// Retry request if the server closed the keep-alive connection during the first attempt.
+	return hc.Do(req, resp)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -0,0 +1,199 @@
+package remotewrite
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
+		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage. "+
+		"Minimum supported interval is 1 second")
+	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+)
+
+// the maximum number of rows to send per each block.
+const maxRowsPerBlock = 10000
+
+type pendingSeries struct {
+	mu sync.Mutex
+	wr writeRequest
+
+	stopCh            chan struct{}
+	periodicFlusherWG sync.WaitGroup
+}
+
+func newPendingSeries(pushBlock func(block []byte)) *pendingSeries {
+	var ps pendingSeries
+	ps.wr.pushBlock = pushBlock
+	ps.stopCh = make(chan struct{})
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
+	return &ps
+}
+
+func (ps *pendingSeries) MustStop() {
+	close(ps.stopCh)
+	ps.periodicFlusherWG.Wait()
+}
+
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+	ps.mu.Lock()
+	ps.wr.push(tss)
+	ps.mu.Unlock()
+}
+
+func (ps *pendingSeries) periodicFlusher() {
+	flushSeconds := int64(flushInterval.Seconds())
+	if flushSeconds <= 0 {
+		flushSeconds = 1
+	}
+	ticker := time.NewTicker(*flushInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-ps.stopCh:
+			mustStop = true
+		case <-ticker.C:
+			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime < uint64(flushSeconds) {
+				continue
+			}
+		}
+		ps.mu.Lock()
+		ps.wr.flush()
+		ps.mu.Unlock()
+	}
+}
+
+type writeRequest struct {
+	wr            prompbmarshal.WriteRequest
+	pushBlock     func(block []byte)
+	lastFlushTime uint64
+
+	tss []prompbmarshal.TimeSeries
+
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
+}
+
+func (wr *writeRequest) reset() {
+	wr.wr.Timeseries = nil
+
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	wr.tss = wr.tss[:0]
+
+	for i := range wr.labels {
+		label := &wr.labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	wr.labels = wr.labels[:0]
+
+	wr.samples = wr.samples[:0]
+	wr.buf = wr.buf[:0]
+}
+
+func (wr *writeRequest) flush() {
+	wr.wr.Timeseries = wr.tss
+	wr.lastFlushTime = fasttime.UnixTimestamp()
+	pushWriteRequest(&wr.wr, wr.pushBlock)
+	wr.reset()
+}
+
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+	tssDst := wr.tss
+	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		dst := &tssDst[len(tssDst)-1]
+		wr.copyTimeSeries(dst, &src[i])
+		if len(wr.tss) >= maxRowsPerBlock {
+			wr.flush()
+			tssDst = wr.tss
+		}
+	}
+	wr.tss = tssDst
+}
+
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
+	labelsLen := len(wr.labels)
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
+
+		buf = append(buf, srcLabel.Name...)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
+		buf = append(buf, srcLabel.Value...)
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
+	}
+	dst.Labels = labelsDst[labelsLen:]
+
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
+}
+
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+	if len(wr.Timeseries) == 0 {
+		// Nothing to push
+		return
+	}
+	bb := writeRequestBufPool.Get()
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	if len(bb.B) <= *maxUnpackedBlockSize {
+		zb := snappyBufPool.Get()
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		writeRequestBufPool.Put(bb)
+		if len(zb.B) <= persistentqueue.MaxBlockSize {
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
+		}
+		snappyBufPool.Put(zb)
+	} else {
+		writeRequestBufPool.Put(bb)
+	}
+
+	// Too big block. Recursively split it into smaller parts.
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries
+}
+
+var (
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+)
+
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/relabel.go

@@ -0,0 +1,139 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+var (
+	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+)
+
+var labelsGlobal []prompbmarshal.Label
+
+// CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
+func CheckRelabelConfigs() error {
+	_, err := loadRelabelConfigs()
+	return err
+}
+
+func loadRelabelConfigs() (*relabelConfigs, error) {
+	var rcs relabelConfigs
+	if *relabelConfigPathGlobal != "" {
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %s", *relabelConfigPathGlobal, err)
+		}
+		rcs.global = global
+	}
+	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
+			len(*relabelConfigPaths), len(*remoteWriteURLs))
+	}
+	rcs.perURL = make([][]promrelabel.ParsedRelabelConfig, len(*remoteWriteURLs))
+	for i, path := range *relabelConfigPaths {
+		prc, err := promrelabel.LoadRelabelConfigs(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %s", path, err)
+		}
+		rcs.perURL[i] = prc
+	}
+	return &rcs, nil
+}
+
+type relabelConfigs struct {
+	global []promrelabel.ParsedRelabelConfig
+	perURL [][]promrelabel.ParsedRelabelConfig
+}
+
+// initLabelsGlobal must be called after parsing command-line flags.
+func initLabelsGlobal() {
+	// Init labelsGlobal
+	labelsGlobal = nil
+	for _, s := range *unparsedLabelsGlobal {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
+			Name:  s[:n],
+			Value: s[n+1:],
+		})
+	}
+}
+
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && len(prcs) == 0 {
+		// Nothing to change.
+		return tss
+	}
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
+		if len(labels) == labelsLen {
+			// Drop the current time series, since relabeling removed all the labels.
+			continue
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	return tssDst
+}
+
+type relabelCtx struct {
+	// pool for labels, which are used during the relabeling.
+	labels []prompbmarshal.Label
+}
+
+func (rctx *relabelCtx) reset() {
+	labels := rctx.labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	rctx.labels = rctx.labels[:0]
+}
+
+var relabelCtxPool = &sync.Pool{
+	New: func() interface{} {
+		return &relabelCtx{}
+	},
+}
+
+func getRelabelCtx() *relabelCtx {
+	return relabelCtxPool.Get().(*relabelCtx)
+}
+
+func putRelabelCtx(rctx *relabelCtx) {
+	rctx.labels = rctx.labels[:0]
+	relabelCtxPool.Put(rctx)
+}

app/vmagent/remotewrite/remotewrite.go

@@ -0,0 +1,232 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var (
+	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
+	queues      = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+		"isn't enough for sending high volume of collected data to remote storage")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensistive auth info")
+	maxPendingBytesPerURL = flag.Int("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
+		"Disk usage is unlimited if the value is set to 0")
+)
+
+var rwctxs []*remoteWriteCtx
+
+// Contains the current relabelConfigs.
+var allRelabelConfigs atomic.Value
+
+// Init initializes remotewrite.
+//
+// It must be called after flag.Parse().
+//
+// Stop must be called for graceful shutdown.
+func Init() {
+	if len(*remoteWriteURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` must be set")
+	}
+
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		httpserver.RegisterSecretFlag("remoteWrite.url")
+	}
+	initLabelsGlobal()
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot load relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
+	if maxInmemoryBlocks > 200 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 200
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	for i, remoteWriteURL := range *remoteWriteURLs {
+		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		if *showRemoteWriteURL {
+			urlLabelValue = remoteWriteURL
+		}
+		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, urlLabelValue)
+		rwctxs = append(rwctxs, rwctx)
+	}
+
+	// Start config reloader.
+	sighupCh := procutil.NewSighupChan()
+	configReloaderWG.Add(1)
+	go func() {
+		defer configReloaderWG.Done()
+		for {
+			select {
+			case <-sighupCh:
+			case <-stopCh:
+				return
+			}
+			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+			rcs, err := loadRelabelConfigs()
+			if err != nil {
+				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+				continue
+			}
+			allRelabelConfigs.Store(rcs)
+			logger.Infof("Successfully reloaded relabel configs")
+		}
+	}()
+}
+
+var stopCh = make(chan struct{})
+var configReloaderWG sync.WaitGroup
+
+// Stop stops remotewrite.
+//
+// It is expected that nobody calls Push during and after the call to this func.
+func Stop() {
+	close(stopCh)
+	configReloaderWG.Wait()
+
+	for _, rwctx := range rwctxs {
+		rwctx.MustStop()
+	}
+	rwctxs = nil
+}
+
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+//
+// Note that wr may be modified by Push due to relabeling.
+func Push(wr *prompbmarshal.WriteRequest) {
+	var rctx *relabelCtx
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	prcsGlobal := rcs.global
+	if len(prcsGlobal) > 0 || len(labelsGlobal) > 0 {
+		rctx = getRelabelCtx()
+	}
+	tss := wr.Timeseries
+	for len(tss) > 0 {
+		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		tssBlock := tss
+		if len(tssBlock) > maxRowsPerBlock {
+			tssBlock = tss[:maxRowsPerBlock]
+			tss = tss[maxRowsPerBlock:]
+		} else {
+			tss = nil
+		}
+		if rctx != nil {
+			tssBlockLen := len(tssBlock)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, prcsGlobal)
+			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+		}
+		for _, rwctx := range rwctxs {
+			rwctx.Push(tssBlock)
+		}
+		if rctx != nil {
+			rctx.reset()
+		}
+	}
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}
+
+var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+
+type remoteWriteCtx struct {
+	idx        int
+	fq         *persistentqueue.FastQueue
+	c          *client
+	pss        []*pendingSeries
+	pssNextIdx uint64
+
+	tss []prompbmarshal.TimeSeries
+
+	relabelMetricsDropped *metrics.Counter
+}
+
+func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+	h := xxhash.Sum64([]byte(remoteWriteURL))
+	path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
+	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, *maxPendingBytesPerURL)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetPendingBytes())
+	})
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetInmemoryQueueLen())
+	})
+	c := newClient(argIdx, remoteWriteURL, urlLabelValue, fq, *queues)
+	pss := make([]*pendingSeries, *queues)
+	for i := range pss {
+		pss[i] = newPendingSeries(fq.MustWriteBlock)
+	}
+	return &remoteWriteCtx{
+		idx: argIdx,
+		fq:  fq,
+		c:   c,
+		pss: pss,
+
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+	}
+}
+
+func (rwctx *remoteWriteCtx) MustStop() {
+	for _, ps := range rwctx.pss {
+		ps.MustStop()
+	}
+	rwctx.idx = 0
+	rwctx.pss = nil
+	rwctx.fq.MustClose()
+	rwctx.fq = nil
+	rwctx.c.MustStop()
+	rwctx.c = nil
+
+	rwctx.relabelMetricsDropped = nil
+}
+
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+	var rctx *relabelCtx
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	prcs := rcs.perURL[rwctx.idx]
+	if len(prcs) > 0 {
+		// Make a copy of tss before applying relabeling in order to prevent
+		// from affecting time series for other remoteWrite.url configs.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467 for details.
+		rwctx.tss = append(rwctx.tss[:0], tss...)
+		tss = rwctx.tss
+		rctx = getRelabelCtx()
+		tssLen := len(tss)
+		tss = rctx.applyRelabeling(tss, nil, prcs)
+		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+	}
+	pss := rwctx.pss
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
+	if rctx != nil {
+		putRelabelCtx(rctx)
+		// Zero rwctx.tss in order to free up GC references.
+		rwctx.tss = prompbmarshal.ResetTimeSeries(rwctx.tss)
+	}
+}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,76 @@
+package remotewrite
+
+import (
+	"net"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/fasthttp"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func statDial(addr string) (conn net.Conn, err error) {
+	if netutil.TCP6Enabled() {
+		conn, err = fasthttp.DialDualStack(addr)
+	} else {
+		conn, err = fasthttp.Dial(addr)
+	}
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/vmimport/request_handler.go

@@ -0,0 +1,70 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		samplesLen := len(samples)
+		for j, value := range values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     value,
+				Timestamp: timestamps[j],
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(values)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/Makefile

@@ -0,0 +1,84 @@
+# All these commands must run from repository root.
+
+vmalert:
+	APP_NAME=vmalert $(MAKE) app-local
+
+vmalert-race:
+	APP_NAME=vmalert RACE=-race $(MAKE) app-local
+
+vmalert-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker
+
+vmalert-pure-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-pure
+
+vmalert-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+
+vmalert-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+
+vmalert-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+
+vmalert-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+
+vmalert-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-386
+
+package-vmalert:
+	APP_NAME=vmalert $(MAKE) package-via-docker
+
+package-vmalert-pure:
+	APP_NAME=vmalert $(MAKE) package-via-docker-pure
+
+package-vmalert-amd64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-amd64
+
+package-vmalert-arm:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm
+
+package-vmalert-arm64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm64
+
+package-vmalert-ppc64le:
+	APP_NAME=vmalert $(MAKE) package-via-docker-ppc64le
+
+package-vmalert-386:
+	APP_NAME=vmalert $(MAKE) package-via-docker-386
+
+publish-vmalert:
+	APP_NAME=vmalert $(MAKE) publish-via-docker
+
+test-vmalert:
+	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/datasource
+	go test -v -race -cover ./app/vmalert/notifier
+	go test -v -race -cover ./app/vmalert/config
+
+run-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-notifier.url=http://localhost:9093 \
+		-remoteWrite.url=http://localhost:8428 \
+		-remoteRead.url=http://localhost:8428 \
+		-evaluationInterval=3s
+
+vmalert-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-amd64 ./app/vmalert
+
+vmalert-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm ./app/vmalert
+
+vmalert-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm64 ./app/vmalert
+
+vmalert-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-ppc64le ./app/vmalert
+
+vmalert-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-386 ./app/vmalert
+
+vmalert-pure:
+	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/README.md

@@ -0,0 +1,235 @@
+## vmalert
+
+`vmalert` executes a list of given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
+or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
+rules against configured address.
+
+### Features:
+* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
+* VictoriaMetrics [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+ support and expressions validation;
+* Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
+ support;
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Lightweight without extra dependencies.
+
+### Limitations:
+* `vmalert` execute queries against remote datasource which has reliability risks because of network. 
+It is recommended to configure alerts thresholds and rules expressions with understanding that network request
+may fail;
+* by default, rules execution is sequential within one group, but persisting of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
+recording rule is reused in next one;
+* there is no `query` function support in templates yet;
+* `vmalert` has no UI, just an API for getting groups and rules statuses.
+
+### QuickStart
+
+To build `vmalert` from sources:
+```
+git clone https://github.com/VictoriaMetrics/VictoriaMetrics
+cd VictoriaMetrics
+make vmalert
+```
+The build binary will be placed to `VictoriaMetrics/bin` folder.
+
+To start using `vmalert` you will need the following things:
+* list of rules - PromQL/MetricsQL expressions to execute;
+* datasource address - reachable VictoriaMetrics instance for rules execution;
+* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing, 
+aggregating alerts and sending notifications.
+* remote write address - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+compatible storage address for storing recording rules results and alerts state in for of timeseries. This is optional.
+
+Then configure `vmalert` accordingly:
+```
+./bin/vmalert -rule=alert.rules \
+		-datasource.url=http://localhost:8428 \
+        -notifier.url=http://localhost:9093
+```
+
+Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) 
+and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very 
+similar to Prometheus rules and configured using YAML. Configuration examples may be found 
+in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
+Every `rule` belongs to `group` and every configuration file may contain arbitrary number of groups:
+```yaml
+groups:
+  [ - <rule_group> ]
+```
+
+#### Groups
+
+Each group has following attributes:
+```yaml
+# The name of the group. Must be unique within a file.
+name: <string>
+
+# How often rules in the group are evaluated.
+[ interval: <duration> | default = global.evaluation_interval ]
+
+# How many rules execute at once. Increasing concurrency may speed
+# up round execution speed. 
+[ concurrency: <integer> | default = 1 ]
+
+rules:
+  [ - <rule> ... ]
+```
+
+#### Rules
+
+There are two types of Rules:
+* [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - 
+Alerting rules allows to define alert conditions via [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+and to send notifications about firing alerts to [Alertmanager](https://github.com/prometheus/alertmanager).
+* [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) - 
+Recording rules allow you to precompute frequently needed or computationally expensive expressions 
+and save their result as a new set of time series.
+
+`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+within one group. 
+
+##### Alerting rules
+
+The syntax for alerting rule is following:
+```yaml
+# The name of the alert. Must be a valid metric name.
+alert: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Alerts are considered firing once they have been returned for this long.
+# Alerts which have not yet fired for long enough are considered pending.
+[ for: <duration> | default = 0s ]
+
+# Labels to add or overwrite for each alert.
+labels:
+  [ <labelname>: <tmpl_string> ]
+
+# Annotations to add to each alert.
+annotations:
+  [ <labelname>: <tmpl_string> ]
+``` 
+
+`vmalert` has no local storage and alerts state is stored in process memory. Hence, after reloading of `vmalert` process
+alerts state will be lost. To avoid this situation, `vmalert` may be configured via following flags:
+* `-remoteWrite.url` - URL to Victoria Metrics or VMInsert. `vmalert` will persist alerts state into the configured
+address in form of timeseries with name `ALERTS` via remote-write protocol.
+* `-remoteRead.url` - URL to Victoria Metrics or VMSelect. `vmalert` will try to restore alerts state from configured
+address by querying `ALERTS` timeseries.
+
+
+##### Recording rules
+
+The syntax for recording rules is following:
+```yaml
+# The name of the time series to output to. Must be a valid metric name.
+record: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Labels to add or overwrite before storing the result.
+labels:
+  [ <labelname>: <labelvalue> ]
+```
+
+For recording rules to work `-remoteWrite.url` must specified.
+
+
+#### WEB
+
+`vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
+* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
+* `http://<vmalert-addr>/api/v1/<groupName>/<alertID>/status" ` - get alert status by ID.
+Used as alert source in AlertManager.
+* `http://<vmalert-addr>/metrics` - application metrics.
+* `http://<vmalert-addr>/-/reload` - hot configuration reload.
+
+
+### Configuration
+
+The shortlist of configuration flags is the following:
+```
+Usage of vmalert:
+  -datasource.basicAuth.password string
+        Optional basic auth password for -datasource.url
+  -datasource.basicAuth.username string
+        Optional basic auth username for -datasource.url
+  -datasource.url string
+        Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
+  -evaluationInterval duration
+        How often to evaluate the rules (default 1m0s)
+  -external.url string
+        External URL is used as alert's source for sent alerts to the notifier
+  -httpListenAddr string
+        Address to listen for http connections (default ":8880")
+  -metricsAuthKey string
+        Auth key for /metrics. It overrides httpAuth settings
+  -notifier.url string
+        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+  -remoteRead.basicAuth.password string
+        Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.username string
+        Optional basic auth username for -remoteRead.url
+  -remoteRead.lookback duration
+        Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteRead.url vmalert
+        Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+  -remoteWrite.basicAuth.password string
+        Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.username string
+        Optional basic auth username for -remoteWrite.url
+  -remoteWrite.concurrency int
+        Defines number of readers that concurrently write into remote storage (default 1)
+  -remoteWrite.maxBatchSize int
+        Defines defines max number of timeseries to be flushed at once (default 1000)
+  -remoteWrite.maxQueueSize int
+        Defines the max number of pending datapoints to remote write endpoint (default 100000)
+  -remoteWrite.url string
+        Optional URL to Victoria Metrics or VMInsert where to persist alerts state in form of timeseries. E.g. http://127.0.0.1:8428
+  -rule value
+        Path to the file with alert rules. 
+        Supports patterns. Flag can be specified multiple times. 
+        Examples:
+         -rule /path/to/file. Path to a single file with alerting rules
+         -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+        absolute path to all .yaml files in root.
+  -rule.validateExpressions
+        Whether to validate rules expressions via MetricsQL engine (default true)
+  -rule.validateTemplates
+        Whether to validate annotation and label templates (default true)
+```
+
+Pass `-help` to `vmalert` in order to see the full list of supported 
+command-line flags with their descriptions.
+
+To reload configuration without `vmalert` restart send SIGHUP signal
+or send GET request to `/-/reload` endpoint.
+
+### Contributing
+
+`vmalert` is mostly designed and built by VictoriaMetrics community.
+Feel free to share your experience and ideas for improving this 
+software. Please keep simplicity as the main priority.
+
+### How to build from sources
+
+It is recommended using 
+[binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) 
+- `vmalert` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmalert` from the root folder of the repository.
+   It builds `vmalert` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmalert-prod` from the root folder of the repository.
+   It builds `vmalert-prod` binary and puts it into the `bin` folder.

app/vmalert/alerting.go

@@ -0,0 +1,374 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// AlertingRule is basic alert entity
+type AlertingRule struct {
+	RuleID      uint64
+	Name        string
+	Expr        string
+	For         time.Duration
+	Labels      map[string]string
+	Annotations map[string]string
+	GroupID     uint64
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+}
+
+func newAlertingRule(gID uint64, cfg config.Rule) *AlertingRule {
+	return &AlertingRule{
+		RuleID:      cfg.ID,
+		Name:        cfg.Alert,
+		Expr:        cfg.Expr,
+		For:         cfg.For,
+		Labels:      cfg.Labels,
+		Annotations: cfg.Annotations,
+		GroupID:     gID,
+		alerts:      make(map[uint64]*notifier.Alert),
+	}
+}
+
+// String implements Stringer interface
+func (ar *AlertingRule) String() string {
+	return ar.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (ar *AlertingRule) ID() uint64 {
+	return ar.RuleID
+}
+
+// Exec executes AlertingRule expression via the given Querier.
+// Based on the Querier results AlertingRule maintains notifier.Alerts
+func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := q.Query(ctx, ar.Expr)
+	ar.mu.Lock()
+	defer ar.mu.Unlock()
+
+	ar.lastExecError = err
+	ar.lastExecTime = time.Now()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %s", ar.Expr, err)
+	}
+
+	for h, a := range ar.alerts {
+		// cleanup inactive alerts from previous Exec
+		if a.State == notifier.StateInactive {
+			delete(ar.alerts, h)
+		}
+	}
+
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		h := hash(m)
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
+			if a.Value != m.Value {
+				// update Value field with latest value
+				a.Value = m.Value
+				// and re-exec template since Value can be used
+				// in templates
+				err = ar.template(a)
+				if err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+		a, err := ar.newAlert(m, ar.lastExecTime)
+		if err != nil {
+			ar.lastExecError = err
+			return nil, fmt.Errorf("failed to create alert: %s", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		ar.alerts[h] = a
+	}
+
+	for h, a := range ar.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(ar.alerts, h)
+				continue
+			}
+			a.State = notifier.StateInactive
+			continue
+		}
+		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+			a.State = notifier.StateFiring
+			alertsFired.Inc()
+		}
+	}
+	if series {
+		return ar.toTimeSeries(ar.lastExecTime), nil
+	}
+	return nil, nil
+}
+
+func (ar *AlertingRule) toTimeSeries(timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		ts := ar.alertToTimeSeries(a, timestamp)
+		tss = append(tss, ts...)
+	}
+	return tss
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(m datasource.Metric) uint64 {
+	hash := fnv.New64a()
+	labels := m.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifier.Alert, error) {
+	a := &notifier.Alert{
+		GroupID: ar.GroupID,
+		Name:    ar.Name,
+		Labels:  map[string]string{},
+		Value:   m.Value,
+		Start:   start,
+		Expr:    ar.Expr,
+	}
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		a.Labels[l.Name] = l.Value
+	}
+	return a, ar.template(a)
+}
+
+func (ar *AlertingRule) template(a *notifier.Alert) error {
+	// 1. template rule labels with data labels
+	rLabels, err := a.ExecTemplate(ar.Labels)
+	if err != nil {
+		return err
+	}
+
+	// 2. merge data labels and rule labels
+	// metric labels may be overridden by
+	// rule labels
+	for k, v := range rLabels {
+		a.Labels[k] = v
+	}
+
+	// 3. template merged labels
+	a.Labels, err = a.ExecTemplate(a.Labels)
+	if err != nil {
+		return err
+	}
+
+	a.Annotations, err = a.ExecTemplate(ar.Annotations)
+	return err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.mu.RLock()
+	defer ar.mu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// RuleAPI returns Rule representation in form
+// of APIAlertingRule
+func (ar *AlertingRule) RuleAPI() APIAlertingRule {
+	var lastErr string
+	if ar.lastExecError != nil {
+		lastErr = ar.lastExecError.Error()
+	}
+	return APIAlertingRule{
+		// encode as strings to avoid rounding
+		ID:          fmt.Sprintf("%d", ar.ID()),
+		GroupID:     fmt.Sprintf("%d", ar.GroupID),
+		Name:        ar.Name,
+		Expression:  ar.Expr,
+		For:         ar.For.String(),
+		LastError:   lastErr,
+		LastExec:    ar.lastExecTime,
+		Labels:      ar.Labels,
+		Annotations: ar.Annotations,
+	}
+}
+
+// AlertsAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.mu.RLock()
+	for _, a := range ar.alerts {
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.mu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	return &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.Start,
+		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+	}
+}
+
+const (
+	// AlertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// AlertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// AlertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+)
+
+// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores only Start field. Field State will be always Pending and supposed
+// to be updated on next Exec, as well as Value field.
+// Only rules with For > 0 will be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+	if q == nil {
+		return fmt.Errorf("querier is nil")
+	}
+	// Get the last datapoint in range via MetricsQL `last_over_time`.
+	// We don't use plain PromQL since Prometheus doesn't support
+	// remote write protocol which is used for state persistence in vmalert.
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q}[%ds])",
+		alertForStateMetricName, ar.Name, int(lookback.Seconds()))
+	qMetrics, err := q.Query(ctx, expr)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range qMetrics {
+		labels := m.Labels
+		m.Labels = make([]datasource.Label, 0)
+		// drop all extra labels, so hash key will
+		// be identical to timeseries received in Exec
+		for _, l := range labels {
+			if l.Name == alertNameLabel {
+				continue
+			}
+			// drop all overridden labels
+			if _, ok := ar.Labels[l.Name]; ok {
+				continue
+			}
+			m.Labels = append(m.Labels, l)
+		}
+
+		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0))
+		if err != nil {
+			return fmt.Errorf("failed to create alert: %s", err)
+		}
+		a.ID = hash(m)
+		a.State = notifier.StatePending
+		ar.alerts[a.ID] = a
+		logger.Infof("alert %q(%d) restored to state at %v", a.Name, a.ID, a.Start)
+	}
+	return nil
+}

app/vmalert/alerting_test.go

@@ -0,0 +1,455 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestAlertingRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *AlertingRule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestAlertingRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant extra labels",
+					"job":           "foo",
+					"instance":      "bar",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant labels override",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "for",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+					alertNameLabel:  "for pending",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for pending",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tc.rule.alerts[tc.alert.ID] = tc.alert
+			tss := tc.rule.toTimeSeries(timestamp)
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Exec(t *testing.T) {
+	const defaultStep = 5 * time.Millisecond
+	testCases := []struct {
+		rule      *AlertingRule
+		steps     [][]datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[][]datasource.Metric{},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[][]datasource.Metric{
+				{datasource.Metric{}},
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "name", "foo"),
+					metricWithLabels(t, "name", "foo1"),
+					metricWithLabels(t, "name", "foo2"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo1")},
+				{metricWithLabels(t, "name", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive, delete first one
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("duplicate", 0),
+			[][]datasource.Metric{
+				{
+					// metrics with the same labelset should result in one alert
+					metricWithLabels(t, "name", "foo", "type", "bar"),
+					metricWithLabels(t, "type", "bar", "name", "foo"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestAlertingRule("for-fired", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>empty", time.Second),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset and delete pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(step...)
+				if _, err := tc.rule.Exec(context.TODO(), fq, false); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Restore(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRuleWithLabels("no extra labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("metric labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("rule labels", "source", "vm"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+					// following pair supposed to be dropped
+					"source", "vm",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("multiple alerts"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-1",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-2",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-3",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(2 * time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(3 * time.Hour)},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			fq.add(tc.metrics...)
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+				if got.Start != exp.Start {
+					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				}
+			}
+		})
+	}
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
+	r := newTestAlertingRule(name, 0)
+	r.Labels = make(map[string]string)
+	for i := 0; i < len(labels); i += 2 {
+		r.Labels[labels[i]] = labels[i+1]
+	}
+	return r
+}
+
+func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
+	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
+}

app/vmalert/config/config.go

@@ -0,0 +1,195 @@
+package config
+
+import (
+	"fmt"
+	"hash/fnv"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/metricsql"
+	"gopkg.in/yaml.v2"
+)
+
+// Group contains list of Rules grouped into
+// entity with one name and evaluation interval
+type Group struct {
+	File        string
+	Name        string        `yaml:"name"`
+	Interval    time.Duration `yaml:"interval,omitempty"`
+	Rules       []Rule        `yaml:"rules"`
+	Concurrency int           `yaml:"concurrency"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// Validate check for internal Group or Rule configuration errors
+func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
+	if g.Name == "" {
+		return fmt.Errorf("group name must be set")
+	}
+	if len(g.Rules) == 0 {
+		return fmt.Errorf("group %q can't contain no rules", g.Name)
+	}
+	uniqueRules := map[uint64]struct{}{}
+	for _, r := range g.Rules {
+		ruleName := r.Record
+		if r.Alert != "" {
+			ruleName = r.Alert
+		}
+		if _, ok := uniqueRules[r.ID]; ok {
+			return fmt.Errorf("rule %q duplicate", ruleName)
+		}
+		uniqueRules[r.ID] = struct{}{}
+		if err := r.Validate(); err != nil {
+			return fmt.Errorf("invalid rule %q.%q: %s", g.Name, ruleName, err)
+		}
+		if validateExpressions {
+			if _, err := metricsql.Parse(r.Expr); err != nil {
+				return fmt.Errorf("invalid expression for rule %q.%q: %s", g.Name, ruleName, err)
+			}
+		}
+		if validateAnnotations {
+			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+				return fmt.Errorf("invalid annotations for rule %q.%q: %s", g.Name, ruleName, err)
+			}
+			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+				return fmt.Errorf("invalid labels for rule %q.%q: %s", g.Name, ruleName, err)
+			}
+		}
+	}
+	return checkOverflow(g.XXX, fmt.Sprintf("group %q", g.Name))
+}
+
+// Rule describes entity that represent either
+// recording rule or alerting rule.
+type Rule struct {
+	ID          uint64
+	Record      string            `yaml:"record,omitempty"`
+	Alert       string            `yaml:"alert,omitempty"`
+	Expr        string            `yaml:"expr"`
+	For         time.Duration     `yaml:"for,omitempty"`
+	Labels      map[string]string `yaml:"labels,omitempty"`
+	Annotations map[string]string `yaml:"annotations,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rule Rule
+	if err := unmarshal((*rule)(r)); err != nil {
+		return err
+	}
+	r.ID = HashRule(*r)
+	return nil
+}
+
+// HashRule hashes significant Rule fields into
+// unique hash value
+func HashRule(r Rule) uint64 {
+	h := fnv.New64a()
+	h.Write([]byte(r.Expr))
+	if r.Record != "" {
+		h.Write([]byte("recording"))
+		h.Write([]byte(r.Record))
+	} else {
+		h.Write([]byte("alerting"))
+		h.Write([]byte(r.Alert))
+	}
+	type item struct {
+		key, value string
+	}
+	var kv []item
+	for k, v := range r.Labels {
+		kv = append(kv, item{key: k, value: v})
+	}
+	sort.Slice(kv, func(i, j int) bool {
+		return kv[i].key < kv[j].key
+	})
+	for _, i := range kv {
+		h.Write([]byte(i.key))
+		h.Write([]byte(i.value))
+		h.Write([]byte("\xff"))
+	}
+	return h.Sum64()
+}
+
+// Validate check for Rule configuration errors
+func (r *Rule) Validate() error {
+	if (r.Record == "" && r.Alert == "") || (r.Record != "" && r.Alert != "") {
+		return fmt.Errorf("either `record` or `alert` must be set")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("expression can't be empty")
+	}
+	return checkOverflow(r.XXX, "rule")
+}
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file pattern %s: %v", pattern, err)
+		}
+		fp = append(fp, matches...)
+	}
+	var groups []Group
+	for _, file := range fp {
+		uniqueGroups := map[string]struct{}{}
+		gr, err := parseFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse file %q: %w", file, err)
+		}
+		for _, g := range gr {
+			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
+				return nil, fmt.Errorf("invalid group %q in file %q: %s", g.Name, file, err)
+			}
+			if _, ok := uniqueGroups[g.Name]; ok {
+				return nil, fmt.Errorf("group name %q duplicate in file %q", g.Name, file)
+			}
+			uniqueGroups[g.Name] = struct{}{}
+			g.File = file
+			groups = append(groups, g)
+		}
+	}
+	if len(groups) < 1 {
+		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parseFile(path string) ([]Group, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
+	}
+	g := struct {
+		Groups []Group `yaml:"groups"`
+		// Catches all undefined fields and must be empty after parsing.
+		XXX map[string]interface{} `yaml:",inline"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	if err != nil {
+		return nil, err
+	}
+	return g.Groups, checkOverflow(g.XXX, "config")
+}
+
+func checkOverflow(m map[string]interface{}, ctx string) error {
+	if len(m) > 0 {
+		var keys []string
+		for k := range m {
+			keys = append(keys, k)
+		}
+		return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", "))
+	}
+	return nil
+}

app/vmalert/config/config_test.go

@@ -0,0 +1,326 @@
+package config
+
+import (
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"value\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/*.yaml"},
+			"no groups found",
+		},
+	}
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, true, true)
+		if err == nil {
+			t.Errorf("expected to get error")
+			return
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestRule_Validate(t *testing.T) {
+	if err := (&Rule{}).Validate(); err == nil {
+		t.Errorf("exptected empty name error")
+	}
+	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
+		t.Errorf("exptected empty expr error")
+	}
+	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("exptected valid rule; got %s", err)
+	}
+}
+
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group:  &Group{Name: "test"},
+			expErr: "contain no rules",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr:              "error parsing annotation",
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+	}
+	for _, tc := range testCases {
+		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: time.Minute},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
+}

app/vmalert/config/testdata/dir/rules0-bad.rules

@@ -0,0 +1,19 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidAnnotations
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }"
+          description: "{{$labels}}"
+      - alert: UnkownAnnotationsFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ value|query }}"
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules0-good.rules

@@ -0,0 +1,14 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          expr: "{{ $expr|queryEscape }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+

app/vmalert/config/testdata/dir/rules1-bad.rules

@@ -0,0 +1,22 @@
+groups:
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules2-bad.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: group
+    rules:
+      - alert: UnkownLabelFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          summary: "{{ value|query }}"
+        annotations:
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules3-bad.rules

@@ -0,0 +1,5 @@
+groups:
+  - name: group
+    rules:
+      - for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules4-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        record: record
+        for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules5-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        expr: vm_rows > 0
+      - record: rows
+        expr: sum(vm_rows)

app/vmalert/config/testdata/rules0-bad.rules

@@ -0,0 +1,28 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidExpr
+        for: 5m
+        expr: vm_rows{ > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: EmptyExpr
+        for: 5m
+        expr: ""
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: ""
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: foo
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/rules0-good.rules

@@ -0,0 +1,23 @@
+groups:
+  - name: groupGorSingleAlert
+    rules:
+      - alert: VMRows
+        for: 10s
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+  - name: TestGroup
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)

app/vmalert/config/testdata/rules1-good.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: groupTest
+    rules:
+      - alert: VMRows
+        for: 1ms
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/config/testdata/rules2-good.rules

@@ -0,0 +1,35 @@
+groups:
+  - name: TestGroup
+    interval: 2s
+    concurrency: 2
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        for: 3m
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)
+      - record: handler:requests:rate5m
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+        labels:
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: dev
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: staging
+          recording: true
+      - record: successful_requests:ratio_rate5m
+        labels:
+          recording: true
+        expr: |2
+            sum(code:requests:rate5m{code="200"})
+          /
+            sum(code:requests:rate5m)

app/vmalert/datasource/datasource.go

@@ -0,0 +1,24 @@
+package datasource
+
+import "context"
+
+// Querier interface wraps Query method which
+// executes given query and returns list of Metrics
+// as result
+type Querier interface {
+	Query(ctx context.Context, query string) ([]Metric, error)
+}
+
+// Metric is the basic entity which should be return by datasource
+// It represents single data point with full list of labels
+type Metric struct {
+	Labels    []Label
+	Timestamp int64
+	Value     float64
+}
+
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}

app/vmalert/datasource/vm.go

@@ -0,0 +1,103 @@
+package datasource
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+type response struct {
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string `json:"resultType"`
+		Result     []struct {
+			Labels map[string]string `json:"metric"`
+			TV     [2]interface{}    `json:"value"`
+		} `json:"result"`
+	} `json:"data"`
+	ErrorType string `json:"errorType"`
+	Error     string `json:"error"`
+}
+
+func (r response) metrics() ([]Metric, error) {
+	var ms []Metric
+	var m Metric
+	var f float64
+	var err error
+	for i, res := range r.Data.Result {
+		f, err = strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %s", res, res.TV[1], err)
+		}
+		m.Labels = nil
+		for k, v := range r.Data.Result[i].Labels {
+			m.Labels = append(m.Labels, Label{Name: k, Value: v})
+		}
+		m.Timestamp = int64(res.TV[0].(float64))
+		m.Value = f
+		ms = append(ms, m)
+	}
+	return ms, nil
+}
+
+const queryPath = "/api/v1/query?query="
+
+// VMStorage represents vmstorage entity with ability to read and write metrics
+type VMStorage struct {
+	c                            *http.Client
+	queryURL                     string
+	basicAuthUser, basicAuthPass string
+}
+
+// NewVMStorage is a constructor for VMStorage
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+	return &VMStorage{
+		c:             c,
+		basicAuthUser: basicAuthUser,
+		basicAuthPass: basicAuthPass,
+		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+	}
+}
+
+// Query reads metrics from datasource by given query
+func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+	const (
+		statusSuccess, statusError, rtVector = "success", "error", "vector"
+	)
+	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if s.basicAuthPass != "" {
+		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	}
+	resp, err := s.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("error getting response from %s:%s", req.URL, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return nil, fmt.Errorf("datasource returns unxeprected response code %d for %s with err %s. Reponse body %s", resp.StatusCode, req.URL, err, body)
+	}
+	r := &response{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing metrics for %s:%s", req.URL, err)
+	}
+	if r.Status == statusError {
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+	}
+	if r.Status != statusSuccess {
+		return nil, fmt.Errorf("unkown status:%s, Expected success or error ", r.Status)
+	}
+	if r.Data.ResultType != rtVector {
+		return nil, fmt.Errorf("unkown restul type:%s. Expected vector", r.Data.ResultType)
+	}
+	return r.metrics()
+}

app/vmalert/datasource/vm_test.go

@@ -0,0 +1,93 @@
+package datasource
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var (
+	ctx           = context.Background()
+	basicAuthName = "foo"
+	basicAuthPass = "bar"
+	query         = "vm_rows"
+)
+
+func TestVMSelectQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("exptected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			w.Write([]byte("[]"))
+		case 3:
+			w.Write([]byte(`{"status":"error", "errorType":"type:", "error":"some error msg"}`))
+		case 4:
+			w.Write([]byte(`{"status":"unknown"}`))
+		case 5:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
+		case 6:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]}]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected connection error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected invalid response status error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected response body error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected error status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected unkown status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected non-vector resultType error  got nil")
+	}
+	m, err := am.Query(ctx, query)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("exptected 1 metric  got %d in %+v", len(m), m)
+	}
+	expected := Metric{
+		Labels:    []Label{{Value: "vm_rows", Name: "__name__"}},
+		Timestamp: 1583786142,
+		Value:     13763,
+	}
+	if m[0].Timestamp != expected.Timestamp &&
+		m[0].Value != expected.Value &&
+		m[0].Labels[0].Value != expected.Labels[0].Value &&
+		m[0].Labels[0].Name != expected.Labels[0].Name {
+		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+	}
+
+}

app/vmalert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-prod"]
+ARG src_binary
+COPY $src_binary ./vmalert-prod

app/vmalert/group.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// Group is an entity for grouping rules
+type Group struct {
+	mu          sync.RWMutex
+	Name        string
+	File        string
+	Rules       []Rule
+	Interval    time.Duration
+	Concurrency int
+
+	doneCh     chan struct{}
+	finishedCh chan struct{}
+	// channel accepts new Group obj
+	// which supposed to update current group
+	updateCh chan *Group
+}
+
+func newGroup(cfg config.Group, defaultInterval time.Duration) *Group {
+	g := &Group{
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval,
+		Concurrency: cfg.Concurrency,
+		doneCh:      make(chan struct{}),
+		finishedCh:  make(chan struct{}),
+		updateCh:    make(chan *Group),
+	}
+	if g.Interval == 0 {
+		g.Interval = defaultInterval
+	}
+	if g.Concurrency < 1 {
+		g.Concurrency = 1
+	}
+	rules := make([]Rule, len(cfg.Rules))
+	for i, r := range cfg.Rules {
+		rules[i] = g.newRule(r)
+	}
+	g.Rules = rules
+	return g
+}
+
+func (g *Group) newRule(rule config.Rule) Rule {
+	if rule.Alert != "" {
+		return newAlertingRule(g.ID(), rule)
+	}
+	return newRecordingRule(g.ID(), rule)
+}
+
+// ID return unique group ID that consists of
+// rules file and group name
+func (g *Group) ID() uint64 {
+	hash := fnv.New64a()
+	hash.Write([]byte(g.File))
+	hash.Write([]byte("\xff"))
+	hash.Write([]byte(g.Name))
+	return hash.Sum64()
+}
+
+// Restore restores alerts state for group rules
+func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+	for _, rule := range g.Rules {
+		rr, ok := rule.(*AlertingRule)
+		if !ok {
+			continue
+		}
+		if rr.For < 1 {
+			continue
+		}
+		if err := rr.Restore(ctx, q, lookback); err != nil {
+			return fmt.Errorf("error while restoring rule %q: %s", rule, err)
+		}
+	}
+	return nil
+}
+
+// updateWith updates existing group with
+// passed group object. This function ignores group
+// evaluation interval change. It supposed to be updated
+// in group.start function.
+// Not thread-safe.
+func (g *Group) updateWith(newGroup *Group) error {
+	rulesRegistry := make(map[uint64]Rule)
+	for _, nr := range newGroup.Rules {
+		rulesRegistry[nr.ID()] = nr
+	}
+
+	for i, or := range g.Rules {
+		nr, ok := rulesRegistry[or.ID()]
+		if !ok {
+			// old rule is not present in the new list
+			// so we mark it for removing
+			g.Rules[i] = nil
+			continue
+		}
+		if err := or.UpdateWith(nr); err != nil {
+			return err
+		}
+		delete(rulesRegistry, nr.ID())
+	}
+
+	var newRules []Rule
+	for _, r := range g.Rules {
+		if r == nil {
+			// skip nil rules
+			continue
+		}
+		newRules = append(newRules, r)
+	}
+	// add the rest of rules from registry
+	for _, nr := range rulesRegistry {
+		newRules = append(newRules, nr)
+	}
+	g.Concurrency = newGroup.Concurrency
+	g.Rules = newRules
+	return nil
+}
+
+var (
+	iterationTotal    = metrics.NewCounter(`vmalert_iteration_total`)
+	iterationDuration = metrics.NewSummary(`vmalert_iteration_duration_seconds`)
+
+	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
+	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+
+	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
+	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
+	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
+
+	remoteWriteSent   = metrics.NewCounter(`vmalert_remotewrite_sent_total`)
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+)
+
+func (g *Group) close() {
+	if g.doneCh == nil {
+		return
+	}
+	close(g.doneCh)
+	<-g.finishedCh
+}
+
+func (g *Group) start(ctx context.Context, querier datasource.Querier, nr notifier.Notifier, rw *remotewrite.Client) {
+	defer func() { close(g.finishedCh) }()
+	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+	e := &executor{querier, nr, rw}
+	t := time.NewTicker(g.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Infof("group %q: context cancelled", g.Name)
+			return
+		case <-g.doneCh:
+			logger.Infof("group %q: received stop signal", g.Name)
+			return
+		case ng := <-g.updateCh:
+			g.mu.Lock()
+			err := g.updateWith(ng)
+			if err != nil {
+				logger.Errorf("group %q: failed to update: %s", g.Name, err)
+				g.mu.Unlock()
+				continue
+			}
+			if g.Interval != ng.Interval {
+				g.Interval = ng.Interval
+				t.Stop()
+				t = time.NewTicker(g.Interval)
+			}
+			g.mu.Unlock()
+			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+		case <-t.C:
+			iterationTotal.Inc()
+			iterationStart := time.Now()
+
+			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
+			for err := range errs {
+				if err != nil {
+					logger.Errorf("group %q: %s", g.Name, err)
+				}
+			}
+
+			iterationDuration.UpdateDuration(iterationStart)
+		}
+	}
+}
+
+type executor struct {
+	querier  datasource.Querier
+	notifier notifier.Notifier
+	rw       *remotewrite.Client
+}
+
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
+	res := make(chan error, len(rules))
+	var returnSeries bool
+	if e.rw != nil {
+		returnSeries = true
+	}
+
+	if concurrency == 1 {
+		// fast path
+		for _, rule := range rules {
+			res <- e.exec(ctx, rule, returnSeries, interval)
+		}
+		close(res)
+		return res
+	}
+
+	sem := make(chan struct{}, concurrency)
+	go func() {
+		wg := sync.WaitGroup{}
+		for _, rule := range rules {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func(r Rule) {
+				res <- e.exec(ctx, r, returnSeries, interval)
+				<-sem
+				wg.Done()
+			}(rule)
+		}
+		wg.Wait()
+		close(res)
+	}()
+	return res
+}
+
+func (e *executor) exec(ctx context.Context, rule Rule, returnSeries bool, interval time.Duration) error {
+	execTotal.Inc()
+	execStart := time.Now()
+	defer func() {
+		execDuration.UpdateDuration(execStart)
+	}()
+
+	tss, err := rule.Exec(ctx, e.querier, returnSeries)
+	if err != nil {
+		execErrors.Inc()
+		return fmt.Errorf("rule %q: failed to execute: %s", rule, err)
+	}
+
+	if len(tss) > 0 && e.rw != nil {
+		remoteWriteSent.Add(len(tss))
+		for _, ts := range tss {
+			if err := e.rw.Push(ts); err != nil {
+				remoteWriteErrors.Inc()
+				return fmt.Errorf("rule %q: remote write failure: %s", rule, err)
+			}
+		}
+	}
+
+	ar, ok := rule.(*AlertingRule)
+	if !ok {
+		return nil
+	}
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		switch a.State {
+		case notifier.StateFiring:
+			// set End to execStart + 3 intervals
+			// so notifier can resolve it automatically if `vmalert`
+			// won't be able to send resolve for some reason
+			a.End = time.Now().Add(3 * interval)
+			alerts = append(alerts, *a)
+		case notifier.StateInactive:
+			// set End to execStart to notify
+			// that it was just resolved
+			a.End = time.Now()
+			alerts = append(alerts, *a)
+		}
+	}
+	if len(alerts) < 1 {
+		return nil
+	}
+	alertsSent.Add(len(alerts))
+	if err := e.notifier.Send(ctx, alerts); err != nil {
+		alertsSendErrors.Inc()
+		return fmt.Errorf("rule %q: failed to send alerts: %s", rule, err)
+	}
+	return nil
+}

app/vmalert/group_test.go

@@ -0,0 +1,207 @@
+package main
+
+import (
+	"context"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestUpdateWith(t *testing.T) {
+	testCases := []struct {
+		name         string
+		currentRules []config.Rule
+		newRules     []config.Rule
+	}{
+		{
+			"new rule",
+			nil,
+			[]config.Rule{{Alert: "bar"}},
+		},
+		{
+			"update alerting rule",
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 0",
+				For:   time.Second,
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+				Annotations: map[string]string{
+					"summary":     "{{ $value|humanize }}",
+					"description": "{{$labels}}",
+				},
+			}},
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 10",
+				For:   time.Second,
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+				Annotations: map[string]string{
+					"summary": "none",
+				},
+			}},
+		},
+		{
+			"update recording rule",
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "max(up)",
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			}},
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "min(up)",
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+			}},
+		},
+		{
+			"empty rule",
+			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
+			nil,
+		},
+		{
+			"multiple rules",
+			[]config.Rule{
+				{Alert: "bar"},
+				{Alert: "baz"},
+				{Alert: "foo"},
+			},
+			[]config.Rule{
+				{Alert: "baz"},
+				{Record: "foo"},
+			},
+		},
+		{
+			"replace rule",
+			[]config.Rule{{Alert: "foo1"}},
+			[]config.Rule{{Alert: "foo2"}},
+		},
+		{
+			"replace multiple rules",
+			[]config.Rule{
+				{Alert: "foo1"},
+				{Record: "foo2"},
+				{Alert: "foo3"},
+			},
+			[]config.Rule{
+				{Alert: "foo3"},
+				{Alert: "foo4"},
+				{Record: "foo5"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &Group{Name: "test"}
+			for _, r := range tc.currentRules {
+				r.ID = config.HashRule(r)
+				g.Rules = append(g.Rules, g.newRule(r))
+			}
+
+			ng := &Group{Name: "test"}
+			for _, r := range tc.newRules {
+				r.ID = config.HashRule(r)
+				ng.Rules = append(ng.Rules, ng.newRule(r))
+			}
+
+			err := g.updateWith(ng)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if len(g.Rules) != len(tc.newRules) {
+				t.Fatalf("expected to have %d rules; got: %d",
+					len(g.Rules), len(tc.newRules))
+			}
+			sort.Slice(g.Rules, func(i, j int) bool {
+				return g.Rules[i].ID() < g.Rules[j].ID()
+			})
+			sort.Slice(ng.Rules, func(i, j int) bool {
+				return ng.Rules[i].ID() < ng.Rules[j].ID()
+			})
+			for i, r := range g.Rules {
+				got, want := r, ng.Rules[i]
+				if got.ID() != want.ID() {
+					t.Fatalf("expected to have rule %q; got %q", want, got)
+				}
+				if err := compareRules(t, got, want); err != nil {
+					t.Fatalf("comparsion error: %s", err)
+				}
+			}
+		})
+	}
+}
+
+func TestGroupStart(t *testing.T) {
+	// TODO: make parsing from string instead of file
+	groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
+	if err != nil {
+		t.Fatalf("failed to parse rules: %s", err)
+	}
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], evalInterval)
+	g.Concurrency = 2
+
+	fn := &fakeNotifier{}
+	fs := &fakeQuerier{}
+
+	const inst1, inst2, job = "foo", "bar", "baz"
+	m1 := metricWithLabels(t, "instance", inst1, "job", job)
+	m2 := metricWithLabels(t, "instance", inst2, "job", job)
+
+	r := g.Rules[0].(*AlertingRule)
+	alert1, err := r.newAlert(m1, time.Now())
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
+	alert1.State = notifier.StateFiring
+	alert1.ID = hash(m1)
+
+	alert2, err := r.newAlert(m2, time.Now())
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
+	alert2.State = notifier.StateFiring
+	alert2.ID = hash(m2)
+
+	finished := make(chan struct{})
+	fs.add(m1)
+	fs.add(m2)
+	go func() {
+		g.start(context.Background(), fs, fn, nil)
+		close(finished)
+	}()
+
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
+
+	gotAlerts := fn.getAlerts()
+	expectedAlerts := []notifier.Alert{*alert1, *alert2}
+	compareAlerts(t, expectedAlerts, gotAlerts)
+
+	// reset previous data
+	fs.reset()
+	// and set only one datapoint for response
+	fs.add(m1)
+
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
+
+	gotAlerts = fn.getAlerts()
+	expectedAlerts = []notifier.Alert{*alert1}
+	compareAlerts(t, expectedAlerts, gotAlerts)
+
+	g.close()
+	<-finished
+}

app/vmalert/helpers_test.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+type fakeQuerier struct {
+	sync.Mutex
+	metrics []datasource.Metric
+	err     error
+}
+
+func (fq *fakeQuerier) setErr(err error) {
+	fq.Lock()
+	fq.err = err
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) reset() {
+	fq.Lock()
+	fq.err = nil
+	fq.metrics = fq.metrics[:0]
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
+	fq.Lock()
+	fq.metrics = append(fq.metrics, metrics...)
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) {
+	fq.Lock()
+	defer fq.Unlock()
+	if fq.err != nil {
+		return nil, fq.err
+	}
+	cp := make([]datasource.Metric, len(fq.metrics))
+	copy(cp, fq.metrics)
+	return cp, nil
+}
+
+type fakeNotifier struct {
+	sync.Mutex
+	alerts []notifier.Alert
+}
+
+func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
+	fn.Lock()
+	defer fn.Unlock()
+	fn.alerts = alerts
+	return nil
+}
+
+func (fn *fakeNotifier) getAlerts() []notifier.Alert {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.alerts
+}
+
+func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
+	t.Helper()
+	m := metricWithLabels(t, labels...)
+	m.Value = value
+	return m
+}
+
+func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	m := datasource.Metric{}
+	for i := 0; i < len(labels); i += 2 {
+		m.Labels = append(m.Labels, datasource.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return m
+}
+
+func compareGroups(t *testing.T, a, b *Group) {
+	t.Helper()
+	if a.Name != b.Name {
+		t.Fatalf("expected group name %q; got %q", a.Name, b.Name)
+	}
+	if a.File != b.File {
+		t.Fatalf("expected group %q file name %q; got %q", a.Name, a.File, b.File)
+	}
+	if a.Interval != b.Interval {
+		t.Fatalf("expected group %q interval %v; got %v", a.Name, a.Interval, b.Interval)
+	}
+	if len(a.Rules) != len(b.Rules) {
+		t.Fatalf("expected group %s to have %d rules; got: %d",
+			a.Name, len(a.Rules), len(b.Rules))
+	}
+	for i, r := range a.Rules {
+		got, want := r, b.Rules[i]
+		if a.ID() != b.ID() {
+			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+		}
+		if err := compareRules(t, want, got); err != nil {
+			t.Fatalf("comparsion error: %s", err)
+		}
+	}
+}
+
+func compareRules(t *testing.T, a, b Rule) error {
+	t.Helper()
+	switch v := a.(type) {
+	case *AlertingRule:
+		br, ok := b.(*AlertingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
+		}
+		return compareAlertingRules(t, v, br)
+	case *RecordingRule:
+		br, ok := b.(*RecordingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
+		}
+		return compareRecordingRules(t, v, br)
+	default:
+		return fmt.Errorf("unexpected rule type received %T", a)
+	}
+}
+
+func compareRecordingRules(t *testing.T, a, b *RecordingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	return nil
+}
+
+func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if a.For != b.For {
+		return fmt.Errorf("expected to have for %q; got %q", a.For, b.For)
+	}
+	if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+		return fmt.Errorf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	return nil
+}
+
+func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
+	t.Helper()
+	if len(a) != len(b) {
+		return fmt.Errorf("expected number of timeseries %d; got %d", len(a), len(b))
+	}
+	for i := range a {
+		expTS, gotTS := a[i], b[i]
+		if len(expTS.Samples) != len(gotTS.Samples) {
+			return fmt.Errorf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+		}
+		for i, exp := range expTS.Samples {
+			got := gotTS.Samples[i]
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+			}
+			// timestamp validation isn't always correct for now.
+			// this must be improved with time mock.
+			/*if got.Timestamp != exp.Timestamp {
+				return fmt.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+			}*/
+		}
+		if len(expTS.Labels) != len(gotTS.Labels) {
+			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+		}
+		for i, exp := range expTS.Labels {
+			got := gotTS.Labels[i]
+			if got.Name != exp.Name {
+				return fmt.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+			}
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+			}
+		}
+	}
+	return nil
+}
+
+func compareAlerts(t *testing.T, as, bs []notifier.Alert) {
+	t.Helper()
+	if len(as) != len(bs) {
+		t.Fatalf("expected to have length %d; got %d", len(as), len(bs))
+	}
+	sort.Slice(as, func(i, j int) bool {
+		return as[i].ID < as[j].ID
+	})
+	sort.Slice(bs, func(i, j int) bool {
+		return bs[i].ID < bs[j].ID
+	})
+	for i := range as {
+		a, b := as[i], bs[i]
+		if a.Name != b.Name {
+			t.Fatalf("expected t have Name %q; got %q", a.Name, b.Name)
+		}
+		if a.State != b.State {
+			t.Fatalf("expected t have State %q; got %q", a.State, b.State)
+		}
+		if a.Value != b.Value {
+			t.Fatalf("expected t have Value %f; got %f", a.Value, b.Value)
+		}
+		if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+			t.Fatalf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+		}
+		if !reflect.DeepEqual(a.Labels, b.Labels) {
+			t.Fatalf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+		}
+	}
+}

app/vmalert/main.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules. 
+Supports patterns. Flag can be specified multiple times. 
+Examples:
+ -rule /path/to/file. Path to a single file with alerting rules
+ -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+absolute path to all .yaml files in root.`)
+
+	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
+	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+
+	httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+
+	datasourceURL = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+
+	remoteWriteURL = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
+		" and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
+	remoteWriteUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	remoteWritePassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	remoteWriteMaxQueueSize = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
+	remoteWriteMaxBatchSize = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
+	remoteWriteConcurrency  = flag.Int("remoteWrite.concurrency", 1, "Defines number of writers for concurrent writing into remote storage")
+
+	remoteReadURL = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
+		" state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state."+
+		" E.g. http://127.0.0.1:8428")
+	remoteReadUsername = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
+	remoteReadPassword = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
+		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+
+	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
+	notifierURL        = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	externalURL        = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	checkFlags()
+	ctx, cancel := context.WithCancel(context.Background())
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, false)
+	if err != nil {
+		logger.Fatalf("can not get external url: %s ", err)
+	}
+	notifier.InitTemplateFunc(eu)
+
+	manager := &manager{
+		groups:  make(map[uint64]*Group),
+		storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
+		notifier: notifier.NewAlertManager(*notifierURL, func(group, alert string) string {
+			return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, alert)
+		}, &http.Client{}),
+	}
+	if *remoteWriteURL != "" {
+		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
+			Addr:          *remoteWriteURL,
+			Concurrency:   *remoteWriteConcurrency,
+			MaxQueueSize:  *remoteWriteMaxQueueSize,
+			MaxBatchSize:  *remoteWriteMaxBatchSize,
+			FlushInterval: *evaluationInterval,
+			BasicAuthUser: *remoteWriteUsername,
+			BasicAuthPass: *remoteWritePassword,
+		})
+		if err != nil {
+			logger.Fatalf("failed to init remotewrite client: %s", err)
+		}
+		manager.rw = c
+	}
+	if *remoteReadURL != "" {
+		manager.rr = datasource.NewVMStorage(*remoteReadURL, *remoteReadUsername, *remoteReadPassword, &http.Client{})
+	}
+
+	if err := manager.start(ctx, *rulePath, *validateTemplates, *validateExpressions); err != nil {
+		logger.Fatalf("failed to start: %s", err)
+	}
+
+	go func() {
+		// init reload metrics with positive values to improve alerting conditions
+		configSuccess.Set(1)
+		configTimestamp.Set(fasttime.UnixTimestamp())
+		sigHup := procutil.NewSighupChan()
+		for {
+			<-sigHup
+			configReloads.Inc()
+			logger.Infof("SIGHUP received. Going to reload rules %q ...", *rulePath)
+			if err := manager.update(ctx, *rulePath, *validateTemplates, *validateExpressions, false); err != nil {
+				configReloadErrors.Inc()
+				configSuccess.Set(0)
+				logger.Errorf("error while reloading rules: %s", err)
+				continue
+			}
+			configSuccess.Set(1)
+			configTimestamp.Set(fasttime.UnixTimestamp())
+			logger.Infof("Rules reloaded successfully from %q", *rulePath)
+		}
+	}()
+
+	rh := &requestHandler{m: manager}
+	go httpserver.Serve(*httpListenAddr, rh.handler)
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	cancel()
+	manager.close()
+}
+
+var (
+	configReloads      = metrics.NewCounter(`vmalert_config_last_reload_total`)
+	configReloadErrors = metrics.NewCounter(`vmalert_config_last_reload_errors_total`)
+	configSuccess      = metrics.NewCounter(`vmalert_config_last_reload_successful`)
+	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
+)
+
+func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
+	if externalURL != "" {
+		return url.Parse(externalURL)
+	}
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+	port := ""
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+		port = ":" + ipport[1]
+	}
+	schema := "http://"
+	if isSecure {
+		schema = "https://"
+	}
+	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
+}
+
+func checkFlags() {
+	if *notifierURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("notifier.url is empty")
+	}
+	if *datasourceURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("datasource.url is empty")
+	}
+}
+
+func usage() {
+	const s = `
+vmalert processes alerts and recording rules.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}

app/vmalert/manager.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// manager controls group states
+type manager struct {
+	storage  datasource.Querier
+	notifier notifier.Notifier
+
+	rw *remotewrite.Client
+	rr datasource.Querier
+
+	wg sync.WaitGroup
+
+	groupsMu sync.RWMutex
+	groups   map[uint64]*Group
+}
+
+// AlertAPI generates APIAlert object from alert by its ID(hash)
+func (m *manager) AlertAPI(gID, aID uint64) (*APIAlert, error) {
+	m.groupsMu.RLock()
+	defer m.groupsMu.RUnlock()
+
+	g, ok := m.groups[gID]
+	if !ok {
+		return nil, fmt.Errorf("can't find group with id %q", gID)
+	}
+	for _, rule := range g.Rules {
+		ar, ok := rule.(*AlertingRule)
+		if !ok {
+			continue
+		}
+		if apiAlert := ar.AlertAPI(aID); apiAlert != nil {
+			return apiAlert, nil
+		}
+	}
+	return nil, fmt.Errorf("can't find alert with id %q in group %q", aID, g.Name)
+}
+
+func (m *manager) start(ctx context.Context, path []string, validateTpl, validateExpr bool) error {
+	return m.update(ctx, path, validateTpl, validateExpr, true)
+}
+
+func (m *manager) close() {
+	if m.rw != nil {
+		err := m.rw.Close()
+		if err != nil {
+			logger.Fatalf("cannot stop the remotewrite: %s", err)
+		}
+	}
+	m.wg.Wait()
+}
+
+func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
+	if restore && m.rr != nil {
+		err := group.Restore(ctx, m.rr, *remoteReadLookBack)
+		if err != nil {
+			logger.Errorf("error while restoring state for group %q: %s", group.Name, err)
+		}
+	}
+
+	m.wg.Add(1)
+	id := group.ID()
+	go func() {
+		group.start(ctx, m.storage, m.notifier, m.rw)
+		m.wg.Done()
+	}()
+	m.groups[id] = group
+}
+
+func (m *manager) update(ctx context.Context, path []string, validateTpl, validateExpr, restore bool) error {
+	logger.Infof("reading rules configuration file from %q", strings.Join(path, ";"))
+	groupsCfg, err := config.Parse(path, validateTpl, validateExpr)
+	if err != nil {
+		return fmt.Errorf("cannot parse configuration file: %s", err)
+	}
+
+	groupsRegistry := make(map[uint64]*Group)
+	for _, cfg := range groupsCfg {
+		ng := newGroup(cfg, *evaluationInterval)
+		groupsRegistry[ng.ID()] = ng
+	}
+
+	m.groupsMu.Lock()
+	for _, og := range m.groups {
+		ng, ok := groupsRegistry[og.ID()]
+		if !ok {
+			// old group is not present in new list
+			// and must be stopped and deleted
+			og.close()
+			delete(m.groups, og.ID())
+			og = nil
+			continue
+		}
+		og.updateCh <- ng
+		delete(groupsRegistry, ng.ID())
+	}
+
+	for _, ng := range groupsRegistry {
+		m.startGroup(ctx, ng, restore)
+	}
+	m.groupsMu.Unlock()
+	return nil
+}
+
+func (g *Group) toAPI() APIGroup {
+	ag := APIGroup{
+		// encode as strings to avoid rounding
+		ID:          fmt.Sprintf("%d", g.ID()),
+		Name:        g.Name,
+		File:        g.File,
+		Interval:    g.Interval.String(),
+		Concurrency: g.Concurrency,
+	}
+	for _, r := range g.Rules {
+		switch v := r.(type) {
+		case *AlertingRule:
+			ag.AlertingRules = append(ag.AlertingRules, v.RuleAPI())
+		case *RecordingRule:
+			ag.RecordingRules = append(ag.RecordingRules, v.RuleAPI())
+		}
+	}
+	return ag
+}

app/vmalert/manager_test.go

@@ -0,0 +1,211 @@
+package main
+
+import (
+	"context"
+	"math/rand"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+func TestManagerUpdateError(t *testing.T) {
+	m := &manager{groups: make(map[uint64]*Group)}
+	path := []string{"foo/bar"}
+	err := m.update(context.Background(), path, true, true, false)
+	if err == nil {
+		t.Fatalf("expected to have err; got nil instead")
+	}
+	expErr := "no groups found"
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to got err %s; got %s", expErr, err)
+	}
+}
+
+// TestManagerUpdateConcurrent supposed to test concurrent
+// execution of configuration update.
+// Should be executed with -race flag
+func TestManagerUpdateConcurrent(t *testing.T) {
+	m := &manager{
+		groups:   make(map[uint64]*Group),
+		storage:  &fakeQuerier{},
+		notifier: &fakeNotifier{},
+	}
+	paths := []string{
+		"config/testdata/dir/rules0-good.rules",
+		"config/testdata/dir/rules0-bad.rules",
+		"config/testdata/dir/rules1-good.rules",
+		"config/testdata/dir/rules1-bad.rules",
+		"config/testdata/rules0-good.rules",
+		"config/testdata/rules1-good.rules",
+		"config/testdata/rules2-good.rules",
+	}
+	*evaluationInterval = time.Millisecond
+	if err := m.start(context.Background(), []string{paths[0]}, true, true); err != nil {
+		t.Fatalf("failed to start: %s", err)
+	}
+
+	const workers = 500
+	const iterations = 10
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < iterations; i++ {
+				rnd := rand.Intn(len(paths))
+				path := []string{paths[rnd]}
+				_ = m.update(context.Background(), path, true, true, false)
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+// TestManagerUpdate tests sequential configuration
+// updates.
+func TestManagerUpdate(t *testing.T) {
+	const defaultEvalInterval = time.Second * 30
+	currentEvalInterval := *evaluationInterval
+	*evaluationInterval = defaultEvalInterval
+	defer func() {
+		*evaluationInterval = currentEvalInterval
+	}()
+
+	var (
+		VMRows = &AlertingRule{
+			Name: "VMRows",
+			Expr: "vm_rows > 0",
+			For:  10 * time.Second,
+			Labels: map[string]string{
+				"label": "bar",
+				"host":  "{{ $labels.instance }}",
+			},
+			Annotations: map[string]string{
+				"summary":     "{{ $value|humanize }}",
+				"description": "{{$labels}}",
+			},
+		}
+		Conns = &AlertingRule{
+			Name: "Conns",
+			Expr: "sum(vm_tcplistener_conns) by(instance) > 1",
+			Annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}}",
+			},
+		}
+		ExampleAlertAlwaysFiring = &AlertingRule{
+			Name: "ExampleAlertAlwaysFiring",
+			Expr: "sum by(job) (up == 1)",
+		}
+	)
+
+	testCases := []struct {
+		name       string
+		initPath   string
+		updatePath string
+		want       []*Group
+	}{
+		{
+			name:       "update good rules",
+			initPath:   "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules1-good.rules",
+			want: []*Group{
+				{
+					File:     "config/testdata/dir/rules1-good.rules",
+					Name:     "duplicatedGroupDiffFiles",
+					Interval: defaultEvalInterval,
+					Rules: []Rule{
+						&AlertingRule{
+							Name:   "VMRows",
+							Expr:   "vm_rows > 0",
+							For:    5 * time.Minute,
+							Labels: map[string]string{"label": "bar"},
+							Annotations: map[string]string{
+								"summary":     "{{ $value }}",
+								"description": "{{$labels}}",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:       "update good rules from 1 to 2 groups",
+			initPath:   "config/testdata/dir/rules1-good.rules",
+			updatePath: "config/testdata/rules0-good.rules",
+			want: []*Group{
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Rules:    []Rule{VMRows},
+					Interval: defaultEvalInterval,
+				},
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					}},
+			},
+		},
+		{
+			name:       "update with one bad rule file",
+			initPath:   "config/testdata/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules2-bad.rules",
+			want: []*Group{
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Interval: defaultEvalInterval,
+					Rules:    []Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					}},
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			m := &manager{groups: make(map[uint64]*Group), storage: &fakeQuerier{}}
+			path := []string{tc.initPath}
+			if err := m.update(ctx, path, true, true, false); err != nil {
+				t.Fatalf("failed to complete initial rules update: %s", err)
+			}
+
+			path = []string{tc.updatePath}
+			_ = m.update(ctx, path, true, true, false)
+			if len(tc.want) != len(m.groups) {
+				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
+			}
+
+			for _, wantG := range tc.want {
+				gotG, ok := m.groups[wantG.ID()]
+				if !ok {
+					t.Fatalf("expected to have group %q", wantG.Name)
+				}
+				compareGroups(t, wantG, gotG)
+			}
+
+			cancel()
+			m.close()
+		})
+	}
+}

app/vmalert/notifier/alert.go

@@ -0,0 +1,107 @@
+package notifier
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+	"time"
+)
+
+// Alert the triggered alert
+// TODO: Looks like alert name isn't unique
+type Alert struct {
+	GroupID     uint64
+	Name        string
+	Labels      map[string]string
+	Annotations map[string]string
+	State       AlertState
+
+	Expr  string
+	Start time.Time
+	End   time.Time
+	Value float64
+	ID    uint64
+}
+
+// AlertState type indicates the Alert state
+type AlertState int
+
+const (
+	// StateInactive is the state of an alert that is neither firing nor pending.
+	StateInactive AlertState = iota
+	// StatePending is the state of an alert that has been active for less than
+	// the configured threshold duration.
+	StatePending
+	// StateFiring is the state of an alert that has been active for longer than
+	// the configured threshold duration.
+	StateFiring
+)
+
+// String stringer for AlertState
+func (as AlertState) String() string {
+	switch as {
+	case StateFiring:
+		return "firing"
+	case StatePending:
+		return "pending"
+	}
+	return "inactive"
+}
+
+type alertTplData struct {
+	Labels map[string]string
+	Value  float64
+	Expr   string
+}
+
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
+
+// ExecTemplate executes the Alert template for give
+// map of annotations.
+func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
+	tplData := alertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
+	return templateAnnotations(annotations, tplHeader, tplData)
+}
+
+// ValidateTemplates validate annotations for possible template error, uses empty data for template population
+func ValidateTemplates(annotations map[string]string) error {
+	_, err := templateAnnotations(annotations, tplHeader, alertTplData{
+		Labels: map[string]string{},
+		Value:  0,
+	})
+	return err
+}
+
+func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	eg := errGroup{}
+	r := make(map[string]string, len(annotations))
+	for key, text := range annotations {
+		r[key] = text
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
+			eg.errs = append(eg.errs, fmt.Sprintf("key %q, template %q: %s", key, text, err))
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r, eg.err()
+}
+
+func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
+	tpl, err := template.New("").Funcs(tmplFunc).Option("missingkey=zero").Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation: %w", err)
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template: %w", err)
+	}
+	return nil
+}

app/vmalert/notifier/alert_test.go

@@ -0,0 +1,85 @@
+package notifier
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestAlert_ExecTemplate(t *testing.T) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	InitTemplateFunc(u)
+	testCases := []struct {
+		name        string
+		alert       *Alert
+		annotations map[string]string
+		expTpl      map[string]string
+	}{
+		{
+			name:        "empty-alert",
+			alert:       &Alert{},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "no-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "label-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}}",
+			},
+			expTpl: map[string]string{
+				"summary":     "Too high connection number for localhost for job staging",
+				"description": "It is 10000 connections for localhost",
+			},
+		},
+		{
+			name: "expression-template",
+			alert: &Alert{
+				Expr: `vm_rows{"label"="bar"}>0`,
+			},
+			annotations: map[string]string{
+				"exprEscapedQuery": "{{ $expr|quotesEscape|queryEscape }}",
+				"exprEscapedPath":  "{{ $expr|quotesEscape|pathEscape }}",
+			},
+			expTpl: map[string]string{
+				"exprEscapedQuery": "vm_rows%7B%5C%22label%5C%22%3D%5C%22bar%5C%22%7D%3E0",
+				"exprEscapedPath":  "vm_rows%7B%5C%22label%5C%22=%5C%22bar%5C%22%7D%3E0",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tpl, err := tc.alert.ExecTemplate(tc.annotations)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(tpl) != len(tc.expTpl) {
+				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
+			}
+			for k := range tc.expTpl {
+				got, exp := tpl[k], tc.expTpl[k]
+				if got != exp {
+					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
+				}
+			}
+		})
+	}
+}

app/vmalert/notifier/alertmanager.go

@@ -0,0 +1,60 @@
+package notifier
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// AlertManager represents integration provider with Prometheus alert manager
+// https://github.com/prometheus/alertmanager
+type AlertManager struct {
+	alertURL string
+	argFunc  AlertURLGenerator
+	client   *http.Client
+}
+
+// Send an alert or resolve message
+func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
+	b := &bytes.Buffer{}
+	writeamRequest(b, alerts, am.argFunc)
+
+	req, err := http.NewRequest("POST", am.alertURL, b)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req = req.WithContext(ctx)
+	resp, err := am.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to read response from %q: %s", am.alertURL, err)
+		}
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+	}
+	return nil
+}
+
+// AlertURLGenerator returns URL to single alert by given name
+type AlertURLGenerator func(group, alert string) string
+
+const alertManagerPath = "/api/v2/alerts"
+
+// NewAlertManager is a constructor for AlertManager
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+	return &AlertManager{
+		alertURL: strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath,
+		argFunc:  fn,
+		client:   c,
+	}
+}

app/vmalert/notifier/alertmanager_request.qtpl

@@ -0,0 +1,34 @@
+{% import (
+    "strconv"
+    "time"
+) %}
+{% stripspace %}
+
+{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %}
+[
+{% for i, alert := range alerts %}
+{
+	"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
+	"generatorURL": {%q= generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)) %},
+    {% if !alert.End.IsZero() %}
+    "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
+    {% endif %}
+    "labels": {
+        "alertname":{%q= alert.Name %}
+        {% for k,v := range alert.Labels %}
+            ,{%q= k %}:{%q= v %}
+        {% endfor %}
+    },
+    "annotations": {
+        {% code c := len(alert.Annotations) %}
+        {% for k,v := range alert.Annotations %}
+          {% code c = c-1 %}
+          {%q= k %}:{%q= v %}{% if c > 0 %},{% endif %}
+        {% endfor %}
+    }
+}
+{% if i != len(alerts)-1 %},{% endif %}
+{% endfor %}
+]
+{% endfunc %}
+{% endstripspace %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -0,0 +1,131 @@
+// Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line notifier/alertmanager_request.qtpl:1
+package notifier
+
+//line notifier/alertmanager_request.qtpl:1
+import (
+	"strconv"
+	"time"
+)
+
+//line notifier/alertmanager_request.qtpl:7
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line notifier/alertmanager_request.qtpl:7
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line notifier/alertmanager_request.qtpl:7
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line notifier/alertmanager_request.qtpl:7
+	qw422016.N().S(`[`)
+//line notifier/alertmanager_request.qtpl:9
+	for i, alert := range alerts {
+//line notifier/alertmanager_request.qtpl:9
+		qw422016.N().S(`{"startsAt":`)
+//line notifier/alertmanager_request.qtpl:11
+		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
+//line notifier/alertmanager_request.qtpl:11
+		qw422016.N().S(`,"generatorURL":`)
+//line notifier/alertmanager_request.qtpl:12
+		qw422016.N().Q(generatorURL(strconv.FormatUint(alert.GroupID, 10), strconv.FormatUint(alert.ID, 10)))
+//line notifier/alertmanager_request.qtpl:12
+		qw422016.N().S(`,`)
+//line notifier/alertmanager_request.qtpl:13
+		if !alert.End.IsZero() {
+//line notifier/alertmanager_request.qtpl:13
+			qw422016.N().S(`"endsAt":`)
+//line notifier/alertmanager_request.qtpl:14
+			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
+//line notifier/alertmanager_request.qtpl:14
+			qw422016.N().S(`,`)
+//line notifier/alertmanager_request.qtpl:15
+		}
+//line notifier/alertmanager_request.qtpl:15
+		qw422016.N().S(`"labels": {"alertname":`)
+//line notifier/alertmanager_request.qtpl:17
+		qw422016.N().Q(alert.Name)
+//line notifier/alertmanager_request.qtpl:18
+		for k, v := range alert.Labels {
+//line notifier/alertmanager_request.qtpl:18
+			qw422016.N().S(`,`)
+//line notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(k)
+//line notifier/alertmanager_request.qtpl:19
+			qw422016.N().S(`:`)
+//line notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(v)
+//line notifier/alertmanager_request.qtpl:20
+		}
+//line notifier/alertmanager_request.qtpl:20
+		qw422016.N().S(`},"annotations": {`)
+//line notifier/alertmanager_request.qtpl:23
+		c := len(alert.Annotations)
+
+//line notifier/alertmanager_request.qtpl:24
+		for k, v := range alert.Annotations {
+//line notifier/alertmanager_request.qtpl:25
+			c = c - 1
+
+//line notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(k)
+//line notifier/alertmanager_request.qtpl:26
+			qw422016.N().S(`:`)
+//line notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(v)
+//line notifier/alertmanager_request.qtpl:26
+			if c > 0 {
+//line notifier/alertmanager_request.qtpl:26
+				qw422016.N().S(`,`)
+//line notifier/alertmanager_request.qtpl:26
+			}
+//line notifier/alertmanager_request.qtpl:27
+		}
+//line notifier/alertmanager_request.qtpl:27
+		qw422016.N().S(`}}`)
+//line notifier/alertmanager_request.qtpl:30
+		if i != len(alerts)-1 {
+//line notifier/alertmanager_request.qtpl:30
+			qw422016.N().S(`,`)
+//line notifier/alertmanager_request.qtpl:30
+		}
+//line notifier/alertmanager_request.qtpl:31
+	}
+//line notifier/alertmanager_request.qtpl:31
+	qw422016.N().S(`]`)
+//line notifier/alertmanager_request.qtpl:33
+}
+
+//line notifier/alertmanager_request.qtpl:33
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line notifier/alertmanager_request.qtpl:33
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line notifier/alertmanager_request.qtpl:33
+	streamamRequest(qw422016, alerts, generatorURL)
+//line notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseWriter(qw422016)
+//line notifier/alertmanager_request.qtpl:33
+}
+
+//line notifier/alertmanager_request.qtpl:33
+func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
+//line notifier/alertmanager_request.qtpl:33
+	qb422016 := qt422016.AcquireByteBuffer()
+//line notifier/alertmanager_request.qtpl:33
+	writeamRequest(qb422016, alerts, generatorURL)
+//line notifier/alertmanager_request.qtpl:33
+	qs422016 := string(qb422016.B)
+//line notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseByteBuffer(qb422016)
+//line notifier/alertmanager_request.qtpl:33
+	return qs422016
+//line notifier/alertmanager_request.qtpl:33
+}

app/vmalert/notifier/alertmanager_test.go

@@ -0,0 +1,81 @@
+package notifier
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAlertManager_Send(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			var a []struct {
+				Labels       map[string]string `json:"labels"`
+				StartsAt     time.Time         `json:"startsAt"`
+				EndAt        time.Time         `json:"endsAt"`
+				Annotations  map[string]string `json:"annotations"`
+				GeneratorURL string            `json:"generatorURL"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
+			}
+			if len(a) != 1 {
+				t.Errorf("expected 1 alert in array got %d", len(a))
+			}
+			if a[0].GeneratorURL != "0/0" {
+				t.Errorf("exptected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+			}
+			if a[0].Labels["alertname"] != "alert0" {
+				t.Errorf("exptected alert0 as alert name got %s", a[0].Labels["alertname"])
+			}
+			if a[0].StartsAt.IsZero() {
+				t.Errorf("exptected non-zero start time")
+			}
+			if a[0].EndAt.IsZero() {
+				t.Errorf("exptected non-zero end time")
+			}
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewAlertManager(srv.URL, func(group, name string) string {
+		return group + "/" + name
+	}, srv.Client())
+	if err := am.Send(context.Background(), []Alert{{}, {}}); err == nil {
+		t.Error("expected connection error got nil")
+	}
+	if err := am.Send(context.Background(), []Alert{}); err == nil {
+		t.Error("expected wrong http code error got nil")
+	}
+	if err := am.Send(context.Background(), []Alert{{
+		GroupID:     0,
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if c != 2 {
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+	}
+}

app/vmalert/notifier/notifier.go

@@ -0,0 +1,8 @@
+package notifier
+
+import "context"
+
+// Notifier is common interface for alert manager provider
+type Notifier interface {
+	Send(ctx context.Context, alerts []Alert) error
+}

app/vmalert/notifier/template_func.go

@@ -0,0 +1,180 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package notifier
+
+import (
+	"fmt"
+	html_template "html/template"
+	"math"
+	"net/url"
+	"regexp"
+	"strings"
+	text_template "text/template"
+	"time"
+)
+
+var tmplFunc text_template.FuncMap
+
+// InitTemplateFunc returns template helper functions
+func InitTemplateFunc(externalURL *url.URL) {
+	tmplFunc = text_template.FuncMap{
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+		"safeHtml": func(text string) html_template.HTML {
+			return html_template.HTML(text)
+		},
+		"match":   regexp.MatchString,
+		"title":   strings.Title,
+		"toUpper": strings.ToUpper,
+		"toLower": strings.ToLower,
+		"humanize": func(v float64) string {
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanize1024": func(v float64) string {
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			prefix := ""
+			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
+				if math.Abs(v) < 1024 {
+					break
+				}
+				prefix = p
+				v /= 1024
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanizeDuration": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v)
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix)
+		},
+		"humanizePercentage": func(v float64) string {
+			return fmt.Sprintf("%.4g%%", v*100)
+		},
+		"humanizeTimestamp": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
+			return fmt.Sprint(t)
+		},
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+		"pathEscape": func(u string) string {
+			return url.PathEscape(u)
+		},
+		"queryEscape": func(q string) string {
+			return url.QueryEscape(q)
+		},
+		"quotesEscape": func(q string) string {
+			return strings.Replace(q, `"`, `\"`, -1)
+		},
+	}
+}
+
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// TimeFromUnixNano returns the Time equivalent to the Unix Time
+// t provided in nanoseconds.
+func TimeFromUnixNano(t int64) Time {
+	return Time(t / nanosPerTick)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}

app/vmalert/notifier/utils.go

@@ -0,0 +1,21 @@
+package notifier
+
+import (
+	"fmt"
+	"strings"
+)
+
+type errGroup struct {
+	errs []string
+}
+
+func (eg *errGroup) err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+func (eg *errGroup) Error() string {
+	return fmt.Sprintf("errors: %s", strings.Join(eg.errs, "\n"))
+}

app/vmalert/recording.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// RecordingRule is a Rule that supposed
+// to evaluate configured Expression and
+// return TimeSeries as result.
+type RecordingRule struct {
+	RuleID  uint64
+	Name    string
+	Expr    string
+	Labels  map[string]string
+	GroupID uint64
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+}
+
+// String implements Stringer interface
+func (rr *RecordingRule) String() string {
+	return rr.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (rr *RecordingRule) ID() uint64 {
+	return rr.RuleID
+}
+
+func newRecordingRule(gID uint64, cfg config.Rule) *RecordingRule {
+	return &RecordingRule{
+		RuleID:  cfg.ID,
+		Name:    cfg.Record,
+		Expr:    cfg.Expr,
+		Labels:  cfg.Labels,
+		GroupID: gID,
+	}
+}
+
+var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
+
+// Exec executes RecordingRule expression via the given Querier.
+func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
+	if !series {
+		return nil, nil
+	}
+
+	qMetrics, err := q.Query(ctx, rr.Expr)
+
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+
+	rr.lastExecTime = time.Now()
+	rr.lastExecError = err
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %s", rr.Expr, err)
+	}
+
+	duplicates := make(map[uint64]prompbmarshal.TimeSeries, len(qMetrics))
+	var tss []prompbmarshal.TimeSeries
+	for _, r := range qMetrics {
+		ts := rr.toTimeSeries(r, rr.lastExecTime)
+		h := hashTimeSeries(ts)
+		if _, ok := duplicates[h]; ok {
+			rr.lastExecError = errDuplicate
+			return nil, errDuplicate
+		}
+		duplicates[h] = ts
+		tss = append(tss, ts)
+	}
+	return tss, nil
+}
+
+func hashTimeSeries(ts prompbmarshal.TimeSeries) uint64 {
+	hash := fnv.New64a()
+	labels := ts.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (rr *RecordingRule) toTimeSeries(m datasource.Metric, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for _, l := range m.Labels {
+		labels[l.Name] = l.Value
+	}
+	labels["__name__"] = rr.Name
+	// override existing labels with configured ones
+	for k, v := range rr.Labels {
+		labels[k] = v
+	}
+	return newTimeSeries(m.Value, labels, timestamp)
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (rr *RecordingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*RecordingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update recroding rule with wrong type %#v", r)
+	}
+	rr.Expr = nr.Expr
+	rr.Labels = nr.Labels
+	return nil
+}
+
+// RuleAPI returns Rule representation in form
+// of APIRecordingRule
+func (rr *RecordingRule) RuleAPI() APIRecordingRule {
+	var lastErr string
+	if rr.lastExecError != nil {
+		lastErr = rr.lastExecError.Error()
+	}
+	return APIRecordingRule{
+		// encode as strings to avoid rounding
+		ID:         fmt.Sprintf("%d", rr.ID()),
+		GroupID:    fmt.Sprintf("%d", rr.GroupID),
+		Name:       rr.Name,
+		Expression: rr.Expr,
+		LastError:  lastErr,
+		LastExec:   rr.lastExecTime,
+		Labels:     rr.Labels,
+	}
+}

app/vmalert/recording_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestRecoridngRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValueAndLabels(t, 10,
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(10, map[string]string{
+					"__name__": "foo",
+				}, timestamp),
+			},
+		},
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}, timestamp),
+				newTimeSeries(2, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}, timestamp),
+				newTimeSeries(3, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "baz",
+				}, timestamp),
+			},
+		},
+		{
+			&RecordingRule{Name: "job:foo", Labels: map[string]string{
+				"source": "test",
+			}},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar")},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(2, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}, timestamp),
+				newTimeSeries(1, map[string]string{
+					"__name__": "job:foo",
+					"job":      "bar",
+					"source":   "test",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			fq.add(tc.metrics...)
+			tss, err := tc.rule.Exec(context.TODO(), fq, true)
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestRecoridngRule_ToTimeSeriesNegative(t *testing.T) {
+	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
+		"job": "test",
+	}}
+
+	fq := &fakeQuerier{}
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+
+	_, err := rr.Exec(context.TODO(), fq, true)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+
+	fq.reset()
+
+	// add metrics which differs only by `job` label
+	// which will be overridden by rule
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
+	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
+
+	_, err = rr.Exec(context.TODO(), fq, true)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), errDuplicate.Error()) {
+		t.Fatalf("expected to get err %q; got %q insterad", errDuplicate, err)
+	}
+}

app/vmalert/remotewrite/remotewrite.go

@@ -0,0 +1,203 @@
+package remotewrite
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+// Client is an asynchronous HTTP client for writing
+// timeseries via remote write protocol.
+type Client struct {
+	addr           string
+	c              *http.Client
+	input          chan prompbmarshal.TimeSeries
+	baUser, baPass string
+	flushInterval  time.Duration
+	maxBatchSize   int
+	maxQueueSize   int
+
+	wg     sync.WaitGroup
+	doneCh chan struct{}
+}
+
+// Config is config for remote write.
+type Config struct {
+	// Addr of remote storage
+	Addr string
+
+	BasicAuthUser string
+	BasicAuthPass string
+
+	// Concurrency defines number of readers that
+	// concurrently read from the queue and flush data
+	Concurrency int
+	// MaxBatchSize defines max number of timeseries
+	// to be flushed at once
+	MaxBatchSize int
+	// MaxQueueSize defines max length of input queue
+	// populated by Push method.
+	// Push will be rejected once queue is full.
+	MaxQueueSize int
+	// FlushInterval defines time interval for flushing batches
+	FlushInterval time.Duration
+	// WriteTimeout defines timeout for HTTP write request
+	// to remote storage
+	WriteTimeout time.Duration
+}
+
+const (
+	defaultConcurrency   = 4
+	defaultMaxBatchSize  = 1e3
+	defaultMaxQueueSize  = 1e5
+	defaultFlushInterval = time.Second
+	defaultWriteTimeout  = 30 * time.Second
+)
+
+const writePath = "/api/v1/write"
+
+// NewClient returns asynchronous client for
+// writing timeseries via remotewrite protocol.
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Addr == "" {
+		return nil, fmt.Errorf("config.Addr can't be empty")
+	}
+	if cfg.MaxBatchSize == 0 {
+		cfg.MaxBatchSize = defaultMaxBatchSize
+	}
+	if cfg.MaxQueueSize == 0 {
+		cfg.MaxQueueSize = defaultMaxQueueSize
+	}
+	if cfg.FlushInterval == 0 {
+		cfg.FlushInterval = defaultFlushInterval
+	}
+	if cfg.WriteTimeout == 0 {
+		cfg.WriteTimeout = defaultWriteTimeout
+	}
+	c := &Client{
+		c: &http.Client{
+			Timeout: cfg.WriteTimeout,
+		},
+		addr:          strings.TrimSuffix(cfg.Addr, "/") + writePath,
+		baUser:        cfg.BasicAuthUser,
+		baPass:        cfg.BasicAuthPass,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
+	for i := 0; i < cc; i++ {
+		c.run(ctx)
+	}
+	return c, nil
+}
+
+// Push adds timeseries into queue for writing into remote storage.
+// Push returns and error if client is stopped or if queue is full.
+func (c *Client) Push(s prompbmarshal.TimeSeries) error {
+	select {
+	case <-c.doneCh:
+		return fmt.Errorf("client is closed")
+	case c.input <- s:
+		return nil
+	default:
+		return fmt.Errorf("failed to push timeseries - queue is full (%d entries). "+
+			"Queue size is controlled by -remoteWrite.maxQueueSize flag",
+			c.maxQueueSize)
+	}
+}
+
+// Close stops the client and waits for all goroutines
+// to exit.
+func (c *Client) Close() error {
+	if c.doneCh == nil {
+		return fmt.Errorf("client is already closed")
+	}
+	close(c.input)
+	close(c.doneCh)
+	c.wg.Wait()
+	return nil
+}
+
+func (c *Client) run(ctx context.Context) {
+	ticker := time.NewTicker(c.flushInterval)
+	wr := prompbmarshal.WriteRequest{}
+	shutdown := func() {
+		for ts := range c.input {
+			wr.Timeseries = append(wr.Timeseries, ts)
+		}
+		if len(wr.Timeseries) < 1 {
+			return
+		}
+		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
+		c.flush(lastCtx, wr)
+		cancel()
+	}
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer ticker.Stop()
+		for {
+			select {
+			case <-c.doneCh:
+				shutdown()
+				return
+			case <-ctx.Done():
+				shutdown()
+				return
+			case <-ticker.C:
+				c.flush(ctx, wr)
+				wr = prompbmarshal.WriteRequest{}
+			case ts := <-c.input:
+				wr.Timeseries = append(wr.Timeseries, ts)
+				if len(wr.Timeseries) >= c.maxBatchSize {
+					c.flush(ctx, wr)
+					wr = prompbmarshal.WriteRequest{}
+				}
+			}
+		}
+	}()
+}
+
+func (c *Client) flush(ctx context.Context, wr prompbmarshal.WriteRequest) {
+	if len(wr.Timeseries) < 1 {
+		return
+	}
+	data, err := wr.Marshal()
+	if err != nil {
+		logger.Errorf("failed to marshal WriteRequest: %s", err)
+		return
+	}
+	req, err := http.NewRequest("POST", c.addr, bytes.NewReader(snappy.Encode(nil, data)))
+	if err != nil {
+		logger.Errorf("failed to create new HTTP request: %s", err)
+		return
+	}
+	if c.baPass != "" {
+		req.SetBasicAuth(c.baUser, c.baPass)
+	}
+	resp, err := c.c.Do(req.WithContext(ctx))
+	if err != nil {
+		logger.Errorf("error getting response from %s:%s", req.URL, err)
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusNoContent {
+		body, _ := ioutil.ReadAll(resp.Body)
+		logger.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return
+	}
+}

app/vmalert/rule.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"context"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// Rule represents alerting or recording rule
+// that has unique ID, can be Executed and
+// updated with other Rule.
+type Rule interface {
+	// Returns unique ID that may be used for
+	// identifying this Rule among others.
+	ID() uint64
+	// Exec executes the rule with given context
+	// and Querier. If returnSeries is true, Exec
+	// may return TimeSeries as result of execution
+	Exec(ctx context.Context, q datasource.Querier, returnSeries bool) ([]prompbmarshal.TimeSeries, error)
+	// UpdateWith performs modification of current Rule
+	// with fields of the given Rule.
+	UpdateWith(Rule) error
+}

app/vmalert/utils.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"sort"
+	"time"
+)
+
+func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{}
+	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
+		Value:     value,
+		Timestamp: timestamp.UnixNano() / 1e6,
+	})
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}

app/vmalert/web.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+)
+
+type requestHandler struct {
+	m *manager
+}
+
+var pathList = [][]string{
+	{"/api/v1/groups", "list all loaded groups and rules"},
+	{"/api/v1/alerts", "list all active alerts"},
+	{"/api/v1/groupID/alertID/status", "get alert status by ID"},
+	// /metrics is served by httpserver by default
+	{"/metrics", "list of application metrics"},
+	{"/-/reload", "reload configuration"},
+}
+
+func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	resph := responseHandler{w}
+	switch r.URL.Path {
+	case "/":
+		for _, path := range pathList {
+			p, doc := path[0], path[1]
+			fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
+		}
+		return true
+	case "/api/v1/groups":
+		resph.handle(rh.listGroups())
+		return true
+	case "/api/v1/alerts":
+		resph.handle(rh.listAlerts())
+		return true
+	case "/-/reload":
+		logger.Infof("api config reload was called, sending sighup")
+		procutil.SelfSIGHUP()
+		w.WriteHeader(http.StatusOK)
+		return true
+	default:
+		// /api/v1/<groupName>/<alertID>/status
+		if strings.HasSuffix(r.URL.Path, "/status") {
+			resph.handle(rh.alert(r.URL.Path))
+			return true
+		}
+		return false
+	}
+}
+
+type listGroupsResponse struct {
+	Data struct {
+		Groups []APIGroup `json:"groups"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	lr := listGroupsResponse{Status: "success"}
+	for _, g := range rh.m.groups {
+		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+	}
+
+	// sort list of alerts for deterministic output
+	sort.Slice(lr.Data.Groups, func(i, j int) bool {
+		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	})
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of active alerts: %s`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
+type listAlertsResponse struct {
+	Data struct {
+		Alerts []*APIAlert `json:"alerts"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+func (rh *requestHandler) listAlerts() ([]byte, error) {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	lr := listAlertsResponse{Status: "success"}
+	for _, g := range rh.m.groups {
+		for _, r := range g.Rules {
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsAPI()...)
+		}
+	}
+
+	// sort list of alerts for deterministic output
+	sort.Slice(lr.Data.Alerts, func(i, j int) bool {
+		return lr.Data.Alerts[i].ID < lr.Data.Alerts[j].ID
+	})
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of active alerts: %s`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
+func (rh *requestHandler) alert(path string) ([]byte, error) {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	if len(parts) != 3 {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+
+	groupID, err := uint64FromPath(parts[0])
+	if err != nil {
+		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %s`, err))
+	}
+	alertID, err := uint64FromPath(parts[1])
+	if err != nil {
+		return nil, badRequest(fmt.Errorf(`cannot parse alertID: %s`, err))
+	}
+	resp, err := rh.m.AlertAPI(groupID, alertID)
+	if err != nil {
+		return nil, errResponse(err, http.StatusNotFound)
+	}
+	return json.Marshal(resp)
+}
+
+// responseHandler wrapper on http.ResponseWriter with sugar
+type responseHandler struct{ http.ResponseWriter }
+
+func (w responseHandler) handle(b []byte, err error) {
+	if err != nil {
+		httpserver.Errorf(w, "%s", err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(b)
+}
+
+func uint64FromPath(path string) (uint64, error) {
+	s := strings.TrimRight(path, "/")
+	return strconv.ParseUint(s, 10, 0)
+}
+
+func badRequest(err error) *httpserver.ErrorWithStatusCode {
+	return errResponse(err, http.StatusBadRequest)
+}
+
+func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
+	return &httpserver.ErrorWithStatusCode{
+		Err:        err,
+		StatusCode: sc,
+	}
+}

app/vmalert/web_test.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestHandler(t *testing.T) {
+	ar := &AlertingRule{
+		Name: "alert",
+		alerts: map[uint64]*notifier.Alert{
+			0: {},
+		},
+	}
+	g := &Group{
+		Name:  "group",
+		Rules: []Rule{ar},
+	}
+	m := &manager{groups: make(map[uint64]*Group)}
+	m.groups[0] = g
+	rh := &requestHandler{m: m}
+
+	getResp := func(url string, to interface{}, code int) {
+		t.Helper()
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Errorf("unexpected err %s", err)
+		}
+		if code != resp.StatusCode {
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				t.Errorf("err closing body %s", err)
+			}
+		}()
+		if to != nil {
+			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
+				t.Errorf("unexpected err %s", err)
+			}
+		}
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rh.handler(w, r) }))
+	defer ts.Close()
+	t.Run("/api/v1/alerts", func(t *testing.T) {
+		lr := listAlertsResponse{}
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		if length := len(lr.Data.Alerts); length != 1 {
+			t.Errorf("expected 1 alert got %d", length)
+		}
+	})
+	t.Run("/api/v1/groups", func(t *testing.T) {
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/groups", &lr, 200)
+		if length := len(lr.Data.Groups); length != 1 {
+			t.Errorf("expected 1 group got %d", length)
+		}
+	})
+	t.Run("/api/v1/0/0/status", func(t *testing.T) {
+		alert := &APIAlert{}
+		getResp(ts.URL+"/api/v1/0/0/status", alert, 200)
+		expAlert := ar.newAlertAPI(*ar.alerts[0])
+		if !reflect.DeepEqual(alert, expAlert) {
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
+		}
+	})
+	t.Run("/api/v1/0/1/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/0/1/status", nil, 404)
+	})
+	t.Run("/api/v1/1/0/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/1/0/status", nil, 404)
+	})
+	t.Run("/", func(t *testing.T) {
+		getResp(ts.URL, nil, 200)
+	})
+}

app/vmalert/web_types.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"time"
+)
+
+// APIAlert represents an notifier.AlertingRule state
+// for WEB view
+type APIAlert struct {
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	State       string            `json:"state"`
+	Value       string            `json:"value"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	ActiveAt    time.Time         `json:"activeAt"`
+}
+
+// APIGroup represents Group for WEB view
+type APIGroup struct {
+	Name           string             `json:"name"`
+	ID             string             `json:"id"`
+	File           string             `json:"file"`
+	Interval       string             `json:"interval"`
+	Concurrency    int                `json:"concurrency"`
+	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
+	RecordingRules []APIRecordingRule `json:"recording_rules"`
+}
+
+// APIAlertingRule represents AlertingRule for WEB view
+type APIAlertingRule struct {
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	For         string            `json:"for"`
+	LastError   string            `json:"last_error"`
+	LastExec    time.Time         `json:"last_exec"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+}
+
+// APIRecordingRule represents RecordingRule for WEB view
+type APIRecordingRule struct {
+	ID         string            `json:"id"`
+	Name       string            `json:"name"`
+	GroupID    string            `json:"group_id"`
+	Expression string            `json:"expression"`
+	LastError  string            `json:"last_error"`
+	LastExec   time.Time         `json:"last_exec"`
+	Labels     map[string]string `json:"labels"`
+}

app/vmauth/Makefile

@@ -0,0 +1,76 @@
+# All these commands must run from repository root.
+
+vmauth:
+	APP_NAME=vmauth $(MAKE) app-local
+
+vmauth-race:
+	APP_NAME=vmauth RACE=-race $(MAKE) app-local
+
+vmauth-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker
+
+vmauth-pure-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-pure
+
+vmauth-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-amd64
+
+vmauth-arm-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-arm
+
+vmauth-arm64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-arm64
+
+vmauth-ppc64le-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-ppc64le
+
+vmauth-386-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-386
+
+package-vmauth:
+	APP_NAME=vmauth $(MAKE) package-via-docker
+
+package-vmauth-pure:
+	APP_NAME=vmauth $(MAKE) package-via-docker-pure
+
+package-vmauth-amd64:
+	APP_NAME=vmauth $(MAKE) package-via-docker-amd64
+
+package-vmauth-arm:
+	APP_NAME=vmauth $(MAKE) package-via-docker-arm
+
+package-vmauth-arm64:
+	APP_NAME=vmauth $(MAKE) package-via-docker-arm64
+
+package-vmauth-ppc64le:
+	APP_NAME=vmauth $(MAKE) package-via-docker-ppc64le
+
+package-vmauth-386:
+	APP_NAME=vmauth $(MAKE) package-via-docker-386
+
+publish-vmauth:
+	APP_NAME=vmauth $(MAKE) publish-via-docker
+
+run-vmauth:
+	APP_NAME=vmauth \
+	DOCKER_OPTS='-v $(shell pwd)/app/vmauth/:/app/vmauth' \
+	ARGS='-auth.config=app/vmauth/example_config.yml' \
+	$(MAKE) run-via-docker
+
+vmauth-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-amd64 ./app/vmauth
+
+vmauth-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm ./app/vmauth
+
+vmauth-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm64 ./app/vmauth
+
+vmauth-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-ppc64le ./app/vmauth
+
+vmauth-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-386 ./app/vmauth
+
+vmauth-pure:
+	APP_NAME=vmauth $(MAKE) app-local-pure

app/vmauth/README.md

@@ -0,0 +1,139 @@
+## vmauth
+
+`vmauth` is a simple auth proxy and router for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
+It reads username and password from [Basic Auth headers](https://en.wikipedia.org/wiki/Basic_access_authentication),
+matches them against configs pointed by `-auth.config` command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
+
+
+### Quick start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flag to `vmauth` binary in order to start authorizing and routing requests:
+
+```
+/path/to/vmauth -auth.config=/path/to/auth/config.yml
+```
+
+After that `vmauth` starts accepting HTTP requests on port `8427` and routing them according to the provided [-auth.config](#auth-config).
+The port can be modified via `-httpListenAddr` command-line flag.
+
+The auth config can be reloaded by passing `SIGHUP` signal to `vmauth`.
+
+Docker images for `vmauth` are available [here](https://hub.docker.com/r/victoriametrics/vmauth/tags).
+
+Pass `-help` to `vmauth` in order to see all the supported command-line flags with their descriptions.
+
+Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML, accounting, limits, etc.
+
+
+### Auth config
+
+Auth config is represented in the following simple `yml` format:
+
+```yml
+
+# Arbitrary number of usernames may be put here.
+# Usernames must be unique.
+
+users:
+
+  # The user for querying local single-node VictoriaMetrics.
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://localhost:8428 .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query
+- username: "local-single-node"
+  password: "***"
+  url_prefix: "http://localhost:8428"
+
+  # The user for querying account 123 in VictoriaMetrics cluster
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vmselect:8481/select/123/prometheus .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8481/select/123/prometheus/api/v1/select
+- username: "cluster-select-account-123"
+  password: "***"
+  url_prefix: "http://vmselect:8481/select/123/prometheus"
+
+  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the reuqests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vminsert:8480/insert/42/prometheus .
+  # For example, http://vmauth:8427/api/v1/write is routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
+- username: "cluster-insert-account-42"
+  password: "***"
+  url_prefix: "http://vminsert:8480/insert/42/prometheus"
+```
+
+
+### Security
+
+Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
+
+```
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+```
+
+Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.
+
+
+### Monitoring
+
+`vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
+either via [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) or via Prometheus, so the exported metrics could be analyzed later.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmauth` from the root folder of the repository.
+   It builds `vmauth` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmauth-prod` from the root folder of the repository.
+   It builds `vmauth-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmauth
+```
+
+
+### Profiling
+
+`vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmauth-host>:8427/debug/pprof/heap > mem.pprof
+```
+
+* CPU profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

app/vmauth/auth_config.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md "+
+		"for details on the format of this auth config")
+)
+
+// AuthConfig represents auth config.
+type AuthConfig struct {
+	Users []UserInfo `yaml:"users"`
+}
+
+// UserInfo is user information read from authConfigPath
+type UserInfo struct {
+	Username  string `yaml:"username"`
+	Password  string `yaml:"password"`
+	URLPrefix string `yaml:"url_prefix"`
+
+	requests *metrics.Counter
+}
+
+func initAuthConfig() {
+	if len(*authConfigPath) == 0 {
+		logger.Fatalf("missing required `-auth.config` command-line flag")
+	}
+	m, err := readAuthConfig(*authConfigPath)
+	if err != nil {
+		logger.Fatalf("cannot load auth config from `-auth.config=%s`: %s", *authConfigPath, err)
+	}
+	authConfig.Store(m)
+	stopCh = make(chan struct{})
+	authConfigWG.Add(1)
+	go func() {
+		defer authConfigWG.Done()
+		authConfigReloader()
+	}()
+}
+
+func stopAuthConfig() {
+	close(stopCh)
+	authConfigWG.Wait()
+}
+
+func authConfigReloader() {
+	sighupCh := procutil.NewSighupChan()
+	for {
+		select {
+		case <-stopCh:
+			return
+		case <-sighupCh:
+			logger.Infof("SIGHUP received; loading -auth.config=%q", *authConfigPath)
+			m, err := readAuthConfig(*authConfigPath)
+			if err != nil {
+				logger.Errorf("failed to load -auth.config=%q; using the last successfully loaded config; error: %s", *authConfigPath, err)
+				continue
+			}
+			authConfig.Store(m)
+			logger.Infof("Successfully reloaded -auth.config=%q", *authConfigPath)
+		}
+	}
+}
+
+var authConfig atomic.Value
+var authConfigWG sync.WaitGroup
+var stopCh chan struct{}
+
+func readAuthConfig(path string) (map[string]*UserInfo, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read %q: %s", path, err)
+	}
+	m, err := parseAuthConfig(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse %q: %s", path, err)
+	}
+	logger.Infof("Loaded information about %d users from %q", len(m), path)
+	return m, nil
+}
+
+func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
+	var ac AuthConfig
+	if err := yaml.UnmarshalStrict(data, &ac); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %s", err)
+	}
+	uis := ac.Users
+	if len(uis) == 0 {
+		return nil, fmt.Errorf("`users` section cannot be empty in AuthConfig")
+	}
+	m := make(map[string]*UserInfo, len(uis))
+	for i := range uis {
+		ui := &uis[i]
+		if m[ui.Username] != nil {
+			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
+		}
+		urlPrefix := ui.URLPrefix
+		// Remove trailing '/' from urlPrefix
+		for strings.HasSuffix(urlPrefix, "/") {
+			urlPrefix = urlPrefix[:len(urlPrefix)-1]
+		}
+		// Validate urlPrefix
+		target, err := url.Parse(urlPrefix)
+		if err != nil {
+			return nil, fmt.Errorf("invalid `url_prefix: %q`: %s", urlPrefix, err)
+		}
+		if target.Scheme != "http" && target.Scheme != "https" {
+			return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)
+		}
+
+		ui.URLPrefix = urlPrefix
+		ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+		m[ui.Username] = ui
+	}
+	return m, nil
+}

app/vmauth/auth_config_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseAuthConfigFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		_, err := parseAuthConfig([]byte(s))
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// Empty config
+	f(``)
+
+	// Invalid entry
+	f(`foobar`)
+	f(`foobar: baz`)
+
+	// Empty users
+	f(`users: []`)
+
+	// Missing url_prefix
+	f(`
+users:
+- username: foo
+`)
+
+	// Invalid url_prefix
+	f(`
+users:
+- username: foo
+  url_prefix: bar
+`)
+	f(`
+users:
+- username: foo
+  url_prefix: ftp://bar
+`)
+	f(`
+users:
+- username: foo
+  url_prefix: //bar
+`)
+
+	// Duplicate users
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+- username: bar
+  url_prefix: http://xxx.yyy
+- username: foo
+  url_prefix: https://sss.sss
+`)
+}
+
+func TestParseAuthConfigSuccess(t *testing.T) {
+	f := func(s string, expectedAuthConfig map[string]*UserInfo) {
+		t.Helper()
+		m, err := parseAuthConfig([]byte(s))
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		removeMetrics(m)
+		if !reflect.DeepEqual(m, expectedAuthConfig) {
+			t.Fatalf("unexpected auth config\ngot\n%v\nwant\n%v", m, expectedAuthConfig)
+		}
+	}
+
+	// Single user
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: http://aaa:343/bbb
+`, map[string]*UserInfo{
+		"foo": {
+			Username:  "foo",
+			Password:  "bar",
+			URLPrefix: "http://aaa:343/bbb",
+		},
+	})
+
+	// Multiple users
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo
+- username: bar
+  url_prefix: https://bar/x///
+`, map[string]*UserInfo{
+		"foo": {
+			Username:  "foo",
+			URLPrefix: "http://foo",
+		},
+		"bar": {
+			Username:  "bar",
+			URLPrefix: "https://bar/x",
+		},
+	})
+}
+
+func removeMetrics(m map[string]*UserInfo) {
+	for _, info := range m {
+		info.requests = nil
+	}
+}

app/vmauth/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8427
+
+ENTRYPOINT ["/vmauth-prod"]
+ARG src_binary
+COPY $src_binary ./vmauth-prod

app/vmauth/example_config.yml

@@ -0,0 +1,31 @@
+# Arbitrary number of usernames may be put here.
+# Usernames must be unique.
+
+users:
+
+  # The user for querying local single-node VictoriaMetrics.
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://localhost:8428 .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query
+- username: "local-single-node"
+  password: "***"
+  url_prefix: "http://localhost:8428"
+
+  # The user for querying account 123 in VictoriaMetrics cluster
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vmselect:8481/select/123/prometheus .
+  # For example, http://vmauth:8427/api/v1/query is routed to http://vmselect:8481/select/123/prometheus/api/v1/select
+- username: "cluster-select-account-123"
+  password: "***"
+  url_prefix: "http://vmselect:8481/select/123/prometheus"
+
+  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+  # All the reuqests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # will be routed to http://vminsert:8480/insert/42/prometheus .
+  # For example, http://vmauth:8427/api/v1/write is routed to http://vminsert:8480/insert/42/prometheus/api/v1/write
+- username: "cluster-insert-account-42"
+  password: "***"
+  url_prefix: "http://vminsert:8480/insert/42/prometheus"
+

app/vmauth/main.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting vmauth at %q...", *httpListenAddr)
+	startTime := time.Now()
+	initAuthConfig()
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	stopAuthConfig()
+	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	username, password, ok := r.BasicAuth()
+	if !ok {
+		httpserver.Errorf(w, "Missing `Authorization: Basic *` header")
+		return true
+	}
+	ac := authConfig.Load().(map[string]*UserInfo)
+	info := ac[username]
+	if info == nil || info.Password != password {
+		httpserver.Errorf(w, "Cannot find the provided username %q or password in config", username)
+		return true
+	}
+	info.requests.Inc()
+
+	targetURL := createTargetURL(info.URLPrefix, r.URL)
+	if _, err := url.Parse(targetURL); err != nil {
+		httpserver.Errorf(w, "Invalid targetURL=%q: %s", targetURL, err)
+		return true
+	}
+	r.Header.Set("vm-target-url", targetURL)
+	reverseProxy.ServeHTTP(w, r)
+	return true
+}
+
+var reverseProxy = &httputil.ReverseProxy{
+	Director: func(r *http.Request) {
+		targetURL := r.Header.Get("vm-target-url")
+		target, err := url.Parse(targetURL)
+		if err != nil {
+			logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
+		}
+		r.URL = target
+	},
+	Transport: func() *http.Transport {
+		tr := http.DefaultTransport.(*http.Transport).Clone()
+		// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+		tr.DisableCompression = true
+		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+		tr.ForceAttemptHTTP2 = false
+		return tr
+	}(),
+	FlushInterval: time.Second,
+	ErrorLog:      logger.StdErrorLogger(),
+}
+
+func usage() {
+	const s = `
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}

app/vmauth/target_url.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"net/url"
+	"path"
+	"strings"
+)
+
+func createTargetURL(prefix string, u *url.URL) string {
+	// Prevent from attacks with using `..` in r.URL.Path
+	u.Path = path.Clean(u.Path)
+	if !strings.HasPrefix(u.Path, "/") {
+		u.Path = "/" + u.Path
+	}
+	return prefix + u.RequestURI()
+}

app/vmauth/target_url_test.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestCreateTargetURL(t *testing.T) {
+	f := func(prefix, requestURI, expectedTarget string) {
+		t.Helper()
+		u, err := url.Parse(requestURI)
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", requestURI, err)
+		}
+		target := createTargetURL(prefix, u)
+		if target != expectedTarget {
+			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
+		}
+	}
+	f("http://foo.bar", "", "http://foo.bar/.")
+	f("http://foo.bar", "/", "http://foo.bar/")
+	f("http://foo.bar", "a/b?c=d", "http://foo.bar/a/b?c=d")
+	f("https://sss:3894/x/y", "/z", "https://sss:3894/x/y/z")
+	f("https://sss:3894/x/y", "/../../aaa", "https://sss:3894/x/y/aaa")
+	f("https://sss:3894/x/y", "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s/../d")
+}

app/vmbackup/Makefile

@@ -0,0 +1,70 @@
+# All these commands must run from repository root.
+
+vmbackup:
+	APP_NAME=vmbackup $(MAKE) app-local
+
+vmbackup-race:
+	APP_NAME=vmbackup RACE=-race $(MAKE) app-local
+
+vmbackup-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker
+
+vmbackup-pure-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-pure
+
+vmbackup-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-amd64
+
+vmbackup-arm-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm
+
+vmbackup-arm64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm64
+
+vmbackup-ppc64le-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-ppc64le
+
+vmbackup-386-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-386
+
+package-vmbackup:
+	APP_NAME=vmbackup $(MAKE) package-via-docker
+
+package-vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-pure
+
+package-vmbackup-amd64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-amd64
+
+package-vmbackup-arm:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm
+
+package-vmbackup-arm64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm64
+
+package-vmbackup-ppc64le:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le
+
+package-vmbackup-386:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-386
+
+publish-vmbackup:
+	APP_NAME=vmbackup $(MAKE) publish-via-docker
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure
+
+vmbackup-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-amd64 ./app/vmbackup
+
+vmbackup-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+
+vmbackup-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+
+vmbackup-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup
+
+vmbackup-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup

app/vmbackup/README.md

@@ -0,0 +1,209 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+
+See also [vmbackuper](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) tool built on top of `vmbackup`. This tool simplifies
+creation of hourly, daily, weekly and monthly backups.
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+See also [vmbackuper tool](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for automating smart backups.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+* Backups created from [single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md) cannot be restored
+  at [cluster VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md) and vice versa.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshot.createURL string
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+  -snapshot.deleteURL string
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmbackup
+```

app/vmbackup/deployment/Dockerfile

@@ -0,0 +1,6 @@
+ARG base_image
+FROM $base_image
+
+ENTRYPOINT ["/vmbackup-prod"]
+ARG src_binary
+COPY $src_binary ./vmbackup-prod

app/vmbackup/main.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup."+
+		"Example: http://victoriametrics:8428/snaphsot/create")
+	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted."+
+		"Example: http://victoriametrics:8428/snaphsot/delete")
+	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
+	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
+	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+
+	if len(*snapshotCreateURL) > 0 {
+		logger.Infof("%s", "Snapshots enabled")
+		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
+		if len(*snapshotDeleteURL) <= 0 {
+			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
+			if err != nil {
+				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
+			}
+		}
+		logger.Infof("Snapshot delete url %s", *snapshotDeleteURL)
+
+		name, err := snapshot.Create(*snapshotCreateURL)
+		if err != nil {
+			logger.Fatalf("%s", err)
+		}
+		err = flag.Set("snapshotName", name)
+		if err != nil {
+			logger.Fatalf("Failed to set snapshotName flag: %v", err)
+		}
+
+		defer func() {
+			err := snapshot.Delete(*snapshotDeleteURL, name)
+			if err != nil {
+				logger.Fatalf("%s", err)
+			}
+		}()
+	}
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	originFS, err := newOriginFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Backup{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+		Origin:      originFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+or local filesystem. Backed up data can be restored with vmrestore.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newSrcFS() (*fslocal.FS, error) {
+	if len(*snapshotName) == 0 {
+		return nil, fmt.Errorf("`-snapshotName` or `-snapshot.createURL` must be provided")
+	}
+	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
+
+	// Verify the snapshot exists.
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
+	}
+
+	fs := &fslocal.FS{
+		Dir:               snapshotPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newDstFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*dst)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+	}
+	return fs, nil
+}
+
+func newOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, nil
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+	}
+	return fs, nil
+}

app/vmbackup/snapshot/snapshot.go

@@ -0,0 +1,91 @@
+package snapshot
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+type snapshot struct {
+	Status   string `json:"status"`
+	Snapshot string `json:"snapshot"`
+	Msg      string `json:"msg"`
+}
+
+// Create creates a snapshot and the provided api endpoint and returns
+// the snapshot name
+func Create(createSnapshotURL string) (string, error) {
+	logger.Infof("%s", "Creating snapshot")
+	u, err := url.Parse(createSnapshotURL)
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return "", err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s created", snap.Snapshot)
+		return snap.Snapshot, nil
+	} else if snap.Status == "error" {
+		return "", errors.New(snap.Msg)
+	} else {
+		return "", fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}
+
+// Delete deletes a snapshot and the provided api endpoint returns any failure
+func Delete(deleteSnapshotURL string, snapshotName string) error {
+	logger.Infof("Deleting snapshot %s", snapshotName)
+	formData := url.Values{
+		"snapshot": {snapshotName},
+	}
+
+	u, err := url.Parse(deleteSnapshotURL)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.PostForm(u.String(), formData)
+	if err != nil {
+		return err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s deleted", snapshotName)
+		return nil
+	} else if snap.Status == "error" {
+		return errors.New(snap.Msg)
+	} else {
+		return fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}

app/vmbackup/snapshot/snapshot_test.go

@@ -0,0 +1,106 @@
+package snapshot
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestCreateSnapshot(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"ok","snapshot":"mysnapshot"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err != nil {
+		t.Fatalf("Failed taking snapshot: %v", err)
+	}
+
+	if snapshotName != "mysnapshot" {
+		t.Fatalf("Snapshot name is not correct, got %v", snapshotName)
+	}
+}
+
+func TestCreateSnapshotFailed(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"error","msg":"I am unwell"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err == nil {
+		t.Fatalf("Snapshot did not fail, got snapshot: %v", snapshotName)
+	}
+}
+
+func TestDeleteSnapshot(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"ok"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err != nil {
+		t.Fatalf("Failed to delete snapshot: %v", err)
+	}
+}
+
+func TestDeleteSnapshotFailed(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"error", "msg":"failed to delete"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err == nil {
+		t.Fatalf("Snapshot should have failed, got: %v", err)
+	}
+}

app/vminsert/Makefile

@@ -0,0 +1,34 @@
+# All these commands must run from repository root.
+
+run-vminsert:
+	APP_NAME=vminsert ARGS='-storageNode=localhost:8400' $(MAKE) run-via-docker
+
+vminsert:
+	APP_NAME=vminsert $(MAKE) app-local
+
+vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) app-local
+
+vminsert-prod:
+	APP_NAME=vminsert $(MAKE) app-via-docker
+
+vminsert-pure-prod:
+	APP_NAME=vminsert $(MAKE) app-via-docker-pure
+
+vminsert-prod-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) app-via-docker
+
+vminsert-pure:
+	APP_NAME=vminsert $(MAKE) app-local-pure
+
+package-vminsert:
+	APP_NAME=vminsert $(MAKE) package-via-docker
+
+package-vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) package-via-docker
+
+publish-vminsert:
+	APP_NAME=vminsert $(MAKE) publish-via-docker
+
+publish-vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) publish-via-docker

app/vminsert/README.md

@@ -1 +1 @@
-`vminsert` routes the ingested data to `vmstorage`.
+`vminsert` routes the ingested data to `vmstorage` nodes.

app/vminsert/common/insert_ctx.go

@@ -1,106 +0,0 @@
-package common
-
-import (
-	"fmt"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-)
-
-// InsertCtx contains common bits for data points insertion.
-type InsertCtx struct {
-	Labels []prompb.Label
-
-	mrs            []storage.MetricRow
-	metricNamesBuf []byte
-}
-
-// Reset resets ctx for future fill with rowsLen rows.
-func (ctx *InsertCtx) Reset(rowsLen int) {
-	for _, label := range ctx.Labels {
-		label.Name = nil
-		label.Value = nil
-	}
-	ctx.Labels = ctx.Labels[:0]
-
-	for i := range ctx.mrs {
-		mr := &ctx.mrs[i]
-		mr.MetricNameRaw = nil
-	}
-	ctx.mrs = ctx.mrs[:0]
-
-	if n := rowsLen - cap(ctx.mrs); n > 0 {
-		ctx.mrs = append(ctx.mrs[:cap(ctx.mrs)], make([]storage.MetricRow, n)...)
-	}
-	ctx.mrs = ctx.mrs[:rowsLen]
-	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
-}
-
-func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
-	start := len(ctx.metricNamesBuf)
-	ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
-	ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
-	metricNameRaw := ctx.metricNamesBuf[start:]
-	return metricNameRaw[:len(metricNameRaw):len(metricNameRaw)]
-}
-
-// WriteDataPoint writes (timestamp, value) with the given prefix and lables into ctx buffer.
-func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) {
-	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
-	ctx.addRow(metricNameRaw, timestamp, value)
-}
-
-// WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer.
-//
-// It returns metricNameRaw for the given labels if len(metricNameRaw) == 0.
-func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) []byte {
-	if len(metricNameRaw) == 0 {
-		metricNameRaw = ctx.marshalMetricNameRaw(nil, labels)
-	}
-	ctx.addRow(metricNameRaw, timestamp, value)
-	return metricNameRaw
-}
-
-func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float64) {
-	mrs := ctx.mrs
-	if cap(mrs) > len(mrs) {
-		mrs = mrs[:len(mrs)+1]
-	} else {
-		mrs = append(mrs, storage.MetricRow{})
-	}
-	mr := &mrs[len(mrs)-1]
-	ctx.mrs = mrs
-	mr.MetricNameRaw = metricNameRaw
-	mr.Timestamp = timestamp
-	mr.Value = value
-}
-
-// AddLabel adds (name, value) label to ctx.Labels.
-//
-// name and value must exist until ctx.Labels is used.
-func (ctx *InsertCtx) AddLabel(name, value string) {
-	labels := ctx.Labels
-	if cap(labels) > len(labels) {
-		labels = labels[:len(labels)+1]
-	} else {
-		labels = append(labels, prompb.Label{})
-	}
-	label := &labels[len(labels)-1]
-
-	// Do not copy name and value contents for performance reasons.
-	// This reduces GC overhead on the number of objects and allocations.
-	label.Name = bytesutil.ToUnsafeBytes(name)
-	label.Value = bytesutil.ToUnsafeBytes(value)
-
-	ctx.Labels = labels
-}
-
-// FlushBufs flushes buffered rows to the underlying storage.
-func (ctx *InsertCtx) FlushBufs() error {
-	if err := vmstorage.AddRows(ctx.mrs); err != nil {
-		return fmt.Errorf("cannot store metrics: %s", err)
-	}
-	return nil
-}

app/vminsert/concurrencylimiter/concurrencylimiter.go

@@ -1,34 +0,0 @@
-package concurrencylimiter
-
-import (
-	"fmt"
-	"runtime"
-	"time"
-)
-
-var (
-	// ch is the channel for limiting concurrent inserts.
-	// Put an item into it before performing an insert and remove
-	// the item after the insert is complete.
-	ch = make(chan struct{}, runtime.GOMAXPROCS(-1)*2)
-
-	// waitDuration is the amount of time to wait until at least a single
-	// concurrent insert out of cap(Ch) inserts is complete.
-	waitDuration = time.Second * 30
-)
-
-// Do calls f with the limited concurrency.
-func Do(f func() error) error {
-	// Limit the number of conurrent inserts in order to prevent from excess
-	// memory usage and CPU trashing.
-	t := time.NewTimer(waitDuration)
-	select {
-	case ch <- struct{}{}:
-		t.Stop()
-		err := f()
-		<-ch
-		return err
-	case <-t.C:
-		return fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase the number of CPUs or reduce the load", cap(ch))
-	}
-}

app/vminsert/csvimport/request_handler.go

@@ -0,0 +1,48 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes /api/v1/import/csv requests.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(at, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+	rowsInserted.Get(at).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8480
+
+ENTRYPOINT ["/vminsert-prod"]
+ARG src_binary
+COPY $src_binary ./vminsert-prod

app/vminsert/graphite/request_handler.go

@@ -1,166 +1,63 @@
 package graphite
 
 import (
-	"bytes"
-	"fmt"
 	"io"
-	"net"
-	"runtime"
-	"sync"
-	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
 )
 
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="graphite"}`)
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="graphite"}`)
+)
 
-// insertHandler processes remote write for graphite plaintext protocol.
+// InsertHandler processes remote write for graphite plaintext protocol.
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
-func insertHandler(r io.Reader) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
+func InsertHandler(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
 	})
 }
 
-func insertHandlerInternal(r io.Reader) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	atCopy := *at
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if atCopy.AccountID == 0 {
+				// Multi-tenancy support via custom tags.
+				// Do not allow overriding AccountID and ProjectID from atCopy for security reasons.
+				if tag.Key == "VictoriaMetrics_AccountID" {
+					atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+				if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" {
+					atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+			}
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(&atCopy, ctx.Labels, r.Timestamp, r.Value); err != nil {
 			return err
 		}
 	}
-	return ctx.Error()
+	// Assume that all the rows for a single connection belong to the same (AccountID, ProjectID).
+	rowsInserted.Get(&atCopy).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
 }
-
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
-	ic := &ctx.Common
-	ic.Reset(len(rows))
-	for i := range rows {
-		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
-		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
-	}
-	rowsInserted.Add(len(rows))
-	return ic.FlushBufs()
-}
-
-const maxReadPacketSize = 4 * 1024 * 1024
-
-const flushTimeout = 3 * time.Second
-
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	graphiteReadCalls.Inc()
-	if ctx.err != nil {
-		return false
-	}
-	if c, ok := r.(net.Conn); ok {
-		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			graphiteReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
-			return false
-		}
-	}
-	lr := io.LimitReader(r, maxReadPacketSize)
-	ctx.reqBuf.Reset()
-	ctx.reqBuf.B = append(ctx.reqBuf.B[:0], ctx.tailBuf...)
-	n, err := io.CopyBuffer(&ctx.reqBuf, lr, ctx.copyBuf[:])
-	if err != nil {
-		if ne, ok := err.(net.Error); ok && ne.Timeout() {
-			// Flush the read data on timeout and try reading again.
-		} else {
-			graphiteReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot read graphite plaintext protocol data: %s", err)
-			return false
-		}
-	} else if n < maxReadPacketSize {
-		// Mark the end of stream.
-		ctx.err = io.EOF
-	}
-
-	// Parse all the rows until the last newline in ctx.reqBuf.B
-	nn := bytes.LastIndexByte(ctx.reqBuf.B, '\n')
-	ctx.tailBuf = ctx.tailBuf[:0]
-	if nn >= 0 {
-		ctx.tailBuf = append(ctx.tailBuf[:0], ctx.reqBuf.B[nn+1:]...)
-		ctx.reqBuf.B = ctx.reqBuf.B[:nn]
-	}
-	if err = ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf.B)); err != nil {
-		graphiteUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal graphite plaintext protocol data with size %d: %s", len(ctx.reqBuf.B), err)
-		return false
-	}
-
-	// Convert timestamps from seconds to milliseconds
-	for i := range ctx.Rows.Rows {
-		ctx.Rows.Rows[i].Timestamp *= 1e3
-	}
-	return true
-}
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf  bytesutil.ByteBuffer
-	tailBuf []byte
-	copyBuf [16 * 1024]byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf.Reset()
-	ctx.tailBuf = ctx.tailBuf[:0]
-
-	ctx.err = nil
-}
-
-var (
-	graphiteReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
-	graphiteReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
-	graphiteUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="graphite"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/influx/parser_timing_test.go

@@ -1,20 +0,0 @@
-package influx
-
-import (
-	"fmt"
-	"testing"
-)
-
-func BenchmarkRowsUnmarshal(b *testing.B) {
-	s := `cpu usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 1234556768`
-	b.SetBytes(int64(len(s)))
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var rows Rows
-		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
-			}
-		}
-	})
-}

app/vminsert/influx/request_handler.go

@@ -1,222 +1,130 @@
 package influx
 
 import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
+	"flag"
 	"io"
 	"net/http"
 	"runtime"
 	"sync"
-	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
 )
 
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="influx"}`)
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
 
-// InsertHandler processes remote write for influx line protocol.
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
 //
-// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandler(req *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req)
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows, true)
+		})
 	})
 }
 
-func insertHandlerInternal(req *http.Request) error {
-	influxReadCalls.Inc()
-
-	r := req.Body
-	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := getGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped influx line protocol data: %s", err)
-		}
-		defer putGzipReader(zr)
-		r = zr
-	}
-
-	q := req.URL.Query()
-	tsMultiplier := int64(1e6)
-	switch q.Get("precision") {
-	case "ns":
-		tsMultiplier = 1e6
-	case "u":
-		tsMultiplier = 1e3
-	case "ms":
-		tsMultiplier = 1
-	case "s":
-		tsMultiplier = -1e3
-	case "m":
-		tsMultiplier = -1e3 * 60
-	case "h":
-		tsMultiplier = -1e3 * 3600
-	}
-
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r, tsMultiplier) {
-		if err := ctx.InsertRows(db); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows, false)
+		})
+	})
 }
 
-func (ctx *pushCtx) InsertRows(db string) error {
-	rows := ctx.Rows.Rows
-	rowsLen := 0
-	for i := range rows {
-		rowsLen += len(rows[i].Tags)
-	}
+func insertRows(at *auth.Token, db string, rows []parser.Row, mayOverrideAccountProjectID bool) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
 	ic := &ctx.Common
-	ic.Reset(rowsLen)
+	ic.Reset() // This line is required for initializing ic internals.
+	rowsTotal := 0
+	atCopy := *at
 	for i := range rows {
 		r := &rows[i]
 		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("db", db)
+		hasDBLabel := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
+			if mayOverrideAccountProjectID {
+				// Multi-tenancy support via custom tags.
+				if tag.Key == "VictoriaMetrics_AccountID" {
+					atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+				if tag.Key == "VictoriaMetrics_ProjectID" {
+					atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+			}
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
 			ic.AddLabel(tag.Key, tag.Value)
 		}
-		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+		if len(db) > 0 && !hasDBLabel {
+			ic.AddLabel("db", db)
+		}
+		ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], atCopy.AccountID, atCopy.ProjectID, ic.Labels)
+		metricNameBufLen := len(ic.MetricNameBuf)
 		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
-		ctx.metricGroupBuf = append(ctx.metricGroupBuf, '.')
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
 		metricGroupPrefixLen := len(ctx.metricGroupBuf)
+		ic.AddLabel("", "placeholder")
+		placeholderLabel := &ic.Labels[len(ic.Labels)-1]
 		for j := range r.Fields {
 			f := &r.Fields[j]
-			ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+			if !skipFieldKey {
+				ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+			}
 			metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
-			ic.Labels = ic.Labels[:0]
+			ic.Labels = ic.Labels[:len(ic.Labels)-1]
 			ic.AddLabel("", metricGroup)
-			ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels[:1], r.Timestamp, f.Value)
+			ic.MetricNameBuf = storage.MarshalMetricLabelRaw(ic.MetricNameBuf[:metricNameBufLen], placeholderLabel)
+			storageNodeIdx := ic.GetStorageNodeIdx(&atCopy, ic.Labels)
+			if err := ic.WriteDataPointExt(&atCopy, storageNodeIdx, ic.MetricNameBuf, r.Timestamp, f.Value); err != nil {
+				return err
+			}
 		}
-		rowsInserted.Add(len(r.Fields))
+		rowsTotal += len(r.Fields)
 	}
+	rowsInserted.Get(&atCopy).Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
 	return ic.FlushBufs()
 }
 
-func getGzipReader(r io.Reader) (*gzip.Reader, error) {
-	v := gzipReaderPool.Get()
-	if v == nil {
-		return gzip.NewReader(r)
-	}
-	zr := v.(*gzip.Reader)
-	if err := zr.Reset(r); err != nil {
-		return nil, err
-	}
-	return zr, nil
-}
-
-func putGzipReader(zr *gzip.Reader) {
-	_ = zr.Close()
-	gzipReaderPool.Put(zr)
-}
-
-var gzipReaderPool sync.Pool
-
-const maxReadPacketSize = 4 * 1024 * 1024
-
-func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
-	if ctx.err != nil {
-		return false
-	}
-	lr := io.LimitReader(r, maxReadPacketSize)
-	ctx.reqBuf.Reset()
-	ctx.reqBuf.B = append(ctx.reqBuf.B[:0], ctx.tailBuf...)
-	n, err := io.CopyBuffer(&ctx.reqBuf, lr, ctx.copyBuf[:])
-	if err != nil {
-		influxReadErrors.Inc()
-		ctx.err = fmt.Errorf("cannot read influx line protocol data: %s", err)
-		return false
-	}
-	if n < maxReadPacketSize {
-		// Mark the end of stream.
-		ctx.err = io.EOF
-	}
-
-	// Parse all the rows until the last newline in ctx.reqBuf.B
-	nn := bytes.LastIndexByte(ctx.reqBuf.B, '\n')
-	ctx.tailBuf = ctx.tailBuf[:0]
-	if nn >= 0 {
-		ctx.tailBuf = append(ctx.tailBuf[:0], ctx.reqBuf.B[nn+1:]...)
-		ctx.reqBuf.B = ctx.reqBuf.B[:nn]
-	}
-	if err = ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf.B)); err != nil {
-		influxUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal influx line protocol data with size %d: %s", len(ctx.reqBuf.B), err)
-		return false
-	}
-
-	// Adjust timestamps according to tsMultiplier
-	currentTs := time.Now().UnixNano() / 1e6
-	if tsMultiplier >= 1 {
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp /= tsMultiplier
-			}
-		}
-	} else if tsMultiplier < 0 {
-		tsMultiplier = -tsMultiplier
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp *= tsMultiplier
-			}
-		}
-	}
-	return true
-}
-
-var (
-	influxReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
-	influxReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
-	influxUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="influx"}`)
-)
-
 type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf         bytesutil.ByteBuffer
-	tailBuf        []byte
-	copyBuf        [16 * 1024]byte
-	metricNameBuf  []byte
+	Common         netstorage.InsertCtx
 	metricGroupBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
 }
 
 func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-
-	ctx.reqBuf.Reset()
-	ctx.tailBuf = ctx.tailBuf[:0]
-	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+	ctx.Common.Reset()
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
-
-	ctx.err = nil
 }
 
 func getPushCtx() *pushCtx {

app/vminsert/main.go

@@ -1,69 +1,191 @@
-package vminsert
+package main
 
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
-	"strings"
+	"os"
+	"sync/atomic"
+	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	graphiteListenAddr   = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	opentsdbListenAddr   = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty")
-	maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	httpListenAddr         = flag.String("httpListenAddr", ":8480", "Address to listen for http connections")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
+	storageNodes           = flagutil.NewArray("storageNode", "Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400")
 )
 
-// Init initializes vminsert.
-func Init() {
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	logger.Infof("initializing netstorage for storageNodes %s...", *storageNodes)
+	startTime := time.Now()
+	if len(*storageNodes) == 0 {
+		logger.Fatalf("missing -storageNode arg")
+	}
+	netstorage.InitStorageNodes(*storageNodes)
+	logger.Infof("successfully initialized netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return influx.InsertHandlerForReader(&at, r)
+		})
+	}
 	if len(*graphiteListenAddr) > 0 {
-		go graphite.Serve(*graphiteListenAddr)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return graphite.InsertHandler(&at, r)
+		})
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		go opentsdb.Serve(*opentsdbListenAddr)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return opentsdb.InsertHandler(&at, r)
+		}, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
 	}
-}
 
-// Stop stops vminsert.
-func Stop() {
+	go func() {
+		httpserver.Serve(*httpListenAddr, requestHandler)
+	}()
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+
+	logger.Infof("gracefully shutting down http service at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http service: %s", err)
+	}
+	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
 	if len(*graphiteListenAddr) > 0 {
-		graphite.Stop()
+		graphiteServer.MustStop()
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdb.Stop()
+		opentsdbServer.MustStop()
 	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+
+	logger.Infof("shutting down neststorage...")
+	startTime = time.Now()
+	netstorage.Stop()
+	logger.Infof("successfully stopped netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the vminsert has been stopped")
 }
 
-// RequestHandler is a handler for Prometheus remote storage write API
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.Replace(r.URL.Path, "//", "/", -1)
-	switch path {
-	case "/api/v1/write":
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	p, err := httpserver.ParsePath(r.URL.Path)
+	if err != nil {
+		httpserver.Errorf(w, "cannot parse path %q: %s", r.URL.Path, err)
+		return true
+	}
+	if p.Prefix != "insert" {
+		// This is not our link.
+		return false
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, "auth error: %s", err)
+		return true
+	}
+
+	switch p.Suffix {
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := prometheus.InsertHandler(r, int64(*maxInsertRequestSize)); err != nil {
+		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/write", "/api/v2/write":
+	case "prometheus/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(at, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(at, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/write", "influx/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandler(r); err != nil {
+		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
 			influxWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/query":
-		// Emulate fake response for influx query
+	case "influx/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
 		influxQueryRequests.Inc()
 		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
 		return true
@@ -74,11 +196,27 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 var (
-	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/write", protocol="prometheus"}`)
-	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/write", protocol="prometheus"}`)
+	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
+	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/api/v1/import", protocol="vmimport"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/query", protocol="influx"}`)
+	csvimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/api/v1/import/csv", protocol="csvimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/influx/", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+
+	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+	})
 )

app/vminsert/netstorage/insert_ctx.go

@@ -0,0 +1,177 @@
+package netstorage
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	xxhash "github.com/cespare/xxhash/v2"
+	jump "github.com/lithammer/go-jump-consistent-hash"
+)
+
+// InsertCtx is a generic context for inserting data.
+//
+// InsertCtx.Reset must be called before the first usage.
+type InsertCtx struct {
+	Labels        []prompb.Label
+	MetricNameBuf []byte
+
+	bufRowss  []bufRows
+	labelsBuf []byte
+}
+
+type bufRows struct {
+	buf  []byte
+	rows int
+}
+
+func (br *bufRows) reset() {
+	br.buf = br.buf[:0]
+	br.rows = 0
+}
+
+func (br *bufRows) pushTo(sn *storageNode) error {
+	bufLen := len(br.buf)
+	err := sn.push(br.buf, br.rows)
+	br.reset()
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot send %d bytes to storageNode %q: %s", bufLen, sn.dialer.Addr(), err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
+// Reset resets ctx.
+func (ctx *InsertCtx) Reset() {
+	for _, label := range ctx.Labels {
+		label.Name = nil
+		label.Value = nil
+	}
+	ctx.Labels = ctx.Labels[:0]
+	ctx.MetricNameBuf = ctx.MetricNameBuf[:0]
+
+	if ctx.bufRowss == nil {
+		ctx.bufRowss = make([]bufRows, len(storageNodes))
+	}
+	for i := range ctx.bufRowss {
+		ctx.bufRowss[i].reset()
+	}
+	ctx.labelsBuf = ctx.labelsBuf[:0]
+}
+
+// AddLabelBytes adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = name
+	label.Value = value
+
+	ctx.Labels = labels
+}
+
+// AddLabel adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabel(name, value string) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = bytesutil.ToUnsafeBytes(name)
+	label.Value = bytesutil.ToUnsafeBytes(value)
+
+	ctx.Labels = labels
+}
+
+// WriteDataPoint writes (timestamp, value) data point with the given at and labels to ctx buffer.
+func (ctx *InsertCtx) WriteDataPoint(at *auth.Token, labels []prompb.Label, timestamp int64, value float64) error {
+	ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, labels)
+	storageNodeIdx := ctx.GetStorageNodeIdx(at, labels)
+	return ctx.WriteDataPointExt(at, storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
+}
+
+// WriteDataPointExt writes the given metricNameRaw with (timestmap, value) to ctx buffer with the given storageNodeIdx.
+func (ctx *InsertCtx) WriteDataPointExt(at *auth.Token, storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	br := &ctx.bufRowss[storageNodeIdx]
+	sn := storageNodes[storageNodeIdx]
+	bufNew := storage.MarshalMetricRow(br.buf, metricNameRaw, timestamp, value)
+	if len(bufNew) >= maxBufSizePerStorageNode {
+		// Send buf to storageNode, since it is too big.
+		if err := br.pushTo(sn); err != nil {
+			return err
+		}
+		br.buf = storage.MarshalMetricRow(bufNew[:0], metricNameRaw, timestamp, value)
+	} else {
+		br.buf = bufNew
+	}
+	br.rows++
+	return nil
+}
+
+// FlushBufs flushes ctx bufs to remote storage nodes.
+func (ctx *InsertCtx) FlushBufs() error {
+	var firstErr error
+	for i := range ctx.bufRowss {
+		br := &ctx.bufRowss[i]
+		if len(br.buf) == 0 {
+			continue
+		}
+		if err := br.pushTo(storageNodes[i]); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// GetStorageNodeIdx returns storage node index for the given at and labels.
+//
+// The returned index must be passed to WriteDataPoint.
+func (ctx *InsertCtx) GetStorageNodeIdx(at *auth.Token, labels []prompb.Label) int {
+	if len(storageNodes) == 1 {
+		// Fast path - only a single storage node.
+		return 0
+	}
+
+	buf := ctx.labelsBuf[:0]
+	buf = encoding.MarshalUint32(buf, at.AccountID)
+	buf = encoding.MarshalUint32(buf, at.ProjectID)
+	for i := range labels {
+		label := &labels[i]
+		buf = marshalBytesFast(buf, label.Name)
+		buf = marshalBytesFast(buf, label.Value)
+	}
+	h := xxhash.Sum64(buf)
+	ctx.labelsBuf = buf
+
+	idx := int(jump.Hash(h, int32(len(storageNodes))))
+	return idx
+}
+
+func marshalBytesFast(dst []byte, s []byte) []byte {
+	dst = encoding.MarshalUint16(dst, uint16(len(s)))
+	dst = append(dst, s...)
+	return dst
+}

app/vminsert/netstorage/insert_ctx_pool.go

@@ -0,0 +1,36 @@
+package netstorage
+
+import (
+	"runtime"
+	"sync"
+)
+
+// GetInsertCtx returns InsertCtx from the pool.
+//
+// Call PutInsertCtx for returning it to the pool.
+func GetInsertCtx() *InsertCtx {
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
+	}
+}
+
+// PutInsertCtx returns ctx to the pool.
+//
+// ctx cannot be used after the call.
+func PutInsertCtx(ctx *InsertCtx) {
+	ctx.Reset()
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
+}
+
+var insertCtxPool sync.Pool
+var insertCtxPoolCh = make(chan *InsertCtx, runtime.GOMAXPROCS(-1))

app/vminsert/netstorage/netstorage.go

@@ -0,0 +1,648 @@
+package netstorage
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/handshake"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var (
+	disableRPCCompression = flag.Bool(`rpc.disableCompression`, false, "Disable compression of RPC traffic. This reduces CPU usage at the cost of higher network bandwidth usage")
+	replicationFactor     = flag.Int("replicationFactor", 1, "Replication factor for the ingested data, i.e. how many copies to make among distinct -storageNode instances. "+
+		"Note that vmselect must run with -dedup.minScrapeInterval=1ms for data de-duplication when replicationFactor is greater than 1. "+
+		"Higher values for -dedup.minScrapeInterval at vmselect is OK")
+)
+
+func (sn *storageNode) isBroken() bool {
+	return atomic.LoadUint32(&sn.broken) != 0
+}
+
+// push pushes buf to sn internal bufs.
+//
+// This function doesn't block on fast path.
+// It may block only if all the storageNodes cannot handle the incoming ingestion rate.
+// This blocking provides backpressure to the caller.
+//
+// The function falls back to sending data to other vmstorage nodes
+// if sn is currently unavailable or overloaded.
+//
+// rows is the number of rows in the buf.
+func (sn *storageNode) push(buf []byte, rows int) error {
+	if len(buf) > maxBufSizePerStorageNode {
+		logger.Panicf("BUG: len(buf)=%d cannot exceed %d", len(buf), maxBufSizePerStorageNode)
+	}
+	sn.rowsPushed.Add(rows)
+
+	if sn.isBroken() {
+		// The vmstorage node is temporarily broken. Re-route buf to healthy vmstorage nodes.
+		if err := addToReroutedBuf(buf, rows); err != nil {
+			return fmt.Errorf("%d rows dropped because the current vsmtorage is unavailable and %s", rows, err)
+		}
+		sn.rowsReroutedFromHere.Add(rows)
+		return nil
+	}
+
+	sn.brLock.Lock()
+	if len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode {
+		// Fast path: the buf contents fits sn.buf.
+		sn.br.buf = append(sn.br.buf, buf...)
+		sn.br.rows += rows
+		sn.brLock.Unlock()
+		return nil
+	}
+	sn.brLock.Unlock()
+
+	// Slow path: the buf contents doesn't fit sn.buf.
+	// This means that the current vmstorage is slow or will become broken soon.
+	// Re-route buf to healthy vmstorage nodes.
+	if err := addToReroutedBuf(buf, rows); err != nil {
+		return fmt.Errorf("%d rows dropped because the current vmstorage buf is full and %s", rows, err)
+	}
+	sn.rowsReroutedFromHere.Add(rows)
+	return nil
+}
+
+var closedCh = func() <-chan struct{} {
+	ch := make(chan struct{})
+	close(ch)
+	return ch
+}()
+
+func (sn *storageNode) run(stopCh <-chan struct{}, snIdx int) {
+	replicas := *replicationFactor
+	if replicas <= 0 {
+		replicas = 1
+	}
+	if replicas > len(storageNodes) {
+		replicas = len(storageNodes)
+	}
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	var br bufRows
+	brLastResetTime := fasttime.UnixTimestamp()
+	var waitCh <-chan struct{}
+	mustStop := false
+	for !mustStop {
+		sn.brLock.Lock()
+		bufLen := len(sn.br.buf)
+		sn.brLock.Unlock()
+		waitCh = nil
+		if len(br.buf) == 0 && bufLen > maxBufSizePerStorageNode/4 {
+			// Do not sleep, since sn.br.buf contains enough data to process.
+			waitCh = closedCh
+		}
+		select {
+		case <-stopCh:
+			mustStop = true
+			// Make sure the sn.buf is flushed last time before returning
+			// in order to send the remaining bits of data.
+		case <-ticker.C:
+		case <-waitCh:
+		}
+		if len(br.buf) == 0 {
+			sn.brLock.Lock()
+			sn.br, br = br, sn.br
+			sn.brLock.Unlock()
+		}
+		currentTime := fasttime.UnixTimestamp()
+		if len(br.buf) < cap(br.buf)/4 && currentTime-brLastResetTime > 10 {
+			// Free up capacity space occupied by br.buf in order to reduce memory usage after spikes.
+			br.buf = append(br.buf[:0:0], br.buf...)
+			brLastResetTime = currentTime
+		}
+		if len(br.buf) == 0 {
+			// Nothing to send. Just check sn health, so it could be returned to non-broken state.
+			sn.checkHealth()
+			continue
+		}
+
+		// Send br to replicas storageNodes starting from snIdx.
+		if !sendBufToReplicas(&br, snIdx, replicas) {
+			// do not reset br in the hope it will be sent next time.
+			continue
+		}
+		br.reset()
+	}
+}
+
+func sendBufToReplicas(br *bufRows, snIdx, replicas int) bool {
+	usedStorageNodes := make(map[*storageNode]bool, replicas)
+	for i := 0; i < replicas; i++ {
+		idx := snIdx + i
+		attempts := 0
+		for {
+			attempts++
+			if attempts > len(storageNodes) {
+				if i == 0 {
+					// The data wasn't replicated at all.
+					logger.Warnf("cannot push %d bytes with %d rows to storage nodes, since all the nodes are temporarily unavailable; "+
+						"re-trying to send the data soon", len(br.buf), br.rows)
+					return false
+				}
+				// The data is partially replicated, so just emit a warning and return true.
+				// We could retry sending the data again, but this may result in uncontrolled duplicate data.
+				// So it is better returning true.
+				rowsIncompletelyReplicatedTotal.Add(br.rows)
+				logger.Warnf("cannot make a copy #%d out of %d copies according to -replicationFactor=%d for %d bytes with %d rows, "+
+					"since a part of storage nodes is temporarily unavailable", i+1, replicas, *replicationFactor, len(br.buf), br.rows)
+				return true
+			}
+			if idx >= len(storageNodes) {
+				idx %= len(storageNodes)
+			}
+			sn := storageNodes[idx]
+			idx++
+			if usedStorageNodes[sn] {
+				// The br has been already replicated to sn. Skip it.
+				continue
+			}
+			if !sn.sendBufRows(br) {
+				// Cannot send data to sn. Go to the next sn.
+				continue
+			}
+			// Successfully sent data to sn.
+			usedStorageNodes[sn] = true
+			break
+		}
+	}
+	return true
+}
+
+func (sn *storageNode) checkHealth() {
+	sn.bcLock.Lock()
+	defer sn.bcLock.Unlock()
+
+	if !sn.isBroken() {
+		return
+	}
+	if sn.bc != nil {
+		logger.Panicf("BUG: sn.bc must be nil when sn is broken; got %p", sn.bc)
+	}
+	bc, err := sn.dial()
+	if err != nil {
+		logger.Warnf("cannot dial storageNode %q: %s", sn.dialer.Addr(), err)
+		return
+	}
+	sn.bc = bc
+	atomic.StoreUint32(&sn.broken, 0)
+}
+
+func (sn *storageNode) sendBufRows(br *bufRows) bool {
+	sn.bcLock.Lock()
+	defer sn.bcLock.Unlock()
+
+	if sn.bc == nil {
+		bc, err := sn.dial()
+		if err != nil {
+			// Mark sn as broken in order to prevent sending additional data to it until it is recovered.
+			atomic.StoreUint32(&sn.broken, 1)
+			logger.Warnf("cannot dial storageNode %q: %s", sn.dialer.Addr(), err)
+			return false
+		}
+		sn.bc = bc
+	}
+	err := sendToConn(sn.bc, br.buf)
+	if err == nil {
+		// Successfully sent buf to bc. Remove broken flag from sn.
+		atomic.StoreUint32(&sn.broken, 0)
+		sn.rowsSent.Add(br.rows)
+		return true
+	}
+	// Couldn't flush buf to sn. Mark sn as broken.
+	logger.Warnf("cannot send %d bytes with %d rows to %q: %s; re-routing this data to healthy storage nodes", len(br.buf), br.rows, sn.dialer.Addr(), err)
+	if err = sn.bc.Close(); err != nil {
+		logger.Warnf("cannot close connection to storageNode %q: %s", sn.dialer.Addr(), err)
+	}
+	sn.bc = nil
+	sn.connectionErrors.Inc()
+	atomic.StoreUint32(&sn.broken, 1)
+	return false
+}
+
+func sendToConn(bc *handshake.BufferedConn, buf []byte) error {
+	if len(buf) == 0 {
+		// Nothing to send
+		return nil
+	}
+	timeoutSeconds := len(buf) / 3e5
+	if timeoutSeconds < 60 {
+		timeoutSeconds = 60
+	}
+	timeout := time.Duration(timeoutSeconds) * time.Second
+	deadline := time.Now().Add(timeout)
+	if err := bc.SetWriteDeadline(deadline); err != nil {
+		return fmt.Errorf("cannot set write deadline to %s: %s", deadline, err)
+	}
+	// sizeBuf guarantees that the rows batch will be either fully
+	// read or fully discarded on the vmstorage side.
+	// sizeBuf is used for read optimization in vmstorage.
+	sizeBuf := sizeBufPool.Get()
+	defer sizeBufPool.Put(sizeBuf)
+	sizeBuf.B = encoding.MarshalUint64(sizeBuf.B[:0], uint64(len(buf)))
+	if _, err := bc.Write(sizeBuf.B); err != nil {
+		return fmt.Errorf("cannot write data size %d: %s", len(buf), err)
+	}
+	if _, err := bc.Write(buf); err != nil {
+		return fmt.Errorf("cannot write data with size %d: %s", len(buf), err)
+	}
+	if err := bc.Flush(); err != nil {
+		return fmt.Errorf("cannot flush data with size %d: %s", len(buf), err)
+	}
+
+	// Wait for `ack` from vmstorage.
+	// This guarantees that the message has been fully received by vmstorage.
+	deadline = time.Now().Add(timeout)
+	if err := bc.SetReadDeadline(deadline); err != nil {
+		return fmt.Errorf("cannot set read deadline for reading `ack` to vmstorage: %s", err)
+	}
+	if _, err := io.ReadFull(bc, sizeBuf.B[:1]); err != nil {
+		return fmt.Errorf("cannot read `ack` from vmstorage: %s", err)
+	}
+	if sizeBuf.B[0] != 1 {
+		return fmt.Errorf("unexpected `ack` received from vmstorage; got %d; want %d", sizeBuf.B[0], 1)
+	}
+	return nil
+}
+
+var sizeBufPool bytesutil.ByteBufferPool
+
+func (sn *storageNode) dial() (*handshake.BufferedConn, error) {
+	c, err := sn.dialer.Dial()
+	if err != nil {
+		sn.dialErrors.Inc()
+		return nil, err
+	}
+	compressionLevel := 1
+	if *disableRPCCompression {
+		compressionLevel = 0
+	}
+	bc, err := handshake.VMInsertClient(c, compressionLevel)
+	if err != nil {
+		_ = c.Close()
+		sn.handshakeErrors.Inc()
+		return nil, fmt.Errorf("handshake error: %s", err)
+	}
+	return bc, nil
+}
+
+func rerouteWorker(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	var br bufRows
+	brLastResetTime := fasttime.UnixTimestamp()
+	var waitCh <-chan struct{}
+	mustStop := false
+	for !mustStop {
+		reroutedBRLock.Lock()
+		bufLen := len(reroutedBR.buf)
+		reroutedBRLock.Unlock()
+		waitCh = nil
+		if len(br.buf) == 0 && bufLen > reroutedBufMaxSize/4 {
+			// Do not sleep if reroutedBR contains enough data to process.
+			waitCh = closedCh
+		}
+		select {
+		case <-stopCh:
+			mustStop = true
+			// Make sure reroutedBR is re-routed last time before returning
+			// in order to reroute the remaining data to healthy vmstorage nodes.
+		case <-ticker.C:
+		case <-waitCh:
+		}
+		if len(br.buf) == 0 {
+			reroutedBRLock.Lock()
+			reroutedBR, br = br, reroutedBR
+			reroutedBRLock.Unlock()
+		}
+		reroutedBRCond.Broadcast()
+		currentTime := fasttime.UnixTimestamp()
+		if len(br.buf) < cap(br.buf)/4 && currentTime-brLastResetTime > 10 {
+			// Free up capacity space occupied by br.buf in order to reduce memory usage after spikes.
+			br.buf = append(br.buf[:0:0], br.buf...)
+			brLastResetTime = currentTime
+		}
+		if len(br.buf) == 0 {
+			// Nothing to re-route.
+			continue
+		}
+		sns := getHealthyStorageNodes()
+		if len(sns) == 0 {
+			// No more vmstorage nodes to write data to.
+			rerouteErrors.Inc()
+			logger.Errorf("cannot send rerouted rows because all the storage nodes are unhealthy")
+			// Do not reset br in the hope it could be sent next time.
+			continue
+		}
+		spreadReroutedBufToStorageNodes(sns, &br)
+		// There is no need in br.reset() here, since it is already done in spreadReroutedBufToStorageNodes.
+	}
+	// Notify all the blocked addToReroutedBuf callers, so they may finish the work.
+	reroutedBRCond.Broadcast()
+}
+
+// storageNode is a client sending data to vmstorage node.
+type storageNode struct {
+	// broken is set to non-zero if the given vmstorage node is temporarily unhealthy.
+	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
+	broken uint32
+
+	// brLock protects br.
+	brLock sync.Mutex
+
+	// Buffer with data that needs to be written to the storage node.
+	// It must be accessed under brLock.
+	br bufRows
+
+	// bcLock protects bc.
+	bcLock sync.Mutex
+
+	// bc is a single connection to vmstorage for data transfer.
+	// It must be accessed under bcLock.
+	bc *handshake.BufferedConn
+
+	dialer *netutil.TCPDialer
+
+	// The number of dial errors to vmstorage node.
+	dialErrors *metrics.Counter
+
+	// The number of handshake errors to vmstorage node.
+	handshakeErrors *metrics.Counter
+
+	// The number of connection errors to vmstorage node.
+	connectionErrors *metrics.Counter
+
+	// The number of rows pushed to storageNode with push method.
+	rowsPushed *metrics.Counter
+
+	// The number of rows sent to vmstorage node.
+	rowsSent *metrics.Counter
+
+	// The number of rows rerouted from the given vmstorage node
+	// to healthy nodes when the given node was unhealthy.
+	rowsReroutedFromHere *metrics.Counter
+
+	// The number of rows rerouted to the given vmstorage node
+	// from other nodes when they were unhealthy.
+	rowsReroutedToHere *metrics.Counter
+}
+
+// storageNodes contains a list of vmstorage node clients.
+var storageNodes []*storageNode
+
+var (
+	storageNodesWG  sync.WaitGroup
+	rerouteWorkerWG sync.WaitGroup
+)
+
+var (
+	storageNodesStopCh  = make(chan struct{})
+	rerouteWorkerStopCh = make(chan struct{})
+)
+
+// InitStorageNodes initializes vmstorage nodes' connections to the given addrs.
+func InitStorageNodes(addrs []string) {
+	if len(addrs) == 0 {
+		logger.Panicf("BUG: addrs must be non-empty")
+	}
+	if len(addrs) > 255 {
+		logger.Panicf("BUG: too much addresses: %d; max supported %d addresses", len(addrs), 255)
+	}
+
+	storageNodes = storageNodes[:0]
+	for _, addr := range addrs {
+		sn := &storageNode{
+			dialer: netutil.NewTCPDialer("vminsert", addr),
+
+			dialErrors:           metrics.NewCounter(fmt.Sprintf(`vm_rpc_dial_errors_total{name="vminsert", addr=%q}`, addr)),
+			handshakeErrors:      metrics.NewCounter(fmt.Sprintf(`vm_rpc_handshake_errors_total{name="vminsert", addr=%q}`, addr)),
+			connectionErrors:     metrics.NewCounter(fmt.Sprintf(`vm_rpc_connection_errors_total{name="vminsert", addr=%q}`, addr)),
+			rowsPushed:           metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_pushed_total{name="vminsert", addr=%q}`, addr)),
+			rowsSent:             metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_sent_total{name="vminsert", addr=%q}`, addr)),
+			rowsReroutedFromHere: metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_from_here_total{name="vminsert", addr=%q}`, addr)),
+			rowsReroutedToHere:   metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_to_here_total{name="vminsert", addr=%q}`, addr)),
+		}
+		_ = metrics.NewGauge(fmt.Sprintf(`vm_rpc_rows_pending{name="vminsert", addr=%q}`, addr), func() float64 {
+			sn.brLock.Lock()
+			n := sn.br.rows
+			sn.brLock.Unlock()
+			return float64(n)
+		})
+		_ = metrics.NewGauge(fmt.Sprintf(`vm_rpc_buf_pending_bytes{name="vminsert", addr=%q}`, addr), func() float64 {
+			sn.brLock.Lock()
+			n := len(sn.br.buf)
+			sn.brLock.Unlock()
+			return float64(n)
+		})
+		storageNodes = append(storageNodes, sn)
+	}
+
+	maxBufSizePerStorageNode = memory.Allowed() / 8 / len(storageNodes)
+	if maxBufSizePerStorageNode > consts.MaxInsertPacketSize {
+		maxBufSizePerStorageNode = consts.MaxInsertPacketSize
+	}
+	reroutedBufMaxSize = memory.Allowed() / 16
+	if reroutedBufMaxSize < maxBufSizePerStorageNode {
+		reroutedBufMaxSize = maxBufSizePerStorageNode
+	}
+	if reroutedBufMaxSize > maxBufSizePerStorageNode*len(storageNodes) {
+		reroutedBufMaxSize = maxBufSizePerStorageNode * len(storageNodes)
+	}
+
+	for idx, sn := range storageNodes {
+		storageNodesWG.Add(1)
+		go func(sn *storageNode, idx int) {
+			sn.run(storageNodesStopCh, idx)
+			storageNodesWG.Done()
+		}(sn, idx)
+	}
+
+	rerouteWorkerWG.Add(1)
+	go func() {
+		rerouteWorker(rerouteWorkerStopCh)
+		rerouteWorkerWG.Done()
+	}()
+}
+
+// Stop gracefully stops netstorage.
+func Stop() {
+	close(rerouteWorkerStopCh)
+	rerouteWorkerWG.Wait()
+
+	close(storageNodesStopCh)
+	storageNodesWG.Wait()
+}
+
+// addToReroutedBuf adds buf to reroutedBR.
+//
+// It waits until the reroutedBR has enough space for buf or if Stop is called.
+// This guarantees backpressure if the ingestion rate exceeds vmstorage nodes'
+// ingestion rate capacity.
+//
+// It returns non-nil error only in the following cases:
+//
+//   - if all the storage nodes are unhealthy.
+//   - if Stop is called.
+func addToReroutedBuf(buf []byte, rows int) error {
+	if len(buf) > reroutedBufMaxSize {
+		logger.Panicf("BUG: len(buf)=%d cannot exceed reroutedBufMaxSize=%d", len(buf), reroutedBufMaxSize)
+	}
+
+	reroutedBRLock.Lock()
+	defer reroutedBRLock.Unlock()
+
+	for len(reroutedBR.buf)+len(buf) > reroutedBufMaxSize {
+		if getHealthyStorageNodesCount() == 0 {
+			rowsLostTotal.Add(rows)
+			return fmt.Errorf("all the vmstorage nodes are unavailable and reroutedBR has no enough space for storing %d bytes; only %d bytes left in reroutedBR",
+				len(buf), reroutedBufMaxSize-len(reroutedBR.buf))
+		}
+		select {
+		case <-rerouteWorkerStopCh:
+			rowsLostTotal.Add(rows)
+			return fmt.Errorf("rerouteWorker cannot send the data since it is stopped")
+		default:
+		}
+
+		// The reroutedBR.buf has no enough space for len(buf). Wait while the reroutedBR.buf is be sent by rerouteWorker.
+		reroutedBufWaits.Inc()
+		reroutedBRCond.Wait()
+	}
+	reroutedBR.buf = append(reroutedBR.buf, buf...)
+	reroutedBR.rows += rows
+	reroutesTotal.Inc()
+	return nil
+}
+
+func getHealthyStorageNodesCount() int {
+	n := 0
+	for _, sn := range storageNodes {
+		if !sn.isBroken() {
+			n++
+		}
+	}
+	return n
+}
+
+func getHealthyStorageNodes() []*storageNode {
+	sns := make([]*storageNode, 0, len(storageNodes)-1)
+	for _, sn := range storageNodes {
+		if !sn.isBroken() {
+			sns = append(sns, sn)
+		}
+	}
+	return sns
+}
+
+func spreadReroutedBufToStorageNodes(sns []*storageNode, br *bufRows) {
+	var mr storage.MetricRow
+	rowsProcessed := 0
+	src := br.buf
+	for len(src) > 0 {
+		tail, err := mr.Unmarshal(src)
+		if err != nil {
+			logger.Panicf("BUG: cannot unmarshal MetricRow from reroutedBR.buf: %s", err)
+		}
+		rowBuf := src[:len(src)-len(tail)]
+		src = tail
+
+		idx := uint64(0)
+		if len(sns) > 1 {
+			h := xxhash.Sum64(mr.MetricNameRaw)
+			// Do not use jump.Hash(h, int32(len(sns))) here,
+			// since this leads to uneven distribution of rerouted rows among sns -
+			// they all go to the original or to the next sn.
+			idx = h % uint64(len(sns))
+		}
+		attempts := 0
+		for {
+			sn := sns[idx]
+			if sn.sendReroutedRow(rowBuf) {
+				// The row has been successfully re-routed to sn.
+				break
+			}
+
+			// Cannot re-route data to sn. Try sending to the next vmstorage node.
+			idx++
+			if idx >= uint64(len(sns)) {
+				idx = 0
+			}
+			attempts++
+			if attempts < len(sns) {
+				continue
+			}
+
+			// There is no enough buffer space in all the vmstorage nodes.
+			// Return the remaining data to br.buf, so it may be processed later.
+			br.buf = append(br.buf[:0], rowBuf...)
+			br.buf = append(br.buf, src...)
+			br.rows -= rowsProcessed
+			return
+		}
+		rowsProcessed++
+	}
+	if rowsProcessed != br.rows {
+		logger.Panicf("BUG: unexpected number of rows processed; got %d; want %d", rowsProcessed, br.rows)
+	}
+	reroutedRowsProcessed.Add(rowsProcessed)
+	br.reset()
+}
+
+func (sn *storageNode) sendReroutedRow(buf []byte) bool {
+	if sn.isBroken() {
+		return false
+	}
+	sn.brLock.Lock()
+	ok := len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode
+	if ok {
+		sn.br.buf = append(sn.br.buf, buf...)
+		sn.br.rows++
+		sn.rowsReroutedToHere.Inc()
+	}
+	sn.brLock.Unlock()
+	return ok
+}
+
+var (
+	maxBufSizePerStorageNode int
+
+	reroutedBR         bufRows
+	reroutedBRLock     sync.Mutex
+	reroutedBRCond     = sync.NewCond(&reroutedBRLock)
+	reroutedBufMaxSize int
+
+	reroutedRowsProcessed = metrics.NewCounter(`vm_rpc_rerouted_rows_processed_total{name="vminsert"}`)
+	reroutedBufWaits      = metrics.NewCounter(`vm_rpc_rerouted_buf_waits_total{name="vminsert"}`)
+	reroutesTotal         = metrics.NewCounter(`vm_rpc_reroutes_total{name="vminsert"}`)
+	_                     = metrics.NewGauge(`vm_rpc_rerouted_rows_pending{name="vminsert"}`, func() float64 {
+		reroutedBRLock.Lock()
+		n := reroutedBR.rows
+		reroutedBRLock.Unlock()
+		return float64(n)
+	})
+	_ = metrics.NewGauge(`vm_rpc_rerouted_buf_pending_bytes{name="vminsert"}`, func() float64 {
+		reroutedBRLock.Lock()
+		n := len(reroutedBR.buf)
+		reroutedBRLock.Unlock()
+		return float64(n)
+	})
+
+	rerouteErrors                   = metrics.NewCounter(`vm_rpc_reroute_errors_total{name="vminsert"}`)
+	rowsLostTotal                   = metrics.NewCounter(`vm_rpc_rows_lost_total{name="vminsert"}`)
+	rowsIncompletelyReplicatedTotal = metrics.NewCounter(`vm_rpc_rows_incompletely_replicated_total{name="vminsert"}`)
+)