docs/{vmagent,vmauth}: add Profiling section

* app/vmalert: restore alerts state from datasource metrics

Vmalert will restore alerts state for rules that have `rule.For` > 0 from previously written timeseries via `remotewrite.url` flag.

* app/vmalert: mention remotewerite and remoteread configuration in README

.dockerignore

@@ -0,0 +1,6 @@
+.git
+vendor
+gocache-for-docker
+victoria-metrics-data
+vmstorage-data
+vmselect-cache

.github/workflows/main.yml

@@ -0,0 +1,42 @@
+name: main
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@master
+        with:
+          go-version: 1.14
+        id: go
+      - name: Dependencies
+        env:
+          GO111MODULE: on
+        run: |
+          go get -u golang.org/x/lint/golint
+          go get -u github.com/kisielk/errcheck
+          go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+            export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+            make check-all
+            git diff --exit-code
+            make test-full
+            make test-pure
+            make test-full-386
+            make vminsert vmselect vmstorage
+            make vminsert-pure vmselect-pure vmstorage-pure
+            make vmutils
+            GOOS=freebsd go build -mod=vendor ./app/vminsert
+            GOOS=freebsd go build -mod=vendor ./app/vmselect
+            GOOS=freebsd go build -mod=vendor ./app/vmstorage
+            GOOS=darwin go build -mod=vendor ./app/vminsert
+            GOOS=darwin go build -mod=vendor ./app/vmselect
+            GOOS=darwin go build -mod=vendor ./app/vmstorage

.gitignore

@@ -0,0 +1,19 @@
+/tags
+/pkg
+*.pprof
+/bin
+.idea
+*.test
+*.swp
+/gocache-for-docker
+/victoria-metrics-data
+/vmagent-remotewrite-data
+/vmstorage-data
+/vmselect-cache
+.DS_Store
+
+
+### terraform
+terraform.tfstate
+terraform.tfstate.*
+.terraform/

LICENSE

@@ -0,0 +1,190 @@
+ Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2019-2020 VictoriaMetrics, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,166 @@
+PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
+
+BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
+
+PKG_TAG ?= $(shell git tag -l --points-at HEAD)
+ifeq ($(PKG_TAG),)
+PKG_TAG := $(BUILDINFO_TAG)
+endif
+
+GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
+
+all: \
+	vminsert \
+	vmselect \
+	vmstorage \
+	vmagent \
+	vmalert \
+	vmauth \
+	vmbackup \
+	vmrestore
+
+all-pure: \
+	vminsert-pure \
+	vmselect-pure \
+	vmstorage-pure \
+	vmagent-pure \
+	vmalert-pure \
+	vmauth-pure \
+	vmbackup-pure \
+	vmrestore-pure
+
+include app/*/Makefile
+include deployment/*/Makefile
+
+clean:
+	rm -rf bin/*
+
+publish: \
+	publish-vminsert \
+	publish-vmselect \
+	publish-vmstorage \
+	publish-vmagent \
+	publish-vmalert \
+	publish-vmauth \
+	publish-vmbackup \
+	publish-vmrestore
+
+package: \
+	package-vminsert \
+	package-vmselect \
+	package-vmstorage \
+	package-vmagent \
+	package-vmalert \
+	package-vmauth \
+	package-vmbackup \
+	package-vmrestore
+
+vmutils: \
+	vmagent \
+	vmalert \
+	vmauth \
+	vmbackup \
+	vmrestore
+
+release: \
+	release-vmcluster \
+	release-vmutils
+
+release-vmcluster: \
+	vminsert-prod \
+	vmselect-prod \
+	vmstorage-prod
+	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz vminsert-prod vmselect-prod vmstorage-prod && \
+		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
+
+release-vmutils: \
+	vmagent-prod \
+	vmalert-prod \
+	vmauth-prod \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmauth-prod vmbackup-prod vmrestore-prod && \
+		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+
+fmt:
+	GO111MODULE=on gofmt -l -w -s ./lib
+	GO111MODULE=on gofmt -l -w -s ./app
+
+vet:
+	GO111MODULE=on go vet -mod=vendor ./lib/...
+	GO111MODULE=on go vet -mod=vendor ./app/...
+
+lint: install-golint
+	golint lib/...
+	golint app/...
+
+install-golint:
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
+
+errcheck: install-errcheck
+	errcheck -exclude=errcheck_excludes.txt ./lib/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+
+install-errcheck:
+	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
+
+check-all: fmt vet lint errcheck golangci-lint
+
+test:
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+
+test-race:
+	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
+
+test-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+
+test-full:
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
+test-full-386:
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
+benchmark:
+	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+
+benchmark-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
+
+vendor-update:
+	GO111MODULE=on go get -u ./lib/...
+	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+
+app-local:
+	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+quicktemplate-gen: install-qtc
+	qtc
+
+install-qtc:
+	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
+
+
+golangci-lint: install-golangci-lint
+	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck -D structcheck --timeout 2m
+
+install-golangci-lint:
+	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint

README.md

@@ -1,41 +1,345 @@
-<img  text-align="center" alt="Victoria Metrics" src="logo.png">
+<img alt="Victoria Metrics" src="logo.png">
 
-## VictoriaMetrics - the best long-term remote storage for Prometheus
+# Cluster version
 
-[![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+VictoriaMetrics is fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
 
-### VictoriaMetrics features
+It is recommended using [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics) instead of cluster version
+for ingestion rates lower than a million of data points per second.
+Single-node version [scales perfectly](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+with the number of CPU cores, RAM and available storage space.
+Single-node version is easier to configure and operate comparing to cluster version, so think twice before sticking to cluster version.
 
-- Native [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) support. Additionally, VictoriaMetrics extends PromQL with useful features. See [Extended PromQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL) for more details.
-- Simple configuration. Just copy-n-paste remote storage URL to Prometheus config and that's it! See [Quick Start](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Quick-Start) for more info.
-- Reduced operational overhead. Prometheus local storage retention may be set to the minimum possible value when using VictoriaMetrics remote storage. This effectively makes Prometheus stateless, so it may be run as a stateless service in Kubernetes.
-- Insertion rate scales to [millions of metric values per second](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893).
-- Storage scales to [millions of metrics](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) with trillions of metric values.
-- Wide range of retention periods - from 1 month to 5 years. Users may create different projects (aka `storage namespaces`) with different retention periods.
-- [Fast query engine](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4). It excels on heavy queries over thousands of metrics with millions of metric values.
-- The same remote storage URL may be used by multiple Prometheus instances collecting distinct metric sets, so all these metrics may be used in a single query (aka `global querying view`). This works ideally for multiple Prometheus instances located in different subnetworks / datacenters.
-- Accepts data in [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_reference/), so [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/) and other influx-compatible agents may send data to VictoriaMetrics.
-- [Single-server VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) accepts data in [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol) if `-graphiteListenAddr` is set.
-- VictoriaMetrics supports backfilling, i.e. data points from the past may be inserted into the DB.
-- Time series data may be exported via `/api/v1/export?match=<metric_selector>`. Optional `start` and `end` timestamps may be added for exporting time series in the given time range.
-- Time series may be deleted via `/api/v1/admin/tsdb/delete_series?match[]=<metric_selector>`.
-- [Instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) are supported.
+Join [our Slack](http://slack.victoriametrics.com/) or [contact us](mailto:info@victoriametrics.com) with consulting and support questions.
 
 
-### Useful links
+## Prominent features
 
-* [Site](https://victoriametrics.com/)
-* [`WITH` templates playground](https://play.victoriametrics.com/promql/expand-with-exprs)
-* [Grafana playground](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo)
-* [Docs](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki)
-* [FAQ](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/FAQ)
-* [Issues](https://github.com/VictoriaMetrics/VictoriaMetrics/issues)
-* [Google group](https://groups.google.com/forum/#!forum/victoriametrics)
-* [Creating the best remote storage for Prometheus](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac) - an article with technical details about VictoriaMetrics.
-* [Docker images](https://hub.docker.com/r/valyala/victoria-metrics/) and the corresponding [binaries](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) for single-server VictoriaMetrics
+- Supports all the features of [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- Performance and capacity scales horizontally.
+- Supports multiple independent namespaces for time series data (aka multi-tenancy).
 
 
-### Victoria Metrics Logo
+## Architecture overview
+
+VictoriaMetrics cluster consists of the following services:
+
+- `vmstorage` - stores the data
+- `vminsert` - proxies the ingested data to `vmstorage` shards using consistent hashing
+- `vmselect` - performs incoming queries using the data from `vmstorage`
+
+Each service may scale independently and may run on the most suitable hardware.
+`vmstorage` nodes don't know about each other, don't communicate with each other and don't share any data.
+This is [shared nothing architecture](https://en.wikipedia.org/wiki/Shared-nothing_architecture).
+It increases cluster availability, simplifies cluster maintenance and cluster scaling.
+
+<img src="https://docs.google.com/drawings/d/e/2PACX-1vTvk2raU9kFgZ84oF-OKolrGwHaePhHRsZEcfQ1I_EC5AB_XPWwB392XshxPramLJ8E4bqptTnFn5LL/pub?w=1104&amp;h=746">
+
+
+## Multitenancy
+
+VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces).
+Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request urls.
+See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics:
+
+* Each `accountID` and `projectID` is identified by an arbitrary 32-bit integer in the range `[0 .. 2^32)`.
+If `projectID` is missing, then it is automatically assigned to `0`. It is expected that other information about tenants
+such as auth tokens, tenant names, limits, accounting, etc. is stored in a separate relational database. This database must be managed
+by a separate service sitting in front of VictoriaMetrics cluster.
+
+* Tenants are automatically created when the first data point is written into the given tenant.
+
+* Data for all the tenants is evenly spread among available `vmstorage` nodes. This guarantees even load among `vmstorage` nodes
+when different tenants have different amounts of data and different query load.
+
+* VictoriaMetrics doesn't support querying multiple tenants in a single request.
+
+
+## Binaries
+
+Compiled binaries for cluster version are available in the `assets` section of [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+See archives containing `cluster` word.
+
+Docker images for cluster version are available here:
+
+- `vminsert` - https://hub.docker.com/r/victoriametrics/vminsert/tags
+- `vmselect` - https://hub.docker.com/r/victoriametrics/vmselect/tags
+- `vmstorage` - https://hub.docker.com/r/victoriametrics/vmstorage/tags
+
+
+## Building from sources
+
+Source code for cluster version is available at [cluster branch](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+### Production builds
+
+There is no need in installing Go on a host system since binaries are built
+inside [the official docker container for Go](https://hub.docker.com/_/golang).
+This makes reproducible builds.
+So [install docker](https://docs.docker.com/install/) and run the following command:
+
+```
+make vminsert-prod vmselect-prod vmstorage-prod
+```
+
+Production binaries are built into statically linked binaries. They are put into `bin` folder with `-prod` suffixes:
+```
+$ make vminsert-prod vmselect-prod vmstorage-prod
+$ ls -1 bin
+vminsert-prod
+vmselect-prod
+vmstorage-prod
+```
+
+### Development Builds
+
+1. [Install go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make` from the repository root. It should build `vmstorage`, `vmselect`
+   and `vminsert` binaries and put them into the `bin` folder.
+
+
+### Building docker images
+
+Run `make package`. It will build the following docker images locally:
+
+* `victoriametrics/vminsert:<PKG_TAG>`
+* `victoriametrics/vmselect:<PKG_TAG>`
+* `victoriametrics/vmstorage:<PKG_TAG>`
+
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package`.
+
+By default images are built on top of `scratch` image. It is possible to build on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds images on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package
+```
+
+## Operation
+
+### Cluster setup
+
+A minimal cluster must contain the following nodes:
+
+* a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags
+* a single `vminsert` node with `-storageNode=<vmstorage_host>:8400`
+* a single `vmselect` node with `-storageNode=<vmstorage_host>:8401`
+
+It is recommended to run at least two nodes for each service
+for high availability purposes.
+
+An http load balancer must be put in front of `vminsert` and `vmselect` nodes:
+- requests starting with `/insert` must be routed to port `8480` on `vminsert` nodes.
+- requests starting with `/select` must be routed to port `8481` on `vmselect` nodes.
+
+Ports may be altered by setting `-httpListenAddr` on the corresponding nodes.
+
+It is recommended setting up [monitoring](#monitoring) for the cluster.
+
+#### Environment variables
+
+Each flag values can be set thru environment variables by following these rules:
+
+- The `-envflag.enable` flag must be set
+- Each `.` in flag names must be substituted by `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`)
+- For repeating flags, an alternative syntax can be used by joining the different values into one using `,` as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`)
+- It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
+
+
+### Monitoring
+
+All the cluster components expose various metrics in Prometheus-compatible format at `/metrics` page on the TCP port set in `-httpListenAddr` command-line flag.
+By default the following TCP ports are used:
+- `vminsert` - 8480
+- `vmselect` - 8481
+- `vmstorage` - 8482
+
+It is recommended setting up Prometheus to scrape `/metrics` pages from all the cluster components, so they can be monitored and analyzed
+with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11831).
+
+
+### URL format
+
+* URLs for data ingestion: `http://<vminsert>:8480/insert/<accountID>/<suffix>`, where:
+  - `<accountID>` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`,
+    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`.
+  - `<suffix>` may have the following values:
+     - `prometheus` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+     - `influx/write` or `influx/api/v2/write` - for inserting data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+     - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html).
+     - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
+
+* URLs for querying: `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
+  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
+  - `<suffix>` may have the following values:
+    - `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries)
+    - `api/v1/query_range` - performs [PromQL range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)
+    - `api/v1/series` - performs [series query](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
+    - `api/v1/labels` - returns a [list of label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
+    - `api/v1/label/<label_name>/values` - returns values for the given `<label_name>` according [to API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
+    - `federate` - returns [federated metrics](https://prometheus.io/docs/prometheus/latest/federation/)
+    - `api/v1/export` - exports raw data. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details
+
+* URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
+  Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
+  be used on a regular basis, since it carries non-zero overhead.
+
+* `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
+  - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
+    which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
+    command-line flag value.
+  - `/snapshot/list` - list available snasphots.
+  - `/snapshot/delete?snapshot=<id>` - delete the given snapshot.
+  - `/snapshot/delete_all` - delete all the snapshots.
+
+  Snapshots may be created independently on each `vmstorage` node. There is no need in synchronizing snapshots' creation
+  across `vmstorage` nodes.
+
+
+### Cluster resizing and scalability.
+
+Cluster performance and capacity scales with adding new nodes.
+
+* `vminsert` and `vmselect` nodes are stateless and may be added / removed at any time.
+  Do not forget updating the list of these nodes on http load balancer.
+  Adding more `vminsert` nodes scales data ingestion rate. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/175#issuecomment-536925841)
+  about ingestion rate scalability.
+  Adding more `vmselect` nodes scales select queries rate.
+* `vmstorage` nodes own the ingested data, so they cannot be removed without data loss.
+  Adding more `vmstorage` nodes scales cluster capacity.
+
+Steps to add `vmstorage` node:
+
+1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster.
+2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8401`.
+3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8400`.
+
+
+### Cluster availability
+
+* HTTP load balancer must stop routing requests to unavailable `vminsert` and `vmselect` nodes.
+* The cluster remains available if at least a single `vmstorage` node exists:
+
+  - `vminsert` re-routes incoming data from unavailable `vmstorage` nodes to healthy `vmstorage` nodes
+  - `vmselect` continues serving partial responses if at least a single `vmstorage` node is available.
+
+
+### Updating / reconfiguring cluster nodes
+
+All the node types - `vminsert`, `vmselect` and `vmstorage` - may be updated via graceful shutdown.
+Send `SIGINT` signal to the corresponding process, wait until it finishes and then start new version
+with new configs.
+
+Cluster should remain in working state if at least a single node of each type remains available during
+the update process. See [cluster availability](#cluster-availability) section for details.
+
+
+### Capacity planning
+
+Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the most suitable hardware.
+
+#### vminsert
+
+* The recommended total number of vCPU cores for all the `vminsert` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended number of vCPU cores per each `vminsert` instance should equal to the number of `vmstorage` instances in the cluster.
+* The amount of RAM per each `vminsert` instance should be 1GB or more. RAM is used as a buffer for spikes in ingestion rate.
+* Sometimes `-rpc.disableCompression` command-line flag on `vminsert` instances could increase ingestion capacity at the cost
+  of higher network bandwidth usage between `vminsert` and `vmstorage`.
+
+#### vmstorage
+
+* The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
+  Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
+* The recommended total amount of storage space for all the `vmstorage` instances can be calculated
+  from the ingestion rate and retention: `storage_space = ingestion_rate * retention_seconds`.
+
+#### vmselect
+
+The recommended hardware for `vmselect` instances highly depends on the type of queries. Lightweight queries over small number of time series usually require
+small number of vCPU cores and small amount of RAM on `vmselect`, while heavy queries over big number of time series (>10K) usually require
+bigger number of vCPU cores and bigger amounts of RAM.
+
+In general it is recommended increasing the number of vCPU cores and RAM per `vmselect` node for higher query performance,
+while adding new `vmselect` nodes only when old nodes are overloaded with incoming query stream.
+
+
+### Helm
+
+Helm chart simplifies managing cluster version of VictoriaMetrics in Kubernetes.
+It is available in the [helm-charts](https://github.com/VictoriaMetrics/helm-charts) repository.
+
+Upgrade follows `Cluster resizing procedure` under the hood.
+
+
+### Replication and data safety
+
+VictoriaMetrics offloads replication to the underlying storage pointed by `-storageDataPath`.
+It is recommended storing data on [Google Compute Engine persistent disks](https://cloud.google.com/compute/docs/disks/#pdspecs),
+since they are protected from data loss and data corruption. They also provide consistently high performance
+and [may be resized](https://cloud.google.com/compute/docs/disks/add-persistent-disk) without downtime.
+HDD-based persistent disks should be enough for the majority of use cases.
+
+It is recommended using durable replicated persistent volumes in Kubernetes.
+
+Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883).
+
+
+### Backups
+
+It is recommended performing periodical backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+for protecting from user errors such as accidental data deletion.
+
+The following steps must be performed for each `vmstorage` node for creating a backup:
+
+1. Create an instant snapshot by navigating to `/snapshot/create` HTTP handler. It will create snapshot and return its name.
+2. Archive the created snapshot from `<-storageDataPath>/snapshots/<snapshot_name>` folder using [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmbackup/README.md).
+   The archival process doesn't interfere with `vmstorage` work, so it may be performed at any suitable time.
+3. Delete unused snapshots via `/snapshot/delete?snapshot=<snapshot_name>` or `/snapshot/delete_all` in order to free up occupied storage space.
+
+There is no need in synchronizing backups among all the `vmstorage` nodes.
+
+Restoring from backup:
+
+1. Stop `vmstorage` node with `kill -INT`.
+2. Restore data from backup using [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmrestore/README.md) into `-storageDataPath` directory.
+3. Start `vmstorage` node.
+
+
+## Community and contributions
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
+
+Due to `KISS` cluster version of VictoriaMetrics has no the following "features" popular in distributed computing world:
+
+- Fragile gossip protocols. See [failed attempt in Thanos](https://github.com/improbable-eng/thanos/blob/030bc345c12c446962225221795f4973848caab5/docs/proposals/completed/201809_gossip-removal.md).
+- Hard-to-understand-and-implement-properly [Paxos protocols](https://www.quora.com/In-distributed-systems-what-is-a-simple-explanation-of-the-Paxos-algorithm).
+- Complex replication schemes, which may go nuts in unforesseen edge cases. The replication is offloaded to the underlying durable replicated storage
+  such as [persistent disks in Google Compute Engine](https://cloud.google.com/compute/docs/disks/#pdspecs).
+- Automatic data reshuffling between storage nodes, which may hurt cluster performance and availability.
+- Automatic cluster resizing, which may cost you a lot of money if improperly configured.
+- Automatic discovering and addition of new nodes in the cluster, which may mix data between dev and prod clusters :)
+- Automatic leader election, which may result in split brain disaster on network errors.
+
+
+## Reporting bugs
+
+Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+
+
+## Victoria Metrics Logo
 
 [Zip](VM_logo.zip) contains three folders with different image orientation (main color and inverted version).
 
@@ -46,24 +350,21 @@ Files included in each folder:
 * 2 EPS Adobe Illustrator EPS10 files
 
 
-#### Logo Usage Guidelines
+### Logo Usage Guidelines
 
-##### Font used: 
+#### Font used:
 
-* Lato Black 
+* Lato Black
 * Lato Regular
 
-##### Color Palette:
+#### Color Palette:
 
-* HEX [#110f0f](https://www.color-hex.com/color/110f0f) 
+* HEX [#110f0f](https://www.color-hex.com/color/110f0f)
 * HEX [#ffffff](https://www.color-hex.com/color/ffffff)
 
-#### We kindly ask:
+### We kindly ask:
 
 - Please don't use any other font instead of suggested.
 - There should be sufficient clear space around the logo.
 - Do not change spacing, alignment, or relative locations of the design elements.
 - Do not change the proportions of any of the design elements or the design itself. You    may resize as needed but must retain all proportions.
-
-
-

app/vmagent/Makefile

@@ -0,0 +1,76 @@
+# All these commands must run from repository root.
+
+vmagent:
+	APP_NAME=vmagent $(MAKE) app-local
+
+vmagent-race:
+	APP_NAME=vmagent RACE=-race $(MAKE) app-local
+
+vmagent-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker
+
+vmagent-pure-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-pure
+
+vmagent-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+
+vmagent-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+
+vmagent-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+
+vmagent-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+
+vmagent-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-386
+
+package-vmagent:
+	APP_NAME=vmagent $(MAKE) package-via-docker
+
+package-vmagent-pure:
+	APP_NAME=vmagent $(MAKE) package-via-docker-pure
+
+package-vmagent-amd64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-amd64
+
+package-vmagent-arm:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm
+
+package-vmagent-arm64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm64
+
+package-vmagent-ppc64le:
+	APP_NAME=vmagent $(MAKE) package-via-docker-ppc64le
+
+package-vmagent-386:
+	APP_NAME=vmagent $(MAKE) package-via-docker-386
+
+publish-vmagent:
+	APP_NAME=vmagent $(MAKE) publish-via-docker
+
+run-vmagent:
+	mkdir -p vmagent-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-data:/vmagent-data' \
+	APP_NAME=vmagent \
+	$(MAKE) run-via-docker
+
+vmagent-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
+
+vmagent-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
+
+vmagent-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
+
+vmagent-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
+
+vmagent-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
+
+vmagent-pure:
+	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -0,0 +1,263 @@
+## vmagent
+
+`vmagent` is a tiny but brave agent, which helps you collecting metrics from various sources
+and storing them to [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports `remote_write` protocol.
+
+<img alt="vmagent" src="vmagent.png">
+
+
+### Motivation
+
+While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
+and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
+Also, we found that users’ infrastructure is like snowflakes - never alike, and we decided to add more flexibility
+to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
+
+
+### Features
+
+* Can be used as drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
+  See [Quick Start](#quick-start) for details.
+* Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
+* Accepts data via all the ingestion protocols supported by VictoriaMetrics:
+  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
+  * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
+* Can replicate collected metrics simultaneously to multiple remote storage systems.
+* Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
+  are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
+  to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth comparing to Prometheus.
+
+
+### Quick Start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
+
+* `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. Multiple `-remoteWrite.url` args can be set in parallel
+  in order to replicate data concurrently to multiple remote storage systems.
+
+Example command line:
+
+```
+/path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+If you need collecting only Influx data, then the following command line would be enough:
+
+```
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+
+`vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/).
+
+Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
+
+
+### Use cases
+
+
+#### IoT and Edge monitoring
+
+`vmagent` can run and collect metrics in IoT and industrial networks with unreliable or scheduled connections to the remote storage.
+It buffers the collected data in local files until the connection to remote storage becomes available and then sends the buffered
+data to the remote storage. It re-tries sending the data to remote storage on any errors.
+The maximum buffer size can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+
+`vmagent` works on various architectures from IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
+See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+
+#### Drop-in replacement for Prometheus
+
+If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such setup.
+See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
+
+
+#### Replication and high availability
+
+`vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
+If a single remote storage instance temporarily goes out of service, then the collected data remains available in another remote storage instances.
+`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
+Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
+
+
+#### Relabeling and filtering
+
+`vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
+it can remove unneeded samples via Prometheus-like relabeling before sending the collected data to remote storage.
+See [these docs](#relabeling) for details.
+
+
+#### Splitting data streams among multiple systems
+
+`vmagent` supports splitting of the collected data among muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
+data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
+Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
+
+
+#### Prometheus remote_write proxy
+
+`vmagent` may be used as a proxy for Prometheus data sent via Prometheus `remote_write` protocol. It can accept data via `remote_write` API
+at `/api/v1/write` endpoint, apply relabeling and filtering and then proxy it to another `remote_write` systems.
+The `vmagent` can be configured to encrypt the incoming `remote_write` requests with `-tls*` command-line flags.
+Additionally, Basic Auth can be enabled for the incoming `remote_write` requests with `-httpAuth.*` command-line flags.
+
+
+
+### How to collect metrics in Prometheus format
+
+Pass the path to `prometheus.yml` to `-promscrape.config` command-line flag. `vmagent` takes into account the following
+sections from [Prometheus config file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/):
+
+* `global`
+* `scrape_configs`
+
+All the other sections are ignored, including [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section.
+Use `-remoteWrite.*` command-line flags instead for configuring remote write settings.
+
+The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
+
+* `static_configs` - for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
+* `file_sd_configs` - for scraping targets defined in external files aka file-based service discover.
+  See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
+* `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
+  See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazone EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
+* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
+  See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
+* `consul_sd_configs` - for scraping targets registered in Consul.
+  See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
+
+The following service discovery mechanisms will be added to `vmagent` soon:
+
+* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
+
+
+File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
+
+
+### Adding labels to metrics
+
+Labels can be added to metrics via the following mechanisms:
+
+* Via `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in `-promscrape.config` file.
+* Via `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+
+
+### Relabeling
+
+`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+Additionally it provides the following extra actions:
+
+* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
+* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+
+The relabeling can be defined in the following places:
+
+* At `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to targets when parsing the file during `vmagent` startup
+  or during config reload after sending `SIGHUP` signal to `vmagent`  via `kill -HUP`.
+* At `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to metrics after each scrape for the configured targets.
+* At `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
+* At `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
+
+Read more about relabeling in the following articles:
+
+* [Life of a label](https://www.robustperception.io/life-of-a-label)
+* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+* [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+* [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
+
+
+### Monitoring
+
+`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
+either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+
+
+### Troubleshooting
+
+* It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
+  since `vmagent` establishes at least a single TCP connection per each target.
+
+* When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+
+* It is recommended increasing `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+  and `vmagent_remotewrite_pending_data_bytes` metric exported by `vmagent` at `/metrics` page constantly grows.
+
+* `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
+  The directory can grow big when remote storage is unavailable during extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want sending all the data from the directory to remote storage, just stop `vmagent` and delete the directory.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmagent` from the root folder of the repository.
+   It builds `vmagent` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-prod` from the root folder of the repository.
+   It builds `vmagent-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmagent
+```
+
+
+### Profiling
+
+`vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8428/debug/pprof/heap > mem.pprof
+```
+
+* CPU profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8428/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

app/vmagent/common/push_ctx.go

@@ -0,0 +1,70 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// PushCtx is a context used for populating WriteRequest.
+type PushCtx struct {
+	WriteRequest prompbmarshal.WriteRequest
+
+	// Labels contains flat list of all the labels used in WriteRequest.
+	Labels []prompbmarshal.Label
+
+	// Samples contains flat list of all the samples used in WriteRequest.
+	Samples []prompbmarshal.Sample
+}
+
+// Reset resets ctx.
+func (ctx *PushCtx) Reset() {
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
+
+	labels := ctx.Labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	ctx.Labels = ctx.Labels[:0]
+
+	ctx.Samples = ctx.Samples[:0]
+}
+
+// GetPushCtx returns PushCtx from pool.
+//
+// Call PutPushCtx when the ctx is no longer needed.
+func GetPushCtx() *PushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
+	}
+}
+
+// PutPushCtx returns ctx to the pool.
+//
+// ctx mustn't be used after returning to the pool.
+func PutPushCtx(ctx *PushCtx) {
+	ctx.Reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/csvimport/request_handler.go

@@ -0,0 +1,63 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes csv data from req.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8429
+
+ENTRYPOINT ["/vmagent-prod"]
+ARG src_binary
+COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -0,0 +1,65 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -0,0 +1,167 @@
+package influx
+
+import (
+	"flag"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", insertRows)
+	})
+}
+
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, insertRows)
+	})
+}
+
+func insertRows(db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.ctx.Labels[:0]
+	samples := ctx.ctx.Samples[:0]
+	commonLabels := ctx.commonLabels[:0]
+	buf := ctx.buf[:0]
+	for i := range rows {
+		r := &rows[i]
+		commonLabels = commonLabels[:0]
+		hasDBLabel := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		if len(db) > 0 && !hasDBLabel {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  "db",
+				Value: db,
+			})
+		}
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			bufLen := len(buf)
+			buf = append(buf, ctx.metricGroupBuf...)
+			if !skipFieldKey {
+				buf = append(buf, f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: metricGroup,
+			})
+			labels = append(labels, commonLabels...)
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: r.Timestamp,
+				Value:     f.Value,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+		}
+		rowsTotal += len(r.Fields)
+	}
+	ctx.buf = buf
+	ctx.ctx.WriteRequest.Timeseries = tssDst
+	ctx.ctx.Labels = labels
+	ctx.ctx.Samples = samples
+	ctx.commonLabels = commonLabels
+	remotewrite.Push(&ctx.ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+
+	return nil
+}
+
+type pushCtx struct {
+	ctx            common.PushCtx
+	commonLabels   []prompbmarshal.Label
+	metricGroupBuf []byte
+	buf            []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.ctx.Reset()
+
+	commonLabels := ctx.commonLabels
+	for i := range commonLabels {
+		label := &commonLabels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+	ctx.buf = ctx.buf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/main.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+)
+
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	startTime := time.Now()
+	remotewrite.Init()
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+
+	promscrape.Init(remotewrite.Push)
+
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, requestHandler)
+	}
+	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	}
+
+	promscrape.Stop()
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+	remotewrite.Stop()
+
+	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	switch path {
+	case "/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/write", "/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
+		influxQueryRequests.Inc()
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
+		return true
+	case "/targets":
+		promscrapeTargetsRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
+		return true
+	case "/-/reload":
+		promscrapeConfigReloadRequests.Inc()
+		procutil.SelfSIGHUP()
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	}
+	return false
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
+
+	vmimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+
+	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+
+	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+)

app/vmagent/opentsdb/request_handler.go

@@ -0,0 +1,65 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/opentsdbhttp/request_handler.go

@@ -0,0 +1,64 @@
+package opentsdbhttp
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -0,0 +1,67 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		labelsLen := len(labels)
+		for i := range ts.Labels {
+			label := &ts.Labels[i]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
+			})
+		}
+		samplesLen := len(samples)
+		for i := range ts.Samples {
+			sample := &ts.Samples[i]
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     sample.Value,
+				Timestamp: sample.Timestamp,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -0,0 +1,281 @@
+package remotewrite
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/fasthttp"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
+	tlsKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
+	tlsCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used")
+
+	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username to use for -remoteWrite.url")
+	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password to use for -remoteWrite.url")
+	bearerToken       = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url")
+)
+
+type client struct {
+	urlLabelValue  string
+	remoteWriteURL string
+	host           string
+	requestURI     string
+	authHeader     string
+	fq             *persistentqueue.FastQueue
+	hc             *fasthttp.HostClient
+
+	requestDuration *metrics.Histogram
+	requestsOKCount *metrics.Counter
+	errorsCount     *metrics.Counter
+	retriesCount    *metrics.Counter
+
+	wg     sync.WaitGroup
+	stopCh chan struct{}
+}
+
+func newClient(remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authHeader := ""
+	if len(*basicAuthUsername) > 0 || len(*basicAuthPassword) > 0 {
+		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		token := *basicAuthUsername + ":" + *basicAuthPassword
+		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		authHeader = "Basic " + token64
+	}
+	if len(*bearerToken) > 0 {
+		if authHeader != "" {
+			logger.Panicf("FATAL: `-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", *bearerToken)
+		}
+		authHeader = "Bearer " + *bearerToken
+	}
+
+	readTimeout := *sendTimeout
+	if readTimeout <= 0 {
+		readTimeout = time.Minute
+	}
+	writeTimeout := readTimeout
+	var u fasthttp.URI
+	u.Update(remoteWriteURL)
+	scheme := string(u.Scheme())
+	switch scheme {
+	case "http", "https":
+	default:
+		logger.Panicf("FATAL: unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
+	}
+	host := string(u.Host())
+	if len(host) == 0 {
+		logger.Panicf("FATAL: invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
+	}
+	requestURI := string(u.RequestURI())
+	isTLS := scheme == "https"
+	var tlsCfg *tls.Config
+	if isTLS {
+		var err error
+		tlsCfg, err = getTLSConfig()
+		if err != nil {
+			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+		}
+	}
+	if !strings.Contains(host, ":") {
+		if isTLS {
+			host += ":443"
+		} else {
+			host += ":80"
+		}
+	}
+	maxConns := 2 * concurrency
+	hc := &fasthttp.HostClient{
+		Addr:                host,
+		Name:                "vmagent",
+		Dial:                statDial,
+		DialDualStack:       netutil.TCP6Enabled(),
+		IsTLS:               isTLS,
+		TLSConfig:           tlsCfg,
+		MaxConns:            maxConns,
+		MaxIdleConnDuration: 10 * readTimeout,
+		ReadTimeout:         readTimeout,
+		WriteTimeout:        writeTimeout,
+		MaxResponseBodySize: 1024 * 1024,
+	}
+	c := &client{
+		urlLabelValue:  urlLabelValue,
+		remoteWriteURL: remoteWriteURL,
+		host:           host,
+		requestURI:     requestURI,
+		authHeader:     authHeader,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
+	}
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	for i := 0; i < concurrency; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
+	}
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	return c
+}
+
+func (c *client) MustStop() {
+	close(c.stopCh)
+	c.wg.Wait()
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+}
+
+func getTLSConfig() (*tls.Config, error) {
+	var tlsRootCA *x509.CertPool
+	var tlsCertificate *tls.Certificate
+	if *tlsCertFile != "" || *tlsKeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(*tlsCertFile, *tlsKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load TLS certificate for -remoteWrite.tlsCertFile=%q and -remoteWrite.tlsKeyFile=%q: %s", *tlsCertFile, *tlsKeyFile, err)
+		}
+		tlsCertificate = &cert
+	}
+	if *tlsCAFile != "" {
+		data, err := ioutil.ReadFile(*tlsCAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read -remoteWrite.tlsCAFile=%q: %s", *tlsCAFile, err)
+		}
+		tlsRootCA = x509.NewCertPool()
+		if !tlsRootCA.AppendCertsFromPEM(data) {
+			return nil, fmt.Errorf("cannot parse data -remoteWrite.tlsCAFile=%q", *tlsCAFile)
+		}
+	}
+	tlsCfg := &tls.Config{
+		RootCAs:            tlsRootCA,
+		ClientSessionCache: tls.NewLRUClientSessionCache(0),
+	}
+	if tlsCertificate != nil {
+		tlsCfg.Certificates = []tls.Certificate{*tlsCertificate}
+	}
+	tlsCfg.InsecureSkipVerify = *tlsInsecureSkipVerify
+	return tlsCfg, nil
+}
+
+func (c *client) runWorker() {
+	var ok bool
+	var block []byte
+	ch := make(chan struct{})
+	for {
+		block, ok = c.fq.MustReadBlock(block[:0])
+		if !ok {
+			return
+		}
+		go func() {
+			c.sendBlock(block)
+			ch <- struct{}{}
+		}()
+		select {
+		case <-ch:
+			// The block has been sent successfully
+			continue
+		case <-c.stopCh:
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
+			select {
+			case <-ch:
+				// The block has been sent successfully.
+			case <-time.After(graceDuration):
+				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
+					len(block), c.remoteWriteURL, graceDuration.Seconds())
+			}
+			return
+		}
+	}
+}
+
+func (c *client) sendBlock(block []byte) {
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(c.requestURI)
+	req.SetHost(c.host)
+	req.Header.SetMethod("POST")
+	req.Header.Add("Content-Type", "application/x-protobuf")
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Add("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.authHeader != "" {
+		req.Header.Set("Authorization", c.authHeader)
+	}
+	req.SetBody(block)
+
+	retryDuration := time.Second
+	resp := fasthttp.AcquireResponse()
+
+again:
+	select {
+	case <-c.stopCh:
+		fasthttp.ReleaseRequest(req)
+		fasthttp.ReleaseResponse(resp)
+		return
+	default:
+	}
+
+	startTime := time.Now()
+	err := doRequestWithPossibleRetry(c.hc, req, resp)
+	c.requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		c.errorsCount.Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	statusCode := resp.StatusCode()
+	if statusCode/100 != 2 {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	c.requestsOKCount.Inc()
+
+	// The block has been successfully sent to the remote storage.
+	fasthttp.ReleaseResponse(resp)
+	fasthttp.ReleaseRequest(req)
+}
+
+func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) error {
+	// There is no need in calling DoTimeout, since the timeout must be already set in hc.ReadTimeout.
+	err := hc.Do(req, resp)
+	if err == nil {
+		return nil
+	}
+	if err != fasthttp.ErrConnectionClosed {
+		return err
+	}
+	// Retry request if the server closed the keep-alive connection during the first attempt.
+	return hc.Do(req, resp)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -0,0 +1,199 @@
+package remotewrite
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
+		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage")
+	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+)
+
+// the maximum number of rows to send per each block.
+const maxRowsPerBlock = 10000
+
+type pendingSeries struct {
+	mu sync.Mutex
+	wr writeRequest
+
+	stopCh            chan struct{}
+	periodicFlusherWG sync.WaitGroup
+}
+
+func newPendingSeries(pushBlock func(block []byte)) *pendingSeries {
+	var ps pendingSeries
+	ps.wr.pushBlock = pushBlock
+	ps.stopCh = make(chan struct{})
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
+	return &ps
+}
+
+func (ps *pendingSeries) MustStop() {
+	close(ps.stopCh)
+	ps.periodicFlusherWG.Wait()
+}
+
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+	ps.mu.Lock()
+	ps.wr.push(tss)
+	ps.mu.Unlock()
+}
+
+func (ps *pendingSeries) periodicFlusher() {
+	ticker := time.NewTicker(*flushInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-ps.stopCh:
+			mustStop = true
+		case <-ticker.C:
+			if time.Since(ps.wr.lastFlushTime) < *flushInterval/2 {
+				continue
+			}
+		}
+		ps.mu.Lock()
+		ps.wr.flush()
+		ps.mu.Unlock()
+	}
+}
+
+type writeRequest struct {
+	wr            prompbmarshal.WriteRequest
+	pushBlock     func(block []byte)
+	lastFlushTime time.Time
+
+	tss []prompbmarshal.TimeSeries
+
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
+}
+
+func (wr *writeRequest) reset() {
+	wr.wr.Timeseries = nil
+
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	wr.tss = wr.tss[:0]
+
+	for i := range wr.labels {
+		label := &wr.labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	wr.labels = wr.labels[:0]
+
+	wr.samples = wr.samples[:0]
+	wr.buf = wr.buf[:0]
+}
+
+func (wr *writeRequest) flush() {
+	wr.wr.Timeseries = wr.tss
+	wr.lastFlushTime = time.Now()
+	pushWriteRequest(&wr.wr, wr.pushBlock)
+	wr.reset()
+}
+
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+	tssDst := wr.tss
+	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		dst := &tssDst[len(tssDst)-1]
+		wr.copyTimeSeries(dst, &src[i])
+		if len(wr.tss) >= maxRowsPerBlock {
+			wr.flush()
+			tssDst = wr.tss
+		}
+	}
+	wr.tss = tssDst
+}
+
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
+	labelsLen := len(wr.labels)
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
+
+		buf = append(buf, srcLabel.Name...)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
+		buf = append(buf, srcLabel.Value...)
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
+	}
+	dst.Labels = labelsDst[labelsLen:]
+
+	samplesDst = append(samplesDst, prompbmarshal.Sample{})
+	dstSample := &samplesDst[len(samplesDst)-1]
+	if len(src.Samples) != 1 {
+		logger.Panicf("BUG: unexpected number of samples in time series; got %d; want 1", len(src.Samples))
+	}
+	*dstSample = src.Samples[0]
+	dst.Samples = samplesDst[len(samplesDst)-1:]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
+}
+
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+	if len(wr.Timeseries) == 0 {
+		// Nothing to push
+		return
+	}
+	bb := writeRequestBufPool.Get()
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	if len(bb.B) <= *maxUnpackedBlockSize {
+		zb := snappyBufPool.Get()
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		writeRequestBufPool.Put(bb)
+		if len(zb.B) <= persistentqueue.MaxBlockSize {
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
+		}
+		snappyBufPool.Put(zb)
+	} else {
+		writeRequestBufPool.Put(bb)
+	}
+
+	// Too big block. Recursively split it into smaller parts.
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries
+}
+
+var (
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+)
+
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/relabel.go

@@ -0,0 +1,113 @@
+package remotewrite
+
+import (
+	"flag"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+var (
+	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+)
+
+var labelsGlobal []prompbmarshal.Label
+var prcsGlobal []promrelabel.ParsedRelabelConfig
+
+// initRelabelGlobal must be called after parsing command-line flags.
+func initRelabelGlobal() {
+	// Init labelsGlobal
+	labelsGlobal = nil
+	for _, s := range *unparsedLabelsGlobal {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Panicf("FATAL: missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
+			Name:  s[:n],
+			Value: s[n+1:],
+		})
+	}
+
+	// Init prcsGlobal
+	prcsGlobal = nil
+	if len(*relabelConfigPathGlobal) > 0 {
+		var err error
+		prcsGlobal, err = promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		if err != nil {
+			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.relabelConfig=%q: %s", *relabelConfigPathGlobal, err)
+		}
+	}
+}
+
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && len(prcs) == 0 {
+		// Nothing to change.
+		return tss
+	}
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
+		if len(labels) == labelsLen {
+			// Drop the current time series, since relabeling removed all the labels.
+			continue
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	return tssDst
+}
+
+type relabelCtx struct {
+	// pool for labels, which are used during the relabeling.
+	labels []prompbmarshal.Label
+}
+
+func (rctx *relabelCtx) reset() {
+	labels := rctx.labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	rctx.labels = rctx.labels[:0]
+}
+
+var relabelCtxPool = &sync.Pool{
+	New: func() interface{} {
+		return &relabelCtx{}
+	},
+}
+
+func getRelabelCtx() *relabelCtx {
+	return relabelCtxPool.Get().(*relabelCtx)
+}
+
+func putRelabelCtx(rctx *relabelCtx) {
+	rctx.labels = rctx.labels[:0]
+	relabelCtxPool.Put(rctx)
+}

app/vmagent/remotewrite/remotewrite.go

@@ -0,0 +1,195 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var (
+	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+	tmpDataPath        = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
+	queues             = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+		"isn't enough for sending high volume of collected data to remote storage")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensistive auth info")
+	maxPendingBytesPerURL = flag.Int("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
+		"Disk usage is unlimited if the value is set to 0")
+)
+
+var rwctxs []*remoteWriteCtx
+
+// Init initializes remotewrite.
+//
+// It must be called after flag.Parse().
+//
+// Stop must be called for graceful shutdown.
+func Init() {
+	if len(*remoteWriteURLs) == 0 {
+		logger.Panicf("FATAL: at least one `-remoteWrite.url` must be set")
+	}
+
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		httpserver.RegisterSecretFlag("remoteWrite.url")
+	}
+	initRelabelGlobal()
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
+	if maxInmemoryBlocks > 200 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 200
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	for i, remoteWriteURL := range *remoteWriteURLs {
+		relabelConfigPath := ""
+		if i < len(*relabelConfigPaths) {
+			relabelConfigPath = (*relabelConfigPaths)[i]
+		}
+		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		if *showRemoteWriteURL {
+			urlLabelValue = remoteWriteURL
+		}
+		rwctx := newRemoteWriteCtx(remoteWriteURL, relabelConfigPath, maxInmemoryBlocks, urlLabelValue)
+		rwctxs = append(rwctxs, rwctx)
+	}
+}
+
+// Stop stops remotewrite.
+//
+// It is expected that nobody calls Push during and after the call to this func.
+func Stop() {
+	for _, rwctx := range rwctxs {
+		rwctx.MustStop()
+	}
+	rwctxs = nil
+}
+
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+//
+// Each timeseries in wr.Timeseries must contain one sample.
+func Push(wr *prompbmarshal.WriteRequest) {
+	var rctx *relabelCtx
+	if len(prcsGlobal) > 0 || len(labelsGlobal) > 0 {
+		rctx = getRelabelCtx()
+	}
+	tss := wr.Timeseries
+	for len(tss) > 0 {
+		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		tssBlock := tss
+		if len(tssBlock) > maxRowsPerBlock {
+			tssBlock = tss[:maxRowsPerBlock]
+			tss = tss[maxRowsPerBlock:]
+		} else {
+			tss = nil
+		}
+		if rctx != nil {
+			tssBlockLen := len(tssBlock)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, prcsGlobal)
+			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+		}
+		for _, rwctx := range rwctxs {
+			rwctx.Push(tssBlock)
+		}
+		if rctx != nil {
+			rctx.reset()
+		}
+	}
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}
+
+var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+
+type remoteWriteCtx struct {
+	fq         *persistentqueue.FastQueue
+	c          *client
+	prcs       []promrelabel.ParsedRelabelConfig
+	pss        []*pendingSeries
+	pssNextIdx uint64
+
+	relabelMetricsDropped *metrics.Counter
+}
+
+func newRemoteWriteCtx(remoteWriteURL, relabelConfigPath string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+	h := xxhash.Sum64([]byte(remoteWriteURL))
+	path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
+	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, *maxPendingBytesPerURL)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetPendingBytes())
+	})
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetInmemoryQueueLen())
+	})
+	c := newClient(remoteWriteURL, urlLabelValue, fq, *queues)
+	var prcs []promrelabel.ParsedRelabelConfig
+	if len(relabelConfigPath) > 0 {
+		var err error
+		prcs, err = promrelabel.LoadRelabelConfigs(relabelConfigPath)
+		if err != nil {
+			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %s", relabelConfigPath, err)
+		}
+	}
+	pss := make([]*pendingSeries, *queues)
+	for i := range pss {
+		pss[i] = newPendingSeries(fq.MustWriteBlock)
+	}
+	return &remoteWriteCtx{
+		fq:   fq,
+		c:    c,
+		prcs: prcs,
+		pss:  pss,
+
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+	}
+}
+
+func (rwctx *remoteWriteCtx) MustStop() {
+	for _, ps := range rwctx.pss {
+		ps.MustStop()
+	}
+	rwctx.pss = nil
+	rwctx.fq.MustClose()
+	rwctx.fq = nil
+	rwctx.prcs = nil
+	rwctx.c.MustStop()
+	rwctx.c = nil
+
+	rwctx.relabelMetricsDropped = nil
+}
+
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+	var rctx *relabelCtx
+	if len(rwctx.prcs) > 0 {
+		rctx = getRelabelCtx()
+		tssLen := len(tss)
+		tss = rctx.applyRelabeling(tss, nil, rwctx.prcs)
+		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+	}
+	pss := rwctx.pss
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,71 @@
+package remotewrite
+
+import (
+	"net"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/fasthttp"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func statDial(addr string) (net.Conn, error) {
+	conn, err := fasthttp.Dial(addr)
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/vmimport/request_handler.go

@@ -0,0 +1,70 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		samplesLen := len(samples)
+		for j, value := range values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     value,
+				Timestamp: timestamps[j],
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(values)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/Makefile

@@ -0,0 +1,81 @@
+# All these commands must run from repository root.
+
+vmalert:
+	APP_NAME=vmalert $(MAKE) app-local
+
+vmalert-race:
+	APP_NAME=vmalert RACE=-race $(MAKE) app-local
+
+vmalert-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker
+
+vmalert-pure-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-pure
+
+vmalert-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+
+vmalert-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+
+vmalert-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+
+vmalert-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+
+vmalert-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-386
+
+package-vmalert:
+	APP_NAME=vmalert $(MAKE) package-via-docker
+
+package-vmalert-pure:
+	APP_NAME=vmalert $(MAKE) package-via-docker-pure
+
+package-vmalert-amd64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-amd64
+
+package-vmalert-arm:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm
+
+package-vmalert-arm64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm64
+
+package-vmalert-ppc64le:
+	APP_NAME=vmalert $(MAKE) package-via-docker-ppc64le
+
+package-vmalert-386:
+	APP_NAME=vmalert $(MAKE) package-via-docker-386
+
+publish-vmalert:
+	APP_NAME=vmalert $(MAKE) publish-via-docker
+
+test-vmalert:
+	go test -race -cover ./app/vmalert
+
+run-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/testdata/rules0-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-notifier.url=http://localhost:9093 \
+		-remotewrite.url=http://localhost:8428 \
+		-remoteread.url=http://localhost:8428 \
+		-evaluationInterval=3s
+
+vmalert-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-amd64 ./app/vmalert
+
+vmalert-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm ./app/vmalert
+
+vmalert-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm64 ./app/vmalert
+
+vmalert-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-ppc64le ./app/vmalert
+
+vmalert-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-386 ./app/vmalert
+
+vmalert-pure:
+	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/README.md

@@ -0,0 +1,116 @@
+## VM Alert
+
+`vmalert` executes a list of given MetricsQL expressions (rules) and
+sends alerts to [Alert Manager](https://github.com/prometheus/alertmanager).   
+
+### Features:
+* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
+* VictoriaMetrics [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+ expressions validation;
+* Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
+ support;
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Lightweight without extra dependencies.
+
+### TODO:
+* Configuration hot reload.
+
+### QuickStart
+
+To build `vmalert` from sources:
+```
+git clone https://github.com/VictoriaMetrics/VictoriaMetrics
+cd VictoriaMetrics
+make vmalert
+```
+The build binary will be placed to `VictoriaMetrics/bin` folder.
+
+To start using `vmalert` you will need the following things:
+* list of alert rules - PromQL/MetricsQL expressions to execute;
+* datasource address - reachable VictoriaMetrics instance for rules execution;
+* notifier address - reachable Alertmanager instance for processing, 
+aggregating alerts and sending notifications.
+
+Then configure `vmalert` accordingly:
+```
+./bin/vmalert -rule=alert.rules \
+		-datasource.url=http://localhost:8428 \
+        -notifier.url=http://localhost:9093
+```
+
+Example for `.rules` file may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/testdata/rules0-good.rules)
+
+`vmalert` runs evaluation for every group in a separate goroutine.
+Rules in group evaluated one-by-one sequentially. 
+
+`vmalert` also runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
+* `http://<vmalert-addr>/api/v1/<groupName>/<alertID>/status" ` - get alert status by ID.
+Used as alert source in AlertManager.
+* `http://<vmalert-addr>/metrics` - application metrics.
+
+`vmalert` may be configured with `-remotewrite` flag to write alerts state in form of timeseries
+via remote write protocol. Alerts state will be written as `ALERTS` timeseries. These timeseries
+may be used to recover alerts state on `vmalert` restarts if `-remoteread` is configured.
+
+
+### Configuration
+
+The shortlist of configuration flags is the following:
+```
+Usage of vmalert:
+  -datasource.basicAuth.password string
+        Optional basic auth password for -datasource.url
+  -datasource.basicAuth.username string
+        Optional basic auth username for -datasource.url
+  -datasource.url string
+        Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
+  -enableTCP6
+        Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+  -evaluationInterval duration
+        How often to evaluate the rules. Default 1m (default 1m0s)
+  -external.url string
+        External URL is used as alert's source for sent alerts to the notifier
+  -http.maxGracefulShutdownDuration duration
+        The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+  -httpAuth.password string
+        Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+        Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+        Address to listen for http connections (default ":8880")
+  -notifier.url string
+        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+  -remoteread.basicAuth.password string
+        Optional basic auth password for -remoteread.url
+  -remoteread.basicAuth.username string
+        Optional basic auth username for -remoteread.url
+  -remoteread.lookback duration
+        Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteread.url vmalert
+        Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remotewrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+  -remotewrite.basicAuth.password string
+        Optional basic auth password for -remotewrite.url
+  -remotewrite.basicAuth.username string
+        Optional basic auth username for -remotewrite.url
+  -remotewrite.url string
+        Optional URL to Victoria Metrics or VMInsert where to persist alerts state in form of timeseries. E.g. http://127.0.0.1:8428
+  -rule value
+        Path to the file with alert rules. 
+        Supports patterns. Flag can be specified multiple times. 
+        Examples:
+         -rule /path/to/file. Path to a single file with alerting rules
+         -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+        absolute path to all .yaml files in root.
+  -rule.validateTemplates
+        Indicates to validate annotation and label templates (default true)
+```
+
+Pass `-help` to `vmalert` in order to see the full list of supported 
+command-line flags with their descriptions.
+
+### Contributing
+
+`vmalert` is mostly designed and built by VictoriaMetrics community.
+Feel free to share your experience and ideas for improving this 
+software. Please keep simplicity as the main priority.

app/vmalert/config.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateAnnotations bool) ([]Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file patther %s:%v", pattern, err)
+		}
+		fp = append(fp, matches...)
+	}
+	var groups []Group
+	for _, file := range fp {
+		groupsNames := map[string]struct{}{}
+		gr, err := parseFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("file %s: %w", file, err)
+		}
+		for _, group := range gr {
+			if _, ok := groupsNames[group.Name]; ok {
+				return nil, fmt.Errorf("one file can not contain groups with the same name %s, filepath:%s", file, group.Name)
+			}
+			groupsNames[group.Name] = struct{}{}
+			for _, rule := range group.Rules {
+				if err = rule.Validate(); err != nil {
+					return nil, fmt.Errorf("invalid rule filepath:%s, group %s:%w", file, group.Name, err)
+				}
+				// TODO: this init looks weird here
+				rule.alerts = make(map[uint64]*notifier.Alert)
+				if validateAnnotations {
+					if err = notifier.ValidateTemplates(rule.Annotations); err != nil {
+						return nil, fmt.Errorf("invalid annotations filepath:%s, group %s:%w", file, group.Name, err)
+					}
+					if err = notifier.ValidateTemplates(rule.Labels); err != nil {
+						return nil, fmt.Errorf("invalid labels filepath:%s, group %s:%w", file, group.Name, err)
+					}
+				}
+				rule.group = group
+			}
+		}
+		groups = append(groups, gr...)
+	}
+	if len(groups) < 1 {
+		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parseFile(path string) ([]Group, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
+	}
+	g := struct {
+		Groups []Group `yaml:"groups"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	return g.Groups, err
+}

app/vmalert/config_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected syntaxt error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected template annotation error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules1-bad.rules"}, true); err == nil {
+		t.Errorf("expected same group error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules2-bad.rules"}, true); err == nil {
+		t.Errorf("expected template label error")
+	}
+	if _, err := Parse([]string{"testdata/*.yaml"}, true); err == nil {
+		t.Errorf("expected empty group")
+	}
+}

app/vmalert/datasource/datasource.go

@@ -0,0 +1,24 @@
+package datasource
+
+import "context"
+
+// Querier interface wraps Query method which
+// executes given query and returns list of Metrics
+// as result
+type Querier interface {
+	Query(ctx context.Context, query string) ([]Metric, error)
+}
+
+// Metric is the basic entity which should be return by datasource
+// It represents single data point with full list of labels
+type Metric struct {
+	Labels    []Label
+	Timestamp int64
+	Value     float64
+}
+
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}

app/vmalert/datasource/vm.go

@@ -0,0 +1,103 @@
+package datasource
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+type response struct {
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string `json:"resultType"`
+		Result     []struct {
+			Labels map[string]string `json:"metric"`
+			TV     [2]interface{}    `json:"value"`
+		} `json:"result"`
+	} `json:"data"`
+	ErrorType string `json:"errorType"`
+	Error     string `json:"error"`
+}
+
+func (r response) metrics() ([]Metric, error) {
+	var ms []Metric
+	var m Metric
+	var f float64
+	var err error
+	for i, res := range r.Data.Result {
+		f, err = strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %s", res, res.TV[1], err)
+		}
+		m.Labels = nil
+		for k, v := range r.Data.Result[i].Labels {
+			m.Labels = append(m.Labels, Label{Name: k, Value: v})
+		}
+		m.Timestamp = int64(res.TV[0].(float64))
+		m.Value = f
+		ms = append(ms, m)
+	}
+	return ms, nil
+}
+
+const queryPath = "/api/v1/query?query="
+
+// VMStorage represents vmstorage entity with ability to read and write metrics
+type VMStorage struct {
+	c                            *http.Client
+	queryURL                     string
+	basicAuthUser, basicAuthPass string
+}
+
+// NewVMStorage is a constructor for VMStorage
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+	return &VMStorage{
+		c:             c,
+		basicAuthUser: basicAuthUser,
+		basicAuthPass: basicAuthPass,
+		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+	}
+}
+
+// Query reads metrics from datasource by given query
+func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+	const (
+		statusSuccess, statusError, rtVector = "success", "error", "vector"
+	)
+	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if s.basicAuthPass != "" {
+		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	}
+	resp, err := s.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("error getting response from %s:%s", req.URL, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return nil, fmt.Errorf("datasource returns unxeprected response code %d for %s with err %s. Reponse body %s", resp.StatusCode, req.URL, err, body)
+	}
+	r := &response{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing metrics for %s:%s", req.URL, err)
+	}
+	if r.Status == statusError {
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+	}
+	if r.Status != statusSuccess {
+		return nil, fmt.Errorf("unkown status:%s, Expected success or error ", r.Status)
+	}
+	if r.Data.ResultType != rtVector {
+		return nil, fmt.Errorf("unkown restul type:%s. Expected vector", r.Data.ResultType)
+	}
+	return r.metrics()
+}

app/vmalert/datasource/vm_test.go

@@ -0,0 +1,93 @@
+package datasource
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var (
+	ctx           = context.Background()
+	basicAuthName = "foo"
+	basicAuthPass = "bar"
+	query         = "vm_rows"
+)
+
+func TestVMSelectQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("exptected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			w.Write([]byte("[]"))
+		case 3:
+			w.Write([]byte(`{"status":"error", "errorType":"type:", "error":"some error msg"}`))
+		case 4:
+			w.Write([]byte(`{"status":"unknown"}`))
+		case 5:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
+		case 6:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]}]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected connection error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected invalid response status error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected response body error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected error status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected unkown status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected non-vector resultType error  got nil")
+	}
+	m, err := am.Query(ctx, query)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("exptected 1 metric  got %d in %+v", len(m), m)
+	}
+	expected := Metric{
+		Labels:    []Label{{Value: "vm_rows", Name: "__name__"}},
+		Timestamp: 1583786142,
+		Value:     13763,
+	}
+	if m[0].Timestamp != expected.Timestamp &&
+		m[0].Value != expected.Value &&
+		m[0].Labels[0].Value != expected.Labels[0].Value &&
+		m[0].Labels[0].Name != expected.Labels[0].Name {
+		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+	}
+
+}

app/vmalert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-prod"]
+ARG src_binary
+COPY $src_binary ./vmalert-prod

app/vmalert/main.go

@@ -0,0 +1,239 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules. 
+Supports patterns. Flag can be specified multiple times. 
+Examples:
+ -rule /path/to/file. Path to a single file with alerting rules
+ -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+absolute path to all .yaml files in root.`)
+	validateTemplates = flag.Bool("rule.validateTemplates", true, "Indicates to validate annotation and label templates")
+	httpListenAddr    = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+
+	datasourceURL = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+
+	remoteWriteURL = flag.String("remotewrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
+		" in form of timeseries. E.g. http://127.0.0.1:8428")
+	remoteWriteUsername = flag.String("remotewrite.basicAuth.username", "", "Optional basic auth username for -remotewrite.url")
+	remoteWritePassword = flag.String("remotewrite.basicAuth.password", "", "Optional basic auth password for -remotewrite.url")
+
+	remoteReadURL = flag.String("remoteread.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
+		" state. This configuration makes sense only if `vmalert` was configured with `remotewrite.url` before and has been successfully persisted its state."+
+		" E.g. http://127.0.0.1:8428")
+	remoteReadUsername = flag.String("remoteread.basicAuth.username", "", "Optional basic auth username for -remoteread.url")
+	remoteReadPassword = flag.String("remoteread.basicAuth.password", "", "Optional basic auth password for -remoteread.url")
+	remoteReadLookBack = flag.Duration("remoteread.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
+		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+
+	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules. Default 1m")
+	notifierURL        = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	externalURL        = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+)
+
+// TODO: hot configuration reload
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	checkFlags()
+	ctx, cancel := context.WithCancel(context.Background())
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, false)
+	if err != nil {
+		logger.Fatalf("can not get external url:%s ", err)
+	}
+	notifier.InitTemplateFunc(eu)
+
+	logger.Infof("reading alert rules configuration file from %s", strings.Join(*rulePath, ";"))
+	groups, err := Parse(*rulePath, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("cannot parse configuration file: %s", err)
+	}
+
+	w := &watchdog{
+		storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
+		alertProvider: notifier.NewAlertManager(*notifierURL, func(group, name string) string {
+			return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, name)
+		}, &http.Client{}),
+	}
+
+	if *remoteWriteURL != "" {
+		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
+			Addr:          *remoteWriteURL,
+			FlushInterval: *evaluationInterval,
+			BasicAuthUser: *remoteWriteUsername,
+			BasicAuthPass: *remoteWritePassword,
+		})
+		if err != nil {
+			logger.Fatalf("failed to init remotewrite client: %s", err)
+		}
+		w.rw = c
+	}
+
+	var restoreDS *datasource.VMStorage
+	if *remoteReadURL != "" {
+		restoreDS = datasource.NewVMStorage(*remoteReadURL, *remoteReadUsername, *remoteReadPassword, &http.Client{})
+	}
+
+	wg := sync.WaitGroup{}
+	for _, g := range groups {
+		if restoreDS != nil {
+			err := g.Restore(ctx, restoreDS, *remoteReadLookBack)
+			if err != nil {
+				logger.Errorf("error while restoring state for group %q: %s", g.Name, err)
+			}
+		}
+		wg.Add(1)
+		go func(group Group) {
+			w.run(ctx, group, *evaluationInterval)
+			wg.Done()
+		}(g)
+	}
+
+	go httpserver.Serve(*httpListenAddr, (&requestHandler{groups: groups}).handler)
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	cancel()
+	if w.rw != nil {
+		err := w.rw.Close()
+		if err != nil {
+			logger.Fatalf("cannot stop the remotewrite: %s", err)
+		}
+	}
+	wg.Wait()
+}
+
+type watchdog struct {
+	storage       *datasource.VMStorage
+	alertProvider notifier.Notifier
+	rw            *remotewrite.Client
+}
+
+var (
+	iterationTotal    = metrics.NewCounter(`vmalert_iteration_total`)
+	iterationDuration = metrics.NewSummary(`vmalert_iteration_duration_seconds`)
+
+	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
+	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+
+	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
+	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
+	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
+
+	remoteWriteSent   = metrics.NewCounter(`vmalert_remotewrite_sent_total`)
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+)
+
+func (w *watchdog) run(ctx context.Context, group Group, evaluationInterval time.Duration) {
+	logger.Infof("watchdog for %s has been started", group.Name)
+	t := time.NewTicker(evaluationInterval)
+	defer t.Stop()
+	for {
+
+		select {
+		case <-t.C:
+			iterationTotal.Inc()
+			iterationStart := time.Now()
+			for _, rule := range group.Rules {
+				execTotal.Inc()
+
+				execStart := time.Now()
+				err := rule.Exec(ctx, w.storage)
+				execDuration.UpdateDuration(execStart)
+
+				if err != nil {
+					execErrors.Inc()
+					logger.Errorf("failed to execute rule %q.%q: %s", group.Name, rule.Name, err)
+					continue
+				}
+
+				var alertsToSend []notifier.Alert
+				for _, a := range rule.alerts {
+					if a.State != notifier.StatePending {
+						alertsToSend = append(alertsToSend, *a)
+					}
+					if a.State == notifier.StateInactive || w.rw == nil {
+						continue
+					}
+					tss := rule.AlertToTimeSeries(a, execStart)
+					for _, ts := range tss {
+						remoteWriteSent.Inc()
+						if err := w.rw.Push(ts); err != nil {
+							remoteWriteErrors.Inc()
+							logger.Errorf("failed to push timeseries to remotewrite: %s", err)
+						}
+					}
+				}
+				alertsSent.Add(len(alertsToSend))
+				if err := w.alertProvider.Send(alertsToSend); err != nil {
+					alertsSendErrors.Inc()
+					logger.Errorf("failed to send alert for rule %q.%q: %s", group.Name, rule.Name, err)
+				}
+			}
+			iterationDuration.UpdateDuration(iterationStart)
+		case <-ctx.Done():
+			logger.Infof("%s received stop signal", group.Name)
+			return
+		}
+	}
+}
+
+func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
+	if externalURL != "" {
+		return url.Parse(externalURL)
+	}
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+	port := ""
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+		port = ":" + ipport[1]
+	}
+	schema := "http://"
+	if isSecure {
+		schema = "https://"
+	}
+	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
+}
+
+func checkFlags() {
+	if *notifierURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("notifier.url is empty")
+	}
+	if *datasourceURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("datasource.url is empty")
+	}
+}

app/vmalert/notifier/alert.go

@@ -0,0 +1,105 @@
+package notifier
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+	"time"
+)
+
+// Alert the triggered alert
+// TODO: Looks like alert name isn't unique
+type Alert struct {
+	Group       string
+	Name        string
+	Labels      map[string]string
+	Annotations map[string]string
+	State       AlertState
+
+	Start time.Time
+	End   time.Time
+	Value float64
+	ID    uint64
+}
+
+// AlertState type indicates the Alert state
+type AlertState int
+
+const (
+	// StateInactive is the state of an alert that is neither firing nor pending.
+	StateInactive AlertState = iota
+	// StatePending is the state of an alert that has been active for less than
+	// the configured threshold duration.
+	StatePending
+	// StateFiring is the state of an alert that has been active for longer than
+	// the configured threshold duration.
+	StateFiring
+)
+
+// String stringer for AlertState
+func (as AlertState) String() string {
+	switch as {
+	case StateFiring:
+		return "firing"
+	case StatePending:
+		return "pending"
+	}
+	return "inactive"
+}
+
+type alertTplData struct {
+	Labels map[string]string
+	Value  float64
+}
+
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}`
+
+// ExecTemplate executes the Alert template for give
+// map of annotations.
+func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
+	tplData := alertTplData{Value: a.Value, Labels: a.Labels}
+	return templateAnnotations(annotations, tplHeader, tplData)
+}
+
+// ValidateTemplates validate annotations for possible template error, uses empty data for template population
+func ValidateTemplates(annotations map[string]string) error {
+	_, err := templateAnnotations(annotations, tplHeader, alertTplData{
+		Labels: map[string]string{},
+		Value:  0,
+	})
+	return err
+}
+
+func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	eg := errGroup{}
+	r := make(map[string]string, len(annotations))
+	for key, text := range annotations {
+		r[key] = text
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
+			eg.errs = append(eg.errs, fmt.Sprintf("key %s, template %s:%s", key, text, err))
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r, eg.err()
+}
+
+func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
+	tpl, err := template.New("").Funcs(tmplFunc).Option("missingkey=zero").Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation:%w", err)
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template:%w", err)
+	}
+	return nil
+}

app/vmalert/notifier/alert_test.go

@@ -0,0 +1,65 @@
+package notifier
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestAlert_ExecTemplate(t *testing.T) {
+	testCases := []struct {
+		alert       *Alert
+		annotations map[string]string
+		expTpl      map[string]string
+	}{
+		{
+			alert:       &Alert{},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}}",
+			},
+			expTpl: map[string]string{
+				"summary":     "Too high connection number for localhost for job staging",
+				"description": "It is 10000 connections for localhost",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			tpl, err := tc.alert.ExecTemplate(tc.annotations)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(tpl) != len(tc.expTpl) {
+				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
+			}
+			for k := range tc.expTpl {
+				got, exp := tpl[k], tc.expTpl[k]
+				if got != exp {
+					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
+				}
+			}
+		})
+	}
+}

app/vmalert/notifier/alertmanager.go

@@ -0,0 +1,51 @@
+package notifier
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// AlertManager represents integration provider with Prometheus alert manager
+// https://github.com/prometheus/alertmanager
+type AlertManager struct {
+	alertURL string
+	argFunc  AlertURLGenerator
+	client   *http.Client
+}
+
+// Send an alert or resolve message
+func (am *AlertManager) Send(alerts []Alert) error {
+	b := &bytes.Buffer{}
+	writeamRequest(b, alerts, am.argFunc)
+	resp, err := am.client.Post(am.alertURL, "application/json", b)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to read response from %q: %s", am.alertURL, err)
+		}
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+	}
+	return nil
+}
+
+// AlertURLGenerator returns URL to single alert by given name
+type AlertURLGenerator func(group, id string) string
+
+const alertManagerPath = "/api/v2/alerts"
+
+// NewAlertManager is a constructor for AlertManager
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+	return &AlertManager{
+		alertURL: strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath,
+		argFunc:  fn,
+		client:   c,
+	}
+}

app/vmalert/notifier/alertmanager_request.qtpl

@@ -0,0 +1,34 @@
+{% import (
+    "strconv"
+    "time"
+) %}
+{% stripspace %}
+
+{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %}
+[
+{% for i, alert := range alerts %}
+{
+	"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
+	"generatorURL": {%q= generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)) %},
+    {% if !alert.End.IsZero() %}
+    "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
+    {% endif %}
+    "labels": {
+        "alertname":{%q= alert.Name %}
+        {% for k,v := range alert.Labels %}
+            ,{%q= k %}:{%q= v %}
+        {% endfor %}
+    },
+    "annotations": {
+        {% code c := len(alert.Annotations) %}
+        {% for k,v := range alert.Annotations %}
+          {% code c = c-1 %}
+          {%q= k %}:{%q= v %}{% if c > 0 %},{% endif %}
+        {% endfor %}
+    }
+}
+{% if i != len(alerts)-1 %},{% endif %}
+{% endfor %}
+]
+{% endfunc %}
+{% endstripspace %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -0,0 +1,131 @@
+// Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
+package notifier
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
+import (
+	"strconv"
+	"time"
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+	qw422016.N().S(`[`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
+	for i, alert := range alerts {
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
+		qw422016.N().S(`{"startsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		qw422016.N().S(`,"generatorURL":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		qw422016.N().Q(generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)))
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
+		if !alert.End.IsZero() {
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
+			qw422016.N().S(`"endsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
+		qw422016.N().Q(alert.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		for k, v := range alert.Labels {
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(k)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(v)
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
+		c := len(alert.Annotations)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:24
+		for k, v := range alert.Annotations {
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
+			c = c - 1
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(k)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(v)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			if c > 0 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+				qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			}
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
+		qw422016.N().S(`}}`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+		if i != len(alerts)-1 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
+	}
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
+	qw422016.N().S(`]`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	streamamRequest(qw422016, alerts, generatorURL)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	writeamRequest(qb422016, alerts, generatorURL)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qs422016 := string(qb422016.B)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	return qs422016
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}

app/vmalert/notifier/alertmanager_test.go

@@ -0,0 +1,80 @@
+package notifier
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAlertManager_Send(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			var a []struct {
+				Labels       map[string]string `json:"labels"`
+				StartsAt     time.Time         `json:"startsAt"`
+				EndAt        time.Time         `json:"endsAt"`
+				Annotations  map[string]string `json:"annotations"`
+				GeneratorURL string            `json:"generatorURL"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
+			}
+			if len(a) != 1 {
+				t.Errorf("expected 1 alert in array got %d", len(a))
+			}
+			if a[0].GeneratorURL != "group0" {
+				t.Errorf("exptected alert0 as generatorURL got %s", a[0].GeneratorURL)
+			}
+			if a[0].Labels["alertname"] != "alert0" {
+				t.Errorf("exptected alert0 as alert name got %s", a[0].Labels["alertname"])
+			}
+			if a[0].StartsAt.IsZero() {
+				t.Errorf("exptected non-zero start time")
+			}
+			if a[0].EndAt.IsZero() {
+				t.Errorf("exptected non-zero end time")
+			}
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewAlertManager(srv.URL, func(group, name string) string {
+		return group + name
+	}, srv.Client())
+	if err := am.Send([]Alert{{}, {}}); err == nil {
+		t.Error("expected connection error got nil")
+	}
+	if err := am.Send([]Alert{}); err == nil {
+		t.Error("expected wrong http code error got nil")
+	}
+	if err := am.Send([]Alert{{
+		Group:       "group",
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if c != 2 {
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+	}
+}

app/vmalert/notifier/notifier.go

@@ -0,0 +1,6 @@
+package notifier
+
+// Notifier is common interface for alert manager provider
+type Notifier interface {
+	Send(alerts []Alert) error
+}

app/vmalert/notifier/template_func.go

@@ -0,0 +1,171 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package notifier
+
+import (
+	"fmt"
+	html_template "html/template"
+	"math"
+	"net/url"
+	"regexp"
+	"strings"
+	text_template "text/template"
+	"time"
+)
+
+var tmplFunc text_template.FuncMap
+
+// InitTemplateFunc returns template helper functions
+func InitTemplateFunc(externalURL *url.URL) {
+	tmplFunc = text_template.FuncMap{
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+		"safeHtml": func(text string) html_template.HTML {
+			return html_template.HTML(text)
+		},
+		"match":   regexp.MatchString,
+		"title":   strings.Title,
+		"toUpper": strings.ToUpper,
+		"toLower": strings.ToLower,
+		"humanize": func(v float64) string {
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanize1024": func(v float64) string {
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			prefix := ""
+			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
+				if math.Abs(v) < 1024 {
+					break
+				}
+				prefix = p
+				v /= 1024
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanizeDuration": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v)
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix)
+		},
+		"humanizePercentage": func(v float64) string {
+			return fmt.Sprintf("%.4g%%", v*100)
+		},
+		"humanizeTimestamp": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
+			return fmt.Sprint(t)
+		},
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+	}
+}
+
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// TimeFromUnixNano returns the Time equivalent to the Unix Time
+// t provided in nanoseconds.
+func TimeFromUnixNano(t int64) Time {
+	return Time(t / nanosPerTick)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}

app/vmalert/notifier/utils.go

@@ -0,0 +1,21 @@
+package notifier
+
+import (
+	"fmt"
+	"strings"
+)
+
+type errGroup struct {
+	errs []string
+}
+
+func (eg *errGroup) err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+func (eg *errGroup) Error() string {
+	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
+}

app/vmalert/remotewrite/remotewrite.go

@@ -0,0 +1,188 @@
+package remotewrite
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+// Client is an asynchronous HTTP client for writing
+// timeseries via remote write protocol.
+type Client struct {
+	addr           string
+	c              *http.Client
+	input          chan prompbmarshal.TimeSeries
+	baUser, baPass string
+	flushInterval  time.Duration
+	maxBatchSize   int
+	maxQueueSize   int
+
+	wg     sync.WaitGroup
+	doneCh chan struct{}
+}
+
+// Config is config for remote write.
+type Config struct {
+	// Addr of remote storage
+	Addr string
+
+	BasicAuthUser string
+	BasicAuthPass string
+
+	// MaxBatchSize defines max number of timeseries
+	// to be flushed at once
+	MaxBatchSize int
+	// MaxQueueSize defines max length of input queue
+	// populated by Push method
+	MaxQueueSize int
+	// FlushInterval defines time interval for flushing batches
+	FlushInterval time.Duration
+	// WriteTimeout defines timeout for HTTP write request
+	// to remote storage
+	WriteTimeout time.Duration
+}
+
+const (
+	defaultMaxBatchSize  = 1e3
+	defaultMaxQueueSize  = 100
+	defaultFlushInterval = 5 * time.Second
+	defaultWriteTimeout  = 30 * time.Second
+)
+
+const writePath = "/api/v1/write"
+
+// NewClient returns asynchronous client for
+// writing timeseries via remotewrite protocol.
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Addr == "" {
+		return nil, fmt.Errorf("config.Addr can't be empty")
+	}
+	if cfg.MaxBatchSize == 0 {
+		cfg.MaxBatchSize = defaultMaxBatchSize
+	}
+	if cfg.MaxQueueSize == 0 {
+		cfg.MaxQueueSize = defaultMaxQueueSize
+	}
+	if cfg.FlushInterval == 0 {
+		cfg.FlushInterval = defaultFlushInterval
+	}
+	if cfg.WriteTimeout == 0 {
+		cfg.WriteTimeout = defaultWriteTimeout
+	}
+	c := &Client{
+		c: &http.Client{
+			Timeout: cfg.WriteTimeout,
+		},
+		addr:          strings.TrimSuffix(cfg.Addr, "/") + writePath,
+		baUser:        cfg.BasicAuthUser,
+		baPass:        cfg.BasicAuthPass,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+	}
+	c.run(ctx)
+	return c, nil
+}
+
+// Push adds timeseries into queue for writing into remote storage.
+// Push returns and error if client is stopped or if queue is full.
+func (c *Client) Push(s prompbmarshal.TimeSeries) error {
+	select {
+	case <-c.doneCh:
+		return fmt.Errorf("client is closed")
+	case c.input <- s:
+		return nil
+	default:
+		return fmt.Errorf("failed to push timeseries - queue is full (%d entries)",
+			c.maxQueueSize)
+	}
+}
+
+// Close stops the client and waits for all goroutines
+// to exit.
+func (c *Client) Close() error {
+	if c.doneCh == nil {
+		return fmt.Errorf("client is already closed")
+	}
+	close(c.input)
+	close(c.doneCh)
+	c.wg.Wait()
+	return nil
+}
+
+func (c *Client) run(ctx context.Context) {
+	ticker := time.NewTicker(c.flushInterval)
+	wr := prompbmarshal.WriteRequest{}
+	shutdown := func() {
+		for ts := range c.input {
+			wr.Timeseries = append(wr.Timeseries, ts)
+		}
+		lastCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		c.flush(lastCtx, wr)
+		cancel()
+	}
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer ticker.Stop()
+		for {
+			select {
+			case <-c.doneCh:
+				shutdown()
+				return
+			case <-ctx.Done():
+				shutdown()
+				return
+			case <-ticker.C:
+				c.flush(ctx, wr)
+				wr = prompbmarshal.WriteRequest{}
+			case ts := <-c.input:
+				wr.Timeseries = append(wr.Timeseries, ts)
+				if len(wr.Timeseries) >= c.maxBatchSize {
+					c.flush(ctx, wr)
+					wr = prompbmarshal.WriteRequest{}
+				}
+			}
+		}
+	}()
+}
+
+func (c *Client) flush(ctx context.Context, wr prompbmarshal.WriteRequest) {
+	if len(wr.Timeseries) < 1 {
+		return
+	}
+	data, err := wr.Marshal()
+	if err != nil {
+		logger.Errorf("failed to marshal WriteRequest: %s", err)
+		return
+	}
+	req, err := http.NewRequest("POST", c.addr, bytes.NewReader(snappy.Encode(nil, data)))
+	if err != nil {
+		logger.Errorf("failed to create new HTTP request: %s", err)
+		return
+	}
+	if c.baPass != "" {
+		req.SetBasicAuth(c.baUser, c.baPass)
+	}
+	resp, err := c.c.Do(req.WithContext(ctx))
+	if err != nil {
+		logger.Errorf("error getting response from %s:%s", req.URL, err)
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusNoContent {
+		body, _ := ioutil.ReadAll(resp.Body)
+		logger.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return
+	}
+}

app/vmalert/rule.go

@@ -0,0 +1,340 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// Group grouping array of alert
+type Group struct {
+	Name  string
+	Rules []*Rule
+}
+
+// Restore restores alerts state for all group rules with For > 0
+func (g *Group) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+	for _, rule := range g.Rules {
+		if rule.For == 0 {
+			return nil
+		}
+		if err := rule.Restore(ctx, q, lookback); err != nil {
+			return fmt.Errorf("error while restoring rule %q: %s", rule.Name, err)
+		}
+	}
+	return nil
+}
+
+// Rule is basic alert entity
+type Rule struct {
+	Name        string            `yaml:"alert"`
+	Expr        string            `yaml:"expr"`
+	For         time.Duration     `yaml:"for"`
+	Labels      map[string]string `yaml:"labels"`
+	Annotations map[string]string `yaml:"annotations"`
+
+	group Group
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+}
+
+// Validate validates rule
+func (r *Rule) Validate() error {
+	if r.Name == "" {
+		return errors.New("rule name can not be empty")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("expression for rule %q can't be empty", r.Name)
+	}
+	if _, err := metricsql.Parse(r.Expr); err != nil {
+		return fmt.Errorf("invalid expression for rule %q: %w", r.Name, err)
+	}
+	return nil
+}
+
+// Exec executes Rule expression via the given Querier.
+// Based on the Querier results Rule maintains notifier.Alerts
+func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
+	qMetrics, err := q.Query(ctx, r.Expr)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.lastExecError = err
+	r.lastExecTime = time.Now()
+	if err != nil {
+		return fmt.Errorf("failed to execute query %q: %s", r.Expr, err)
+	}
+
+	for h, a := range r.alerts {
+		// cleanup inactive alerts from previous Eval
+		if a.State == notifier.StateInactive {
+			delete(r.alerts, h)
+		}
+	}
+
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		h := hash(m)
+		updated[h] = struct{}{}
+		if a, ok := r.alerts[h]; ok {
+			// update Value field with latest value
+			a.Value = m.Value
+			continue
+		}
+		a, err := r.newAlert(m)
+		if err != nil {
+			r.lastExecError = err
+			return fmt.Errorf("failed to create alert: %s", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		r.alerts[h] = a
+	}
+
+	for h, a := range r.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			a.State = notifier.StateInactive
+			// set endTime to last execution time
+			// so it can be sent by notifier on next step
+			a.End = r.lastExecTime
+			continue
+		}
+		if a.State == notifier.StatePending && time.Since(a.Start) >= r.For {
+			a.State = notifier.StateFiring
+			alertsFired.Inc()
+		}
+		if a.State == notifier.StateFiring {
+			a.End = r.lastExecTime.Add(3 * *evaluationInterval)
+		}
+	}
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(m datasource.Metric) uint64 {
+	hash := fnv.New64a()
+	labels := m.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
+	a := &notifier.Alert{
+		Group:  r.group.Name,
+		Name:   r.Name,
+		Labels: map[string]string{},
+		Value:  m.Value,
+		Start:  time.Now(),
+		// TODO: support End time
+	}
+
+	// 1. use data labels
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		a.Labels[l.Name] = l.Value
+	}
+
+	// 2. template rule labels with data labels
+	rLabels, err := a.ExecTemplate(r.Labels)
+	if err != nil {
+		return a, err
+	}
+
+	// 3. merge data labels and rule labels
+	// metric labels may be overridden by
+	// rule labels
+	for k, v := range rLabels {
+		a.Labels[k] = v
+	}
+
+	// 4. template merged labels
+	a.Labels, err = a.ExecTemplate(a.Labels)
+	if err != nil {
+		return a, err
+	}
+
+	a.Annotations, err = a.ExecTemplate(r.Annotations)
+	return a, err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (r *Rule) AlertAPI(id uint64) *APIAlert {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	a, ok := r.alerts[id]
+	if !ok {
+		return nil
+	}
+	return r.newAlertAPI(*a)
+}
+
+// AlertsAPI generates list of APIAlert objects from existing alerts
+func (r *Rule) AlertsAPI() []*APIAlert {
+	var alerts []*APIAlert
+	r.mu.RLock()
+	for _, a := range r.alerts {
+		alerts = append(alerts, r.newAlertAPI(*a))
+	}
+	r.mu.RUnlock()
+	return alerts
+}
+
+func (r *Rule) newAlertAPI(a notifier.Alert) *APIAlert {
+	return &APIAlert{
+		// encode as string to avoid rounding
+		ID:          fmt.Sprintf("%d", a.ID),
+		Name:        a.Name,
+		Group:       a.Group,
+		Expression:  r.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.Start,
+		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+	}
+}
+
+const (
+	// AlertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// AlertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// AlertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+)
+
+// AlertToTimeSeries converts the given alert with the given timestamp to timeseries
+func (r *Rule) AlertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(r.Name, a, timestamp))
+	if r.For > 0 {
+		tss = append(tss, alertForToTimeSeries(r.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{}
+	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
+		Value:     value,
+		Timestamp: timestamp.UnixNano() / 1e6,
+	})
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}
+
+// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores only Start field. Field State will be always Pending and supposed
+// to be updated on next Eval, as well as Value field.
+func (r *Rule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+	// Get the last datapoint in range via MetricsQL `last_over_time`.
+	// We don't use plain PromQL since Prometheus doesn't support
+	// remote write protocol which is used for state persistence in vmalert.
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q}[%ds])",
+		alertForStateMetricName, r.Name, int(lookback.Seconds()))
+	qMetrics, err := q.Query(ctx, expr)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range qMetrics {
+		labels := m.Labels
+		m.Labels = make([]datasource.Label, 0)
+		// drop all extra labels, so hash key will
+		// be identical to timeseries received in Eval
+		for _, l := range labels {
+			if l.Name == alertNameLabel {
+				continue
+			}
+			// drop all overridden labels
+			if _, ok := r.Labels[l.Name]; ok {
+				continue
+			}
+			m.Labels = append(m.Labels, l)
+		}
+
+		a, err := r.newAlert(m)
+		if err != nil {
+			return fmt.Errorf("failed to create alert: %s", err)
+		}
+		a.ID = hash(m)
+		a.State = notifier.StatePending
+		a.Start = time.Unix(int64(m.Value), 0)
+		r.alerts[a.ID] = a
+		logger.Infof("alert %q.%q restored to state at %v", a.Group, a.Name, a.Start)
+	}
+	return nil
+}

app/vmalert/rule_test.go

@@ -0,0 +1,535 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestRule_Validate(t *testing.T) {
+	if err := (&Rule{}).Validate(); err == nil {
+		t.Errorf("exptected empty name error")
+	}
+	if err := (&Rule{Name: "alert"}).Validate(); err == nil {
+		t.Errorf("exptected empty expr error")
+	}
+	if err := (&Rule{Name: "alert", Expr: "test{"}).Validate(); err == nil {
+		t.Errorf("exptected invalid expr error")
+	}
+	if err := (&Rule{Name: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("exptected valid rule got %s", err)
+	}
+}
+
+func TestRule_AlertToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *Rule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant extra labels",
+					"job":           "foo",
+					"instance":      "bar",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant labels override",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "for",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+					alertNameLabel:  "for pending",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for pending",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tss := tc.rule.AlertToTimeSeries(tc.alert, timestamp)
+			if len(tc.expTS) != len(tss) {
+				t.Fatalf("expected number of timeseries %d; got %d", len(tc.expTS), len(tss))
+			}
+			for i := range tc.expTS {
+				expTS, gotTS := tc.expTS[i], tss[i]
+				if len(expTS.Samples) != len(gotTS.Samples) {
+					t.Fatalf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+				}
+				for i, exp := range expTS.Samples {
+					got := gotTS.Samples[i]
+					if got.Value != exp.Value {
+						t.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+					}
+					if got.Timestamp != exp.Timestamp {
+						t.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+					}
+				}
+				if len(expTS.Labels) != len(gotTS.Labels) {
+					t.Fatalf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+				}
+				for i, exp := range expTS.Labels {
+					got := gotTS.Labels[i]
+					if got.Name != exp.Name {
+						t.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+					}
+					if got.Value != exp.Value {
+						t.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+					}
+				}
+			}
+		})
+	}
+}
+
+func newTestRule(name string, waitFor time.Duration) *Rule {
+	return &Rule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
+}
+
+func TestRule_Exec(t *testing.T) {
+	testCases := []struct {
+		rule      *Rule
+		steps     [][]datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRule("empty", 0),
+			[][]datasource.Metric{},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestRule("empty labels", 0),
+			[][]datasource.Metric{
+				{datasource.Metric{}},
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "name", "foo"),
+					metricWithLabels(t, "name", "foo1"),
+					metricWithLabels(t, "name", "foo2"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo1")},
+				{metricWithLabels(t, "name", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive, delete first one
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("duplicate", 0),
+			[][]datasource.Metric{
+				{
+					// metrics with the same labelset should result in one alert
+					metricWithLabels(t, "name", "foo", "type", "bar"),
+					metricWithLabels(t, "type", "bar", "name", "foo"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestRule("for-fired", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("for-pending=>inactive", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive=>pending", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive=>pending=>firing", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.group = fakeGroup
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(step...)
+				if err := tc.rule.Exec(context.TODO(), fq); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(time.Millisecond)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	m := datasource.Metric{}
+	for i := 0; i < len(labels); i += 2 {
+		m.Labels = append(m.Labels, datasource.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return m
+}
+
+type fakeQuerier struct {
+	metrics []datasource.Metric
+}
+
+func (fq *fakeQuerier) reset() {
+	fq.metrics = fq.metrics[:0]
+}
+
+func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
+	fq.metrics = append(fq.metrics, metrics...)
+}
+
+func (fq fakeQuerier) Query(ctx context.Context, query string) ([]datasource.Metric, error) {
+	return fq.metrics, nil
+}
+
+func TestRule_Restore(t *testing.T) {
+	testCases := []struct {
+		rule      *Rule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRuleWithLabels("no extra labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("metric labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("rule labels", "source", "vm"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+					// following pair supposed to be dropped
+					"source", "vm",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("multiple alerts"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-1",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-2",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-3",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(2 * time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(3 * time.Hour)},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.group = fakeGroup
+			fq.add(tc.metrics...)
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+				if got.Start != exp.Start {
+					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				}
+			}
+		})
+	}
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *Rule {
+	r := newTestRule(name, 0)
+	r.Labels = make(map[string]string)
+	for i := 0; i < len(labels); i += 2 {
+		r.Labels[labels[i]] = labels[i+1]
+	}
+	return r
+}
+
+func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
+	t.Helper()
+	m := metricWithLabels(t, labels...)
+	m.Value = value
+	return m
+}

app/vmalert/testdata/dir/rules0-bad.rules

@@ -0,0 +1,19 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidAnnotations
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }"
+          description: "{{$labels}}"
+      - alert: UnkownAnnotationsFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ value|query }}"
+          description: "{{$labels}}"

app/vmalert/testdata/dir/rules0-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+

app/vmalert/testdata/dir/rules1-bad.rules

@@ -0,0 +1,22 @@
+groups:
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/testdata/dir/rules1-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+
+

app/vmalert/testdata/dir/rules2-bad.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: group
+    rules:
+      - alert: UnkownLabelFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          summary: "{{ value|query }}"
+        annotations:
+          description: "{{$labels}}"

app/vmalert/testdata/rules0-bad.rules

@@ -0,0 +1,28 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidExpr
+        for: 5m
+        expr: vm_rows{ > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: EmptyExpr
+        for: 5m
+        expr: ""
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: ""
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: foo
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/testdata/rules0-good.rules

@@ -0,0 +1,23 @@
+groups:
+  - name: groupGorSingleAlert
+    rules:
+      - alert: VMRows
+        for: 10s
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+  - name: TestGroup
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)

app/vmalert/web.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+// APIAlert has info for an alert.
+type APIAlert struct {
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	Group       string            `json:"group"`
+	Expression  string            `json:"expression"`
+	State       string            `json:"state"`
+	Value       string            `json:"value"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	ActiveAt    time.Time         `json:"activeAt"`
+}
+
+type requestHandler struct {
+	groups []Group
+}
+
+var pathList = [][]string{
+	{"/api/v1/alerts", "list all active alerts"},
+	{"/api/v1/groupName/alertID/status", "get alert status by ID"},
+	// /metrics is served by httpserver by default
+	{"/metrics", "list of application metrics"},
+}
+
+func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	resph := responseHandler{w}
+	switch r.URL.Path {
+	case "/":
+		for _, path := range pathList {
+			p, doc := path[0], path[1]
+			fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
+		}
+		return true
+	case "/api/v1/alerts":
+		resph.handle(rh.list())
+		return true
+	default:
+		// /api/v1/<groupName>/<alertID>/status
+		if strings.HasSuffix(r.URL.Path, "/status") {
+			resph.handle(rh.alert(r.URL.Path))
+			return true
+		}
+		return false
+	}
+}
+
+type listAlertsResponse struct {
+	Data struct {
+		Alerts []*APIAlert `json:"alerts"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+func (rh *requestHandler) list() ([]byte, error) {
+	lr := listAlertsResponse{Status: "success"}
+	for _, g := range rh.groups {
+		for _, r := range g.Rules {
+			lr.Data.Alerts = append(lr.Data.Alerts, r.AlertsAPI()...)
+		}
+	}
+
+	// sort list of alerts for deterministic output
+	sort.Slice(lr.Data.Alerts, func(i, j int) bool {
+		return lr.Data.Alerts[i].ID < lr.Data.Alerts[j].ID
+	})
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of active alerts: %s`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
+func (rh *requestHandler) alert(path string) ([]byte, error) {
+	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	if len(parts) != 3 {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+	group := strings.TrimRight(parts[0], "/")
+	idStr := strings.TrimRight(parts[1], "/")
+	id, err := strconv.ParseUint(idStr, 10, 0)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`cannot parse int from %q`, idStr),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+	for _, g := range rh.groups {
+		if g.Name != group {
+			continue
+		}
+		for _, rule := range g.Rules {
+			if apiAlert := rule.AlertAPI(id); apiAlert != nil {
+				return json.Marshal(apiAlert)
+			}
+		}
+	}
+	return nil, &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf(`cannot find alert %s in %q`, idStr, group),
+		StatusCode: http.StatusNotFound,
+	}
+}
+
+// responseHandler wrapper on http.ResponseWriter with sugar
+type responseHandler struct{ http.ResponseWriter }
+
+func (w responseHandler) handle(b []byte, err error) {
+	if err != nil {
+		httpserver.Errorf(w, "%s", err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(b)
+}

app/vmalert/web_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestHandler(t *testing.T) {
+	rule := &Rule{
+		Name: "alert",
+		alerts: map[uint64]*notifier.Alert{
+			0: {},
+		},
+	}
+	rh := &requestHandler{
+		groups: []Group{{
+			Name:  "group",
+			Rules: []*Rule{rule},
+		}},
+	}
+	getResp := func(url string, to interface{}, code int) {
+		t.Helper()
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Errorf("unexpected err %s", err)
+		}
+		if code != resp.StatusCode {
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				t.Errorf("err closing body %s", err)
+			}
+		}()
+		if to != nil {
+			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
+				t.Errorf("unexpected err %s", err)
+			}
+		}
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rh.handler(w, r) }))
+	defer ts.Close()
+	t.Run("/api/v1/alerts", func(t *testing.T) {
+		lr := listAlertsResponse{}
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		if length := len(lr.Data.Alerts); length != 1 {
+			t.Errorf("expected 1 alert got %d", length)
+		}
+	})
+	t.Run("/api/v1/group/0/status", func(t *testing.T) {
+		alert := &APIAlert{}
+		getResp(ts.URL+"/api/v1/group/0/status", alert, 200)
+		expAlert := rule.newAlertAPI(*rule.alerts[0])
+		if !reflect.DeepEqual(alert, expAlert) {
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
+		}
+	})
+	t.Run("/api/v1/group/1/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/group/1/status", nil, 404)
+	})
+	t.Run("/api/v1/unknown-group/0/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/unknown-group/0/status", nil, 404)
+	})
+	t.Run("/", func(t *testing.T) {
+		getResp(ts.URL, nil, 200)
+	})
+}

app/vmauth/Makefile

@@ -0,0 +1,74 @@
+# All these commands must run from repository root.
+
+vmauth:
+	APP_NAME=vmauth $(MAKE) app-local
+
+vmauth-race:
+	APP_NAME=vmauth RACE=-race $(MAKE) app-local
+
+vmauth-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker
+
+vmauth-pure-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-pure
+
+vmauth-amd64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-amd64
+
+vmauth-arm-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-arm
+
+vmauth-arm64-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-arm64
+
+vmauth-ppc64le-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-ppc64le
+
+vmauth-386-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-386
+
+package-vmauth:
+	APP_NAME=vmauth $(MAKE) package-via-docker
+
+package-vmauth-pure:
+	APP_NAME=vmauth $(MAKE) package-via-docker-pure
+
+package-vmauth-amd64:
+	APP_NAME=vmauth $(MAKE) package-via-docker-amd64
+
+package-vmauth-arm:
+	APP_NAME=vmauth $(MAKE) package-via-docker-arm
+
+package-vmauth-arm64:
+	APP_NAME=vmauth $(MAKE) package-via-docker-arm64
+
+package-vmauth-ppc64le:
+	APP_NAME=vmauth $(MAKE) package-via-docker-ppc64le
+
+package-vmauth-386:
+	APP_NAME=vmauth $(MAKE) package-via-docker-386
+
+publish-vmauth:
+	APP_NAME=vmauth $(MAKE) publish-via-docker
+
+run-vmauth:
+	APP_NAME=vmauth \
+	$(MAKE) run-via-docker
+
+vmauth-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-amd64 ./app/vmauth
+
+vmauth-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm ./app/vmauth
+
+vmauth-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-arm64 ./app/vmauth
+
+vmauth-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-ppc64le ./app/vmauth
+
+vmauth-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmauth-386 ./app/vmauth
+
+vmauth-pure:
+	APP_NAME=vmauth $(MAKE) app-local-pure

app/vmauth/README.md

@@ -0,0 +1,125 @@
+## vmauth
+
+`vmauth` is a simple auth proxy and router. It reads username and password from [Basic Auth headers](https://en.wikipedia.org/wiki/Basic_access_authentication)
+and matches them against configs pointed by `-auth.config` command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
+
+
+### Quick start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flag to `vmauth` binary in order to start authorizing and routing requests:
+
+```
+/path/to/vmauth -auth.config=/path/to/auth/config.yml
+```
+
+After that `vmauth` starts accepting HTTP requests on port `8427` and routing them according to the provided [-auth.config](#auth-config).
+The port can be modified via `-httpListenAddr` command-line flag.
+
+The auth config can be reloaded by passing `SIGHUP` signal to `vmauth`.
+
+Pass `-help` to `vmauth` in order to see all the supported command-line flags with their descriptions.
+
+
+### Auth config
+
+Auth config is represented in the following simple `yml` format:
+
+```yml
+
+# Arbitrary number of usernames may be put here.
+# Usernames must be unique.
+
+users:
+
+  # The user for querying local single-node VictoriaMetrics
+- username: "local-single-node"
+  password: "***"
+  url_prefix: "http://localhost:8428"
+
+  # The user for querying account 123 in VictoriaMetrics cluster
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+- username: "cluster-select-account-123"
+  password: "***"
+  url_prefix: "http://vmselect:8481/select/123/prometheus"
+
+  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
+  # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format
+- username: "cluster-insert-account-42"
+  password: "***"
+  url_prefix: "http://localhost:8480/insert/42/prometheus"
+```
+
+
+### Security
+
+Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
+
+```
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+```
+
+Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.
+
+
+### Monitoring
+
+`vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
+either via [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md) or via Prometheus, so the exported metrics could be analyzed later.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmauth` from the root folder of the repository.
+   It builds `vmauth` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmauth-prod` from the root folder of the repository.
+   It builds `vmauth-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmauth
+```
+
+
+### Profiling
+
+`vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmauth-host>:8428/debug/pprof/heap > mem.pprof
+```
+
+* CPU profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmauth-host>:8428/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

app/vmauth/auth_config.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md "+
+		"for details on the format of this auth config")
+)
+
+// AuthConfig represents auth config.
+type AuthConfig struct {
+	Users []UserInfo `yaml:"users"`
+}
+
+// UserInfo is user information read from authConfigPath
+type UserInfo struct {
+	Username  string `yaml:"username"`
+	Password  string `yaml:"password"`
+	URLPrefix string `yaml:"url_prefix"`
+
+	requests *metrics.Counter
+}
+
+func initAuthConfig() {
+	if len(*authConfigPath) == 0 {
+		logger.Panicf("FATAL: missing required `-auth.config` command-line flag")
+	}
+	m, err := readAuthConfig(*authConfigPath)
+	if err != nil {
+		logger.Panicf("FATAL: cannot load auth config from `-auth.config=%s`: %s", *authConfigPath, err)
+	}
+	authConfig.Store(m)
+	stopCh = make(chan struct{})
+	authConfigWG.Add(1)
+	go func() {
+		defer authConfigWG.Done()
+		authConfigReloader()
+	}()
+}
+
+func stopAuthConfig() {
+	close(stopCh)
+	authConfigWG.Wait()
+}
+
+func authConfigReloader() {
+	sighupCh := procutil.NewSighupChan()
+	for {
+		select {
+		case <-stopCh:
+			return
+		case <-sighupCh:
+			m, err := readAuthConfig(*authConfigPath)
+			if err != nil {
+				logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
+				continue
+			}
+			authConfig.Store(m)
+		}
+	}
+}
+
+var authConfig atomic.Value
+var authConfigWG sync.WaitGroup
+var stopCh chan struct{}
+
+func readAuthConfig(path string) (map[string]*UserInfo, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read %q: %s", path, err)
+	}
+	m, err := parseAuthConfig(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse %q: %s", path, err)
+	}
+	logger.Infof("Loaded information about %d users from %q", len(m), path)
+	return m, nil
+}
+
+func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
+	var ac AuthConfig
+	if err := yaml.UnmarshalStrict(data, &ac); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %s", err)
+	}
+	uis := ac.Users
+	if len(uis) == 0 {
+		return nil, fmt.Errorf("`users` section cannot be empty in AuthConfig")
+	}
+	m := make(map[string]*UserInfo, len(uis))
+	for i := range uis {
+		ui := &uis[i]
+		if m[ui.Username] != nil {
+			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
+		}
+		urlPrefix := ui.URLPrefix
+		// Remove trailing '/' from urlPrefix
+		for strings.HasSuffix(urlPrefix, "/") {
+			urlPrefix = urlPrefix[:len(urlPrefix)-1]
+		}
+		// Validate urlPrefix
+		target, err := url.Parse(urlPrefix)
+		if err != nil {
+			return nil, fmt.Errorf("invalid `url_prefix: %q`: %s", urlPrefix, err)
+		}
+		if target.Scheme != "http" && target.Scheme != "https" {
+			return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, target.Scheme)
+		}
+
+		ui.URLPrefix = urlPrefix
+		ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+		m[ui.Username] = ui
+	}
+	return m, nil
+}

app/vmauth/auth_config_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseAuthConfigFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		_, err := parseAuthConfig([]byte(s))
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// Empty config
+	f(``)
+
+	// Invalid entry
+	f(`foobar`)
+	f(`foobar: baz`)
+
+	// Empty users
+	f(`users: []`)
+
+	// Missing url_prefix
+	f(`
+users:
+- username: foo
+`)
+
+	// Invalid url_prefix
+	f(`
+users:
+- username: foo
+  url_prefix: bar
+`)
+	f(`
+users:
+- username: foo
+  url_prefix: ftp://bar
+`)
+	f(`
+users:
+- username: foo
+  url_prefix: //bar
+`)
+
+	// Duplicate users
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+- username: bar
+  url_prefix: http://xxx.yyy
+- username: foo
+  url_prefix: https://sss.sss
+`)
+}
+
+func TestParseAuthConfigSuccess(t *testing.T) {
+	f := func(s string, expectedAuthConfig map[string]*UserInfo) {
+		t.Helper()
+		m, err := parseAuthConfig([]byte(s))
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		removeMetrics(m)
+		if !reflect.DeepEqual(m, expectedAuthConfig) {
+			t.Fatalf("unexpected auth config\ngot\n%v\nwant\n%v", m, expectedAuthConfig)
+		}
+	}
+
+	// Single user
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: http://aaa:343/bbb
+`, map[string]*UserInfo{
+		"foo": {
+			Username:  "foo",
+			Password:  "bar",
+			URLPrefix: "http://aaa:343/bbb",
+		},
+	})
+
+	// Multiple users
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo
+- username: bar
+  url_prefix: https://bar/x///
+`, map[string]*UserInfo{
+		"foo": {
+			Username:  "foo",
+			URLPrefix: "http://foo",
+		},
+		"bar": {
+			Username:  "bar",
+			URLPrefix: "https://bar/x",
+		},
+	})
+}
+
+func removeMetrics(m map[string]*UserInfo) {
+	for _, info := range m {
+		info.requests = nil
+	}
+}

app/vmauth/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8427
+
+ENTRYPOINT ["/vmauth-prod"]
+ARG src_binary
+COPY $src_binary ./vmauth-prod

app/vmauth/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections")
+)
+
+func main() {
+	flag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting vmauth at %q...", *httpListenAddr)
+	startTime := time.Now()
+	initAuthConfig()
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	stopAuthConfig()
+	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	username, password, ok := r.BasicAuth()
+	if !ok {
+		httpserver.Errorf(w, "Missing `Authorization: Basic *` header")
+		return true
+	}
+	ac := authConfig.Load().(map[string]*UserInfo)
+	info := ac[username]
+	if info == nil || info.Password != password {
+		httpserver.Errorf(w, "Cannot find the provided username %q or password in config", username)
+		return true
+	}
+	info.requests.Inc()
+
+	targetURL := info.URLPrefix + r.RequestURI
+	if _, err := url.Parse(targetURL); err != nil {
+		httpserver.Errorf(w, "Invalid targetURL=%q: %s", targetURL, err)
+		return true
+	}
+	r.Header.Set("vm-target-url", targetURL)
+	reverseProxy.ServeHTTP(w, r)
+	return true
+}
+
+var reverseProxy = &httputil.ReverseProxy{
+	Director: func(r *http.Request) {
+		targetURL := r.Header.Get("vm-target-url")
+		target, err := url.Parse(targetURL)
+		if err != nil {
+			logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
+		}
+		r.URL = target
+	},
+	FlushInterval: time.Second,
+	ErrorLog:      logger.StdErrorLogger(),
+}

app/vmbackup/Makefile

@@ -0,0 +1,70 @@
+# All these commands must run from repository root.
+
+vmbackup:
+	APP_NAME=vmbackup $(MAKE) app-local
+
+vmbackup-race:
+	APP_NAME=vmbackup RACE=-race $(MAKE) app-local
+
+vmbackup-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker
+
+vmbackup-pure-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-pure
+
+vmbackup-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-amd64
+
+vmbackup-arm-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm
+
+vmbackup-arm64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm64
+
+vmbackup-ppc64le-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-ppc64le
+
+vmbackup-386-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-386
+
+package-vmbackup:
+	APP_NAME=vmbackup $(MAKE) package-via-docker
+
+package-vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-pure
+
+package-vmbackup-amd64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-amd64
+
+package-vmbackup-arm:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm
+
+package-vmbackup-arm64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm64
+
+package-vmbackup-ppc64le:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le
+
+package-vmbackup-386:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-386
+
+publish-vmbackup:
+	APP_NAME=vmbackup $(MAKE) publish-via-docker
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure
+
+vmbackup-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-amd64 ./app/vmbackup
+
+vmbackup-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+
+vmbackup-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+
+vmbackup-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup
+
+vmbackup-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup

app/vmbackup/README.md

@@ -0,0 +1,202 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshot.createURL string
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+  -snapshot.deleteURL string
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmbackup
+```

app/vmbackup/deployment/Dockerfile

@@ -0,0 +1,6 @@
+ARG base_image
+FROM $base_image
+
+ENTRYPOINT ["/vmbackup-prod"]
+ARG src_binary
+COPY $src_binary ./vmbackup-prod

app/vmbackup/main.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup."+
+		"Example: http://victoriametrics:8428/snaphsot/create")
+	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted."+
+		"Example: http://victoriametrics:8428/snaphsot/delete")
+	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
+	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
+	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+)
+
+func main() {
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+
+	if len(*snapshotCreateURL) > 0 {
+		logger.Infof("%s", "Snapshots enabled")
+		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
+		if len(*snapshotDeleteURL) <= 0 {
+			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
+			if err != nil {
+				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
+			}
+		}
+		logger.Infof("Snapshot delete url %s", *snapshotDeleteURL)
+
+		name, err := snapshot.Create(*snapshotCreateURL)
+		if err != nil {
+			logger.Fatalf("%s", err)
+		}
+		err = flag.Set("snapshotName", name)
+		if err != nil {
+			logger.Fatalf("Failed to set snapshotName flag: %v", err)
+		}
+
+		defer func() {
+			err := snapshot.Delete(*snapshotDeleteURL, name)
+			if err != nil {
+				logger.Fatalf("%s", err)
+			}
+		}()
+	}
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	originFS, err := newOriginFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Backup{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+		Origin:      originFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+or local filesystem. Backed up data can be restored with vmrestore.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newSrcFS() (*fslocal.FS, error) {
+	if len(*snapshotName) == 0 {
+		return nil, fmt.Errorf("`-snapshotName` or `-snapshot.createURL` must be provided")
+	}
+	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
+
+	// Verify the snapshot exists.
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
+	}
+
+	fs := &fslocal.FS{
+		Dir:               snapshotPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newDstFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*dst)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+	}
+	return fs, nil
+}
+
+func newOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, nil
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+	}
+	return fs, nil
+}

app/vmbackup/snapshot/snapshot.go

@@ -0,0 +1,91 @@
+package snapshot
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+type snapshot struct {
+	Status   string `json:"status"`
+	Snapshot string `json:"snapshot"`
+	Msg      string `json:"msg"`
+}
+
+// Create creates a snapshot and the provided api endpoint and returns
+// the snapshot name
+func Create(createSnapshotURL string) (string, error) {
+	logger.Infof("%s", "Creating snapshot")
+	u, err := url.Parse(createSnapshotURL)
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return "", err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s created", snap.Snapshot)
+		return snap.Snapshot, nil
+	} else if snap.Status == "error" {
+		return "", errors.New(snap.Msg)
+	} else {
+		return "", fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}
+
+// Delete deletes a snapshot and the provided api endpoint returns any failure
+func Delete(deleteSnapshotURL string, snapshotName string) error {
+	logger.Infof("Deleting snapshot %s", snapshotName)
+	formData := url.Values{
+		"snapshot": {snapshotName},
+	}
+
+	u, err := url.Parse(deleteSnapshotURL)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.PostForm(u.String(), formData)
+	if err != nil {
+		return err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s deleted", snapshotName)
+		return nil
+	} else if snap.Status == "error" {
+		return errors.New(snap.Msg)
+	} else {
+		return fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}

app/vmbackup/snapshot/snapshot_test.go

@@ -0,0 +1,106 @@
+package snapshot
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestCreateSnapshot(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"ok","snapshot":"mysnapshot"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err != nil {
+		t.Fatalf("Failed taking snapshot: %v", err)
+	}
+
+	if snapshotName != "mysnapshot" {
+		t.Fatalf("Snapshot name is not correct, got %v", snapshotName)
+	}
+}
+
+func TestCreateSnapshotFailed(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"error","msg":"I am unwell"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err == nil {
+		t.Fatalf("Snapshot did not fail, got snapshot: %v", snapshotName)
+	}
+}
+
+func TestDeleteSnapshot(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"ok"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err != nil {
+		t.Fatalf("Failed to delete snapshot: %v", err)
+	}
+}
+
+func TestDeleteSnapshotFailed(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"error", "msg":"failed to delete"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err == nil {
+		t.Fatalf("Snapshot should have failed, got: %v", err)
+	}
+}

app/vminsert/Makefile

@@ -0,0 +1,34 @@
+# All these commands must run from repository root.
+
+run-vminsert:
+	APP_NAME=vminsert ARGS='-storageNode=localhost:8400' $(MAKE) run-via-docker
+
+vminsert:
+	APP_NAME=vminsert $(MAKE) app-local
+
+vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) app-local
+
+vminsert-prod:
+	APP_NAME=vminsert $(MAKE) app-via-docker
+
+vminsert-pure-prod:
+	APP_NAME=vminsert $(MAKE) app-via-docker-pure
+
+vminsert-prod-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) app-via-docker
+
+vminsert-pure:
+	APP_NAME=vminsert $(MAKE) app-local-pure
+
+package-vminsert:
+	APP_NAME=vminsert $(MAKE) package-via-docker
+
+package-vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) package-via-docker
+
+publish-vminsert:
+	APP_NAME=vminsert $(MAKE) publish-via-docker
+
+publish-vminsert-race:
+	APP_NAME=vminsert RACE=-race $(MAKE) publish-via-docker

app/vminsert/README.md

@@ -0,0 +1 @@
+`vminsert` routes the ingested data to `vmstorage` nodes.

app/vminsert/csvimport/request_handler.go

@@ -0,0 +1,48 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes /api/v1/import/csv requests.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(at, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+	rowsInserted.Get(at).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8480
+
+ENTRYPOINT ["/vminsert-prod"]
+ARG src_binary
+COPY $src_binary ./vminsert-prod

app/vminsert/graphite/request_handler.go

@@ -0,0 +1,63 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	atCopy := *at
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if atCopy.AccountID == 0 {
+				// Multi-tenancy support via custom tags.
+				// Do not allow overriding AccountID and ProjectID from atCopy for security reasons.
+				if tag.Key == "VictoriaMetrics_AccountID" {
+					atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+				if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" {
+					atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+			}
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(&atCopy, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+	// Assume that all the rows for a single connection belong to the same (AccountID, ProjectID).
+	rowsInserted.Get(&atCopy).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/influx/request_handler.go

@@ -0,0 +1,141 @@
+package influx
+
+import (
+	"flag"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows)
+		})
+	})
+}
+
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	ic := &ctx.Common
+	ic.Reset() // This line is required for initializing ic internals.
+	rowsTotal := 0
+	for i := range rows {
+		r := &rows[i]
+		ic.Labels = ic.Labels[:0]
+		hasDBLabel := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
+			ic.AddLabel(tag.Key, tag.Value)
+		}
+		if len(db) > 0 && !hasDBLabel {
+			ic.AddLabel("db", db)
+		}
+		ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], at.AccountID, at.ProjectID, ic.Labels)
+		metricNameBufLen := len(ic.MetricNameBuf)
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		metricGroupPrefixLen := len(ctx.metricGroupBuf)
+		ic.AddLabel("", "placeholder")
+		placeholderLabel := &ic.Labels[len(ic.Labels)-1]
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			if !skipFieldKey {
+				ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
+			ic.Labels = ic.Labels[:len(ic.Labels)-1]
+			ic.AddLabel("", metricGroup)
+			ic.MetricNameBuf = storage.MarshalMetricLabelRaw(ic.MetricNameBuf[:metricNameBufLen], placeholderLabel)
+			storageNodeIdx := ic.GetStorageNodeIdx(at, ic.Labels)
+			if err := ic.WriteDataPointExt(at, storageNodeIdx, ic.MetricNameBuf, r.Timestamp, f.Value); err != nil {
+				return err
+			}
+		}
+		rowsTotal += len(r.Fields)
+	}
+	rowsInserted.Get(at).Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ic.FlushBufs()
+}
+
+type pushCtx struct {
+	Common         netstorage.InsertCtx
+	metricGroupBuf []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset()
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/main.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	httpListenAddr         = flag.String("httpListenAddr", ":8480", "Address to listen for http connections")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
+	storageNodes           = flagutil.NewArray("storageNode", "Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400")
+)
+
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	logger.Infof("initializing netstorage for storageNodes %s...", *storageNodes)
+	startTime := time.Now()
+	if len(*storageNodes) == 0 {
+		logger.Fatalf("missing -storageNode arg")
+	}
+	netstorage.InitStorageNodes(*storageNodes)
+	logger.Infof("successfully initialized netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return influx.InsertHandlerForReader(&at, r)
+		})
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return graphite.InsertHandler(&at, r)
+		})
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, func(r io.Reader) error {
+			var at auth.Token // TODO: properly initialize auth token
+			return opentsdb.InsertHandler(&at, r)
+		}, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+
+	go func() {
+		httpserver.Serve(*httpListenAddr, requestHandler)
+	}()
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+
+	logger.Infof("gracefully shutting down http service at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http service: %s", err)
+	}
+	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+
+	logger.Infof("shutting down neststorage...")
+	startTime = time.Now()
+	netstorage.Stop()
+	logger.Infof("successfully stopped netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the vminsert has been stopped")
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	p, err := httpserver.ParsePath(r.URL.Path)
+	if err != nil {
+		httpserver.Errorf(w, "cannot parse path %q: %s", r.URL.Path, err)
+		return true
+	}
+	if p.Prefix != "insert" {
+		// This is not our link.
+		return false
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, "auth error: %s", err)
+		return true
+	}
+
+	switch p.Suffix {
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(at, r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(at, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(at, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/write", "influx/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
+		influxQueryRequests.Inc()
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
+		return true
+	default:
+		// This is not our link
+		return false
+	}
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
+
+	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/api/v1/import/csv", protocol="csvimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/influx/", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+)

app/vminsert/netstorage/insert_ctx.go

@@ -0,0 +1,194 @@
+package netstorage
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	xxhash "github.com/cespare/xxhash/v2"
+	jump "github.com/lithammer/go-jump-consistent-hash"
+)
+
+// InsertCtx is a generic context for inserting data.
+//
+// InsertCtx.Reset must be called before the first usage.
+type InsertCtx struct {
+	Labels        []prompb.Label
+	MetricNameBuf []byte
+
+	bufRowss  []bufRows
+	labelsBuf []byte
+
+	resultCh chan error
+}
+
+type bufRows struct {
+	buf  []byte
+	rows int
+}
+
+func (br *bufRows) pushTo(sn *storageNode) error {
+	bufLen := len(br.buf)
+	err := sn.push(br.buf, br.rows)
+	br.buf = br.buf[:0]
+	br.rows = 0
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot send %d bytes to storageNode %q: %s", bufLen, sn.dialer.Addr(), err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
+// Reset resets ctx.
+func (ctx *InsertCtx) Reset() {
+	for _, label := range ctx.Labels {
+		label.Name = nil
+		label.Value = nil
+	}
+	ctx.Labels = ctx.Labels[:0]
+	ctx.MetricNameBuf = ctx.MetricNameBuf[:0]
+
+	if ctx.bufRowss == nil {
+		ctx.bufRowss = make([]bufRows, len(storageNodes))
+	}
+	for i := range ctx.bufRowss {
+		br := &ctx.bufRowss[i]
+		br.buf = br.buf[:0]
+		br.rows = 0
+	}
+	ctx.labelsBuf = ctx.labelsBuf[:0]
+	if ctx.resultCh == nil {
+		ctx.resultCh = make(chan error, len(storageNodes))
+	} else if len(ctx.resultCh) > 0 {
+		logger.Panicf("BUG: ctx.resultCh must be empty on Reset; got %d items", len(ctx.resultCh))
+	}
+}
+
+// AddLabelBytes adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = name
+	label.Value = value
+
+	ctx.Labels = labels
+}
+
+// AddLabel adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabel(name, value string) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = bytesutil.ToUnsafeBytes(name)
+	label.Value = bytesutil.ToUnsafeBytes(value)
+
+	ctx.Labels = labels
+}
+
+// WriteDataPoint writes (timestamp, value) data point with the given at and labels to ctx buffer.
+func (ctx *InsertCtx) WriteDataPoint(at *auth.Token, labels []prompb.Label, timestamp int64, value float64) error {
+	ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, labels)
+	storageNodeIdx := ctx.GetStorageNodeIdx(at, labels)
+	return ctx.WriteDataPointExt(at, storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
+}
+
+// WriteDataPointExt writes the given metricNameRaw with (timestmap, value) to ctx buffer with the given storageNodeIdx.
+func (ctx *InsertCtx) WriteDataPointExt(at *auth.Token, storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	br := &ctx.bufRowss[storageNodeIdx]
+	sn := storageNodes[storageNodeIdx]
+	bufNew := storage.MarshalMetricRow(br.buf, metricNameRaw, timestamp, value)
+	if len(bufNew) >= consts.MaxInsertPacketSize {
+		// Send buf to storageNode, since it is too big.
+		if err := br.pushTo(sn); err != nil {
+			return err
+		}
+		br.buf = storage.MarshalMetricRow(bufNew[:0], metricNameRaw, timestamp, value)
+	} else {
+		br.buf = bufNew
+	}
+	br.rows++
+	return nil
+}
+
+// FlushBufs flushes ctx bufs to remote storage nodes.
+func (ctx *InsertCtx) FlushBufs() error {
+	// Send per-storageNode bufs in parallel.
+	resultCh := ctx.resultCh
+	resultChLen := 0
+	for i := range ctx.bufRowss {
+		br := &ctx.bufRowss[i]
+		if len(br.buf) == 0 {
+			continue
+		}
+		resultChLen++
+		go func(br *bufRows, sn *storageNode) {
+			resultCh <- br.pushTo(sn)
+		}(br, storageNodes[i])
+	}
+	var lastErr error
+	for i := 0; i < resultChLen; i++ {
+		err := <-resultCh
+		if err != nil {
+			lastErr = err
+		}
+	}
+	return lastErr
+}
+
+// GetStorageNodeIdx returns storage node index for the given at and labels.
+//
+// The returned index must be passed to WriteDataPoint.
+func (ctx *InsertCtx) GetStorageNodeIdx(at *auth.Token, labels []prompb.Label) int {
+	if len(storageNodes) == 1 {
+		// Fast path - only a single storage node.
+		return 0
+	}
+
+	buf := ctx.labelsBuf[:0]
+	buf = encoding.MarshalUint32(buf, at.AccountID)
+	buf = encoding.MarshalUint32(buf, at.ProjectID)
+	for i := range labels {
+		label := &labels[i]
+		buf = marshalBytesFast(buf, label.Name)
+		buf = marshalBytesFast(buf, label.Value)
+	}
+	h := xxhash.Sum64(buf)
+	ctx.labelsBuf = buf
+
+	idx := int(jump.Hash(h, int32(len(storageNodes))))
+	return idx
+}
+
+func marshalBytesFast(dst []byte, s []byte) []byte {
+	dst = encoding.MarshalUint16(dst, uint16(len(s)))
+	dst = append(dst, s...)
+	return dst
+}

app/vminsert/netstorage/insert_ctx_pool.go

@@ -0,0 +1,36 @@
+package netstorage
+
+import (
+	"runtime"
+	"sync"
+)
+
+// GetInsertCtx returns InsertCtx from the pool.
+//
+// Call PutInsertCtx for returning it to the pool.
+func GetInsertCtx() *InsertCtx {
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
+	}
+}
+
+// PutInsertCtx returns ctx to the pool.
+//
+// ctx cannot be used after the call.
+func PutInsertCtx(ctx *InsertCtx) {
+	ctx.Reset()
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
+}
+
+var insertCtxPool sync.Pool
+var insertCtxPoolCh = make(chan *InsertCtx, runtime.GOMAXPROCS(-1))

app/vminsert/netstorage/netstorage.go

@@ -0,0 +1,489 @@
+package netstorage
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/handshake"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var disableRPCCompression = flag.Bool(`rpc.disableCompression`, false, "Disable compression of RPC traffic. This reduces CPU usage at the cost of higher network bandwidth usage")
+
+// push pushes buf to sn.
+//
+// It falls back to sending data to another vmstorage node if sn is currently
+// unavailable.
+//
+// rows is the number of rows in the buf.
+func (sn *storageNode) push(buf []byte, rows int) error {
+	if len(buf) > consts.MaxInsertPacketSize {
+		logger.Panicf("BUG: len(buf)=%d cannot exceed %d", len(buf), consts.MaxInsertPacketSize)
+	}
+	sn.rowsPushed.Add(rows)
+
+	sn.mu.Lock()
+	defer sn.mu.Unlock()
+
+	if sn.broken {
+		// The vmstorage node is broken. Re-route buf to healthy vmstorage nodes.
+		if !addToReroutedBuf(buf, rows) {
+			rowsLostTotal.Add(rows)
+			return fmt.Errorf("%d rows dropped because of reroutedBuf overflows %d bytes", rows, reroutedBufMaxSize)
+		}
+		sn.rowsReroutedFromHere.Add(rows)
+		return nil
+	}
+
+	if len(sn.buf)+len(buf) <= consts.MaxInsertPacketSize {
+		// Fast path: the buf contents fits sn.buf.
+		sn.buf = append(sn.buf, buf...)
+		sn.rows += rows
+		return nil
+	}
+
+	// Slow path: the buf contents doesn't fit sn.buf.
+	// Flush sn.buf to vmstorage and then add buf to sn.buf.
+	if err := sn.flushBufLocked(); err != nil {
+		// Failed to flush or re-route sn.buf to vmstorage nodes.
+		// The sn.buf is already dropped by flushBufLocked.
+		// Drop buf too, since there is litte sense in trying to rescue it.
+		rowsLostTotal.Add(rows)
+		return err
+	}
+
+	// Successful flush.
+	sn.buf = append(sn.buf, buf...)
+	sn.rows += rows
+	return nil
+}
+
+func (sn *storageNode) sendReroutedRow(buf []byte) error {
+	sn.mu.Lock()
+	defer sn.mu.Unlock()
+
+	if sn.broken {
+		return errBrokenStorageNode
+	}
+	if len(sn.buf)+len(buf) > consts.MaxInsertPacketSize {
+		return fmt.Errorf("cannot put %d bytes into vmstorage buffer, since its size cannot exceed %d bytes", len(sn.buf)+len(buf), consts.MaxInsertPacketSize)
+	}
+	sn.buf = append(sn.buf, buf...)
+	sn.rows++
+	return nil
+}
+
+var errBrokenStorageNode = fmt.Errorf("the vmstorage node is temporarily broken")
+
+func (sn *storageNode) flushBufLocked() error {
+	err := sn.sendBufLocked(sn.buf)
+	if err == nil {
+		// Successful flush. Remove broken flag.
+		sn.broken = false
+		sn.rowsSent.Add(sn.rows)
+		sn.buf = sn.buf[:0]
+		sn.rows = 0
+		return nil
+	}
+
+	// Couldn't flush sn.buf to vmstorage. Mark sn as broken
+	// and try re-routing sn.buf to healthy vmstorage nodes.
+	sn.broken = true
+	if !addToReroutedBuf(sn.buf, sn.rows) {
+		// Preserve sn.buf when it cannot be sent to healthy nodes
+		// in the hope the error will disappear on the next call to flushBufLocked.
+		//
+		// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/294 .
+		return err
+	}
+	sn.buf = sn.buf[:0]
+	sn.rows = 0
+	return err
+}
+
+func (sn *storageNode) sendBufLocked(buf []byte) error {
+	if len(buf) == 0 {
+		return nil
+	}
+	if sn.bc == nil {
+		if err := sn.dial(); err != nil {
+			return fmt.Errorf("cannot dial %q: %s", sn.dialer.Addr(), err)
+		}
+	}
+	if err := sn.sendToConn(sn.bc, buf); err != nil {
+		sn.closeBrokenConn()
+		return err
+	}
+	return nil
+}
+
+func (sn *storageNode) sendToConn(bc *handshake.BufferedConn, buf []byte) error {
+	timeoutSeconds := len(buf) / 3e5
+	if timeoutSeconds < 60 {
+		timeoutSeconds = 60
+	}
+	timeout := time.Duration(timeoutSeconds) * time.Second
+	deadline := time.Now().Add(timeout)
+	if err := bc.SetWriteDeadline(deadline); err != nil {
+		return fmt.Errorf("cannot set write deadline to %s: %s", deadline, err)
+	}
+	// sizeBuf guarantees that the rows batch will be either fully
+	// read or fully discarded on the vmstorage side.
+	// sizeBuf is used for read optimization in vmstorage.
+	sn.sizeBuf = encoding.MarshalUint64(sn.sizeBuf[:0], uint64(len(buf)))
+	if _, err := bc.Write(sn.sizeBuf); err != nil {
+		return fmt.Errorf("cannot write data size %d: %s", len(buf), err)
+	}
+	if _, err := bc.Write(buf); err != nil {
+		return fmt.Errorf("cannot write data with size %d: %s", len(buf), err)
+	}
+	if err := bc.Flush(); err != nil {
+		return fmt.Errorf("cannot flush data with size %d: %s", len(buf), err)
+	}
+
+	// Wait for `ack` from vmstorage.
+	// This guarantees that the message has been fully received by vmstorage.
+	deadline = time.Now().Add(timeout)
+	if err := bc.SetReadDeadline(deadline); err != nil {
+		return fmt.Errorf("cannot set read deadline for reading `ack` to vmstorage: %s", err)
+	}
+	if _, err := io.ReadFull(bc, sn.sizeBuf[:1]); err != nil {
+		return fmt.Errorf("cannot read `ack` from vmstorage: %s", err)
+	}
+	if sn.sizeBuf[0] != 1 {
+		return fmt.Errorf("unexpected `ack` received from vmstorage; got %d; want %d", sn.sizeBuf[0], 1)
+	}
+	return nil
+}
+
+func (sn *storageNode) dial() error {
+	c, err := sn.dialer.Dial()
+	if err != nil {
+		sn.dialErrors.Inc()
+		return err
+	}
+	compressionLevel := 1
+	if *disableRPCCompression {
+		compressionLevel = 0
+	}
+	bc, err := handshake.VMInsertClient(c, compressionLevel)
+	if err != nil {
+		_ = c.Close()
+		sn.handshakeErrors.Inc()
+		return fmt.Errorf("handshake error: %s", err)
+	}
+	sn.bc = bc
+	return nil
+}
+
+func (sn *storageNode) closeBrokenConn() {
+	if sn.bc == nil {
+		return
+	}
+	_ = sn.bc.Close()
+	sn.bc = nil
+	sn.connectionErrors.Inc()
+}
+
+func (sn *storageNode) run(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-stopCh:
+			mustStop = true
+			// Make sure flushBufLocked is called last time before returning
+			// in order to send the remaining bits of data.
+		case <-ticker.C:
+		}
+
+		sn.mu.Lock()
+		if err := sn.flushBufLocked(); err != nil {
+			sn.closeBrokenConn()
+			logger.Errorf("cannot flush data to storageNode %q: %s", sn.dialer.Addr(), err)
+		}
+		sn.mu.Unlock()
+	}
+}
+
+func rerouteWorker(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	var buf []byte
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-stopCh:
+			mustStop = true
+			// Make sure spreadReroutedBufToStorageNodes is called last time before returning
+			// in order to reroute the remaining data to healthy vmstorage nodes.
+		case <-ticker.C:
+		}
+
+		var err error
+		buf, err = spreadReroutedBufToStorageNodes(buf[:0])
+		if err != nil {
+			rerouteErrors.Inc()
+			logger.Errorf("cannot reroute data among healthy vmstorage nodes: %s", err)
+		}
+	}
+}
+
+// storageNode is a client sending data to vmstorage node.
+type storageNode struct {
+	mu sync.Mutex
+
+	// Buffer with data that needs to be written to vmstorage node.
+	buf []byte
+
+	// The number of rows buf contains at the moment.
+	rows int
+
+	// Temporary buffer for encoding marshaled buf size.
+	sizeBuf []byte
+
+	// broken is set to true if the given vmstorage node is temporarily unhealthy.
+	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
+	broken bool
+
+	dialer *netutil.TCPDialer
+
+	bc *handshake.BufferedConn
+
+	// The number of dial errors to vmstorage node.
+	dialErrors *metrics.Counter
+
+	// The number of handshake errors to vmstorage node.
+	handshakeErrors *metrics.Counter
+
+	// The number of connection errors to vmstorage node.
+	connectionErrors *metrics.Counter
+
+	// The number of rows pushed to storageNode with push method.
+	rowsPushed *metrics.Counter
+
+	// The number of rows sent to vmstorage node.
+	rowsSent *metrics.Counter
+
+	// The number of rows rerouted from the given vmstorage node
+	// to healthy nodes when the given node was unhealthy.
+	rowsReroutedFromHere *metrics.Counter
+
+	// The number of rows rerouted to the given vmstorage node
+	// from other nodes when they were unhealthy.
+	rowsReroutedToHere *metrics.Counter
+}
+
+// storageNodes contains a list of vmstorage node clients.
+var storageNodes []*storageNode
+
+var (
+	storageNodesWG  sync.WaitGroup
+	rerouteWorkerWG sync.WaitGroup
+)
+
+var (
+	storageNodesStopCh  = make(chan struct{})
+	rerouteWorkerStopCh = make(chan struct{})
+)
+
+// InitStorageNodes initializes vmstorage nodes' connections to the given addrs.
+func InitStorageNodes(addrs []string) {
+	if len(addrs) == 0 {
+		logger.Panicf("BUG: addrs must be non-empty")
+	}
+	if len(addrs) > 255 {
+		logger.Panicf("BUG: too much addresses: %d; max supported %d addresses", len(addrs), 255)
+	}
+
+	for _, addr := range addrs {
+		sn := &storageNode{
+			dialer: netutil.NewTCPDialer("vminsert", addr),
+
+			dialErrors:           metrics.NewCounter(fmt.Sprintf(`vm_rpc_dial_errors_total{name="vminsert", addr=%q}`, addr)),
+			handshakeErrors:      metrics.NewCounter(fmt.Sprintf(`vm_rpc_handshake_errors_total{name="vminsert", addr=%q}`, addr)),
+			connectionErrors:     metrics.NewCounter(fmt.Sprintf(`vm_rpc_connection_errors_total{name="vminsert", addr=%q}`, addr)),
+			rowsPushed:           metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_pushed_total{name="vminsert", addr=%q}`, addr)),
+			rowsSent:             metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_sent_total{name="vminsert", addr=%q}`, addr)),
+			rowsReroutedFromHere: metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_from_here_total{name="vminsert", addr=%q}`, addr)),
+			rowsReroutedToHere:   metrics.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_to_here_total{name="vminsert", addr=%q}`, addr)),
+		}
+		_ = metrics.NewGauge(fmt.Sprintf(`vm_rpc_rows_pending{name="vminsert", addr=%q}`, addr), func() float64 {
+			sn.mu.Lock()
+			n := sn.rows
+			sn.mu.Unlock()
+			return float64(n)
+		})
+		_ = metrics.NewGauge(fmt.Sprintf(`vm_rpc_buf_pending_bytes{name="vminsert", addr=%q}`, addr), func() float64 {
+			sn.mu.Lock()
+			n := len(sn.buf)
+			sn.mu.Unlock()
+			return float64(n)
+		})
+		storageNodes = append(storageNodes, sn)
+		storageNodesWG.Add(1)
+		go func(addr string) {
+			sn.run(storageNodesStopCh)
+			storageNodesWG.Done()
+		}(addr)
+	}
+
+	reroutedBufMaxSize = memory.Allowed() / 16
+	rerouteWorkerWG.Add(1)
+	go func() {
+		rerouteWorker(rerouteWorkerStopCh)
+		rerouteWorkerWG.Done()
+	}()
+}
+
+// Stop gracefully stops netstorage.
+func Stop() {
+	close(rerouteWorkerStopCh)
+	rerouteWorkerWG.Wait()
+
+	close(storageNodesStopCh)
+	storageNodesWG.Wait()
+}
+
+func addToReroutedBuf(buf []byte, rows int) bool {
+	reroutedLock.Lock()
+	defer reroutedLock.Unlock()
+	if len(reroutedBuf)+len(buf) > reroutedBufMaxSize {
+		reroutedBufOverflows.Inc()
+		return false
+	}
+	reroutedBuf = append(reroutedBuf, buf...)
+	reroutedRows += rows
+	reroutesTotal.Inc()
+	return true
+}
+
+func spreadReroutedBufToStorageNodes(swapBuf []byte) ([]byte, error) {
+	healthyStorageNodes := getHealthyStorageNodes()
+	if len(healthyStorageNodes) == 0 {
+		// No more vmstorage nodes to write data to.
+		return swapBuf, fmt.Errorf("all the storage nodes are unhealthy")
+	}
+
+	reroutedLock.Lock()
+	reroutedBuf, swapBuf = swapBuf[:0], reroutedBuf
+	rows := reroutedRows
+	reroutedRows = 0
+	reroutedLock.Unlock()
+
+	if len(swapBuf) == 0 {
+		// Nothing to re-route.
+		return swapBuf, nil
+	}
+
+	var mr storage.MetricRow
+	src := swapBuf
+	rowsProcessed := 0
+	for len(src) > 0 {
+		tail, err := mr.Unmarshal(src)
+		if err != nil {
+			logger.Panicf("BUG: cannot unmarshal recently marshaled MetricRow: %s", err)
+		}
+		rowBuf := src[:len(src)-len(tail)]
+		src = tail
+
+		// Use non-consistent hashing instead of jump hash in order to re-route rows
+		// equally among healthy vmstorage nodes.
+		// This should spread the increased load among healthy vmstorage nodes.
+		h := xxhash.Sum64(mr.MetricNameRaw)
+		idx := h % uint64(len(healthyStorageNodes))
+		attempts := 0
+		for {
+			sn := healthyStorageNodes[idx]
+			err := sn.sendReroutedRow(rowBuf)
+			if err == nil {
+				sn.rowsReroutedToHere.Inc()
+				break
+			}
+
+			// Cannot send data to sn. Try sending to the next vmstorage node.
+			idx++
+			if idx >= uint64(len(healthyStorageNodes)) {
+				idx = 0
+			}
+			attempts++
+			if attempts < len(healthyStorageNodes) {
+				continue
+			}
+			// There are no healthy nodes.
+			// Try returning the remaining data to reroutedBuf if it has enough free space.
+			rowsRemaining := rows - rowsProcessed
+			recovered := false
+			reroutedLock.Lock()
+			if len(rowBuf)+len(tail)+len(reroutedBuf) <= reroutedBufMaxSize {
+				swapBuf = append(swapBuf[:0], rowBuf...)
+				swapBuf = append(swapBuf, tail...)
+				swapBuf = append(swapBuf, reroutedBuf...)
+				reroutedBuf, swapBuf = swapBuf, reroutedBuf[:0]
+				reroutedRows += rowsRemaining
+				recovered = true
+			}
+			reroutedLock.Unlock()
+			if recovered {
+				return swapBuf, nil
+			}
+			rowsLostTotal.Add(rowsRemaining)
+			return swapBuf, fmt.Errorf("all the %d vmstorage nodes are unavailable; lost %d rows; last error: %s", len(storageNodes), rowsRemaining, err)
+		}
+		rowsProcessed++
+	}
+	if rowsProcessed != rows {
+		logger.Panicf("BUG: unexpected number of rows processed; got %d; want %d", rowsProcessed, rows)
+	}
+	reroutedRowsProcessed.Add(rowsProcessed)
+	return swapBuf, nil
+}
+
+var (
+	reroutedLock       sync.Mutex
+	reroutedBuf        []byte
+	reroutedRows       int
+	reroutedBufMaxSize int
+
+	reroutedRowsProcessed = metrics.NewCounter(`vm_rpc_rerouted_rows_processed_total{name="vminsert"}`)
+	reroutedBufOverflows  = metrics.NewCounter(`vm_rpc_rerouted_buf_overflows_total{name="vminsert"}`)
+	reroutesTotal         = metrics.NewCounter(`vm_rpc_reroutes_total{name="vminsert"}`)
+	_                     = metrics.NewGauge(`vm_rpc_rerouted_rows_pending{name="vminsert"}`, func() float64 {
+		reroutedLock.Lock()
+		n := reroutedRows
+		reroutedLock.Unlock()
+		return float64(n)
+	})
+	_ = metrics.NewGauge(`vm_rpc_rerouted_buf_pending_bytes{name="vminsert"}`, func() float64 {
+		reroutedLock.Lock()
+		n := len(reroutedBuf)
+		reroutedLock.Unlock()
+		return float64(n)
+	})
+
+	rerouteErrors = metrics.NewCounter(`vm_rpc_reroute_errors_total{name="vminsert"}`)
+	rowsLostTotal = metrics.NewCounter(`vm_rpc_rows_lost_total{name="vminsert"}`)
+)
+
+func getHealthyStorageNodes() []*storageNode {
+	sns := make([]*storageNode, 0, len(storageNodes)-1)
+	for _, sn := range storageNodes {
+		sn.mu.Lock()
+		if !sn.broken {
+			sns = append(sns, sn)
+		}
+		sn.mu.Unlock()
+	}
+	return sns
+}

app/vminsert/opentsdb/request_handler.go

@@ -0,0 +1,63 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	atCopy := *at
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if atCopy.AccountID == 0 {
+				// Multi-tenancy support via custom tags.
+				// Do not allow overriding AccountID and ProjectID from atCopy for security reasons.
+				if tag.Key == "VictoriaMetrics_AccountID" {
+					atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+				if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" {
+					atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
+				}
+			}
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(&atCopy, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+	// Assume that all the rows for a single connection belong to the same (AccountID, ProjectID).
+	rowsInserted.Get(&atCopy).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/opentsdbhttp/request_handler.go

@@ -0,0 +1,69 @@
+package opentsdbhttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="opentsdb-http"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	path := req.URL.Path
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		return fmt.Errorf("cannot parse path %q: %s", path, err)
+	}
+	if p.Prefix != "insert" {
+		// This is not our link.
+		return fmt.Errorf("unexpected path requested on HTTP OpenTSDB server: %q", path)
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		return fmt.Errorf("auth error: %s", err)
+	}
+	switch p.Suffix {
+	case "api/put", "opentsdb/api/put":
+		return writeconcurrencylimiter.Do(func() error {
+			return parser.ParseStream(req, func(rows []parser.Row) error {
+				return insertRows(at, rows)
+			})
+		})
+	default:
+		return fmt.Errorf("unexpected path requested on HTTP OpenTSDB server: %q", path)
+	}
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		if err := ctx.WriteDataPoint(at, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+	rowsInserted.Get(at).Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/promremotewrite/request_handler.go

@@ -0,0 +1,54 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(timeseries []prompb.TimeSeries) error {
+			return insertRows(at, timeseries)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	rowsTotal := 0
+	for i := range timeseries {
+		ts := &timeseries[i]
+		storageNodeIdx := ctx.GetStorageNodeIdx(at, ts.Labels)
+		ctx.MetricNameBuf = ctx.MetricNameBuf[:0]
+		for i := range ts.Samples {
+			r := &ts.Samples[i]
+			if len(ctx.MetricNameBuf) == 0 {
+				ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, ts.Labels)
+			}
+			if err := ctx.WriteDataPointExt(at, storageNodeIdx, ctx.MetricNameBuf, r.Timestamp, r.Value); err != nil {
+				return err
+			}
+		}
+		rowsTotal += len(ts.Samples)
+	}
+	rowsInserted.Get(at).Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/vmimport/request_handler.go

@@ -0,0 +1,60 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = tenantmetrics.NewCounterMap(`vm_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(at, rows)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	rowsTotal := 0
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabelBytes(tag.Key, tag.Value)
+		}
+		ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, ctx.Labels)
+		storageNodeIdx := ctx.GetStorageNodeIdx(at, ctx.Labels)
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		for j, value := range values {
+			timestamp := timestamps[j]
+			if err := ctx.WriteDataPointExt(at, storageNodeIdx, ctx.MetricNameBuf, timestamp, value); err != nil {
+				return err
+			}
+		}
+		rowsTotal += len(values)
+	}
+	rowsInserted.Get(at).Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vmrestore/Makefile

@@ -0,0 +1,70 @@
+# All these commands must run from repository root.
+
+vmrestore:
+	APP_NAME=vmrestore $(MAKE) app-local
+
+vmrestore-race:
+	APP_NAME=vmrestore RACE=-race $(MAKE) app-local
+
+vmrestore-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker
+
+vmrestore-pure-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-pure
+
+vmrestore-amd64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-amd64
+
+vmrestore-arm-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm
+
+vmrestore-arm64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm64
+
+vmrestore-ppc64le-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-ppc64le
+
+vmrestore-386-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-386
+
+package-vmrestore:
+	APP_NAME=vmrestore $(MAKE) package-via-docker
+
+package-vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-pure
+
+package-vmrestore-amd64:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-amd64
+
+package-vmrestore-arm:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm
+
+package-vmrestore-arm64:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm64
+
+package-vmrestore-ppc64le:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-ppc64le
+
+package-vmrestore-386:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-386
+
+publish-vmrestore:
+	APP_NAME=vmrestore $(MAKE) publish-via-docker
+
+vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) app-local-pure
+
+vmrestore-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-amd64 ./app/vmrestore
+
+vmrestore-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm ./app/vmrestore
+
+vmrestore-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm64 ./app/vmrestore
+
+vmrestore-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-ppc64le ./app/vmrestore
+
+vmrestore-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-386 ./app/vmrestore

app/vmrestore/README.md

@@ -0,0 +1,105 @@
+## vmrestore
+
+`vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
+
+Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+when restarting `vmrestore` with the same args.
+
+
+### Usage
+
+VictoriaMetrics must be stopped during the restore process.
+
+```
+vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+
+```
+
+* `<bucket>` is [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets) name.
+* `<path/to/backup>` is the path to backup made with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) on GCS bucket.
+* `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
+  to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
+
+The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup.
+
+
+### Troubleshooting
+
+* If `vmrestore` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
+
+
+### Advanced usage
+
+Run `vmrestore -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -maxBytesPerSecond int
+    	The maximum download speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -skipBackupCompleteCheck
+    	Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
+  -src string
+    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -storageDataPath string
+    	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmrestore` from the root folder of the repository.
+   It builds `vmrestore` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmrestore-prod` from the root folder of the repository.
+   It builds `vmrestore-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmrestore
+```

app/vmrestore/deployment/Dockerfile

@@ -0,0 +1,6 @@
+ARG base_image
+FROM $base_image
+
+ENTRYPOINT ["/vmrestore-prod"]
+ARG src_binary
+COPY $src_binary ./vmrestore-prod

app/vmrestore/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	src = flag.String("src", "", "Source path with backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
+		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
+	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
+	maxBytesPerSecond       = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file")
+)
+
+func main() {
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Restore{
+		Concurrency:             *concurrency,
+		Src:                     srcFS,
+		Dst:                     dstFS,
+		SkipBackupCompleteCheck: *skipBackupCompleteCheck,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot restore from backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmrestore restores VictoriaMetrics data from backups made by vmbackup.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newDstFS() (*fslocal.FS, error) {
+	if len(*storageDataPath) == 0 {
+		return nil, fmt.Errorf("`-storageDataPath` cannot be empty")
+	}
+	fs := &fslocal.FS{
+		Dir:               *storageDataPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize local fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newSrcFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*src)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %s", *src, err)
+	}
+	return fs, nil
+}

app/vmselect/Makefile

@@ -0,0 +1,38 @@
+# All these commands must run from repository root.
+
+run-vmselect:
+	mkdir -p vmselect-cache
+	DOCKER_OPTS='-v $(shell pwd)/vmselect-cache:/cache' \
+	APP_NAME=vmselect \
+	ARGS='-storageNode=localhost:8401 -selectNode=localhost:8481 -cacheDataPath=/cache' \
+	$(MAKE) run-via-docker
+
+vmselect:
+	APP_NAME=vmselect $(MAKE) app-local
+
+vmselect-race:
+	APP_NAME=vmselect RACE=-race $(MAKE) app-local
+
+vmselect-prod:
+	APP_NAME=vmselect $(MAKE) app-via-docker
+
+vmselect-pure-prod:
+	APP_NAME=vmselect $(MAKE) app-via-docker-pure
+
+vmselect-prod-race:
+	APP_NAME=vmselect RACE=-race $(MAKE) app-via-docker
+
+vmselect-pure:
+	APP_NAME=vmselect $(MAKE) app-local-pure
+
+package-vmselect:
+	APP_NAME=vmselect $(MAKE) package-via-docker
+
+package-vmselect-race:
+	APP_NAME=vmselect RACE=-race $(MAKE) package-via-docker
+
+publish-vmselect:
+	APP_NAME=vmselect $(MAKE) publish-via-docker
+
+publish-vmselect-race:
+	APP_NAME=vmselect RACE=-race $(MAKE) publish-via-docker

app/vmselect/README.md

@@ -0,0 +1,6 @@
+`vmselect` performs the following tasks:
+
+- Splits incoming selects to tasks for `vmstorage` nodes and issues these tasks
+  to all the `vmstorage` nodes in the cluster.
+
+- Merges responses from all the `vmstorage` nodes and returns a single response.

app/vmselect/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8481
+
+ENTRYPOINT ["/vmselect-prod"]
+ARG src_binary
+COPY $src_binary ./vmselect-prod

app/vmselect/main.go

@@ -0,0 +1,362 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr        = flag.String("httpListenAddr", ":8481", "Address to listen for http connections")
+	cacheDataPath         = flag.String("cacheDataPath", "", "Path to directory for cache files. Cache isn't saved if empty")
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
+	maxQueueDuration  = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
+		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
+		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+	resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
+	storageNodes      = flagutil.NewArray("storageNode", "Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401")
+)
+
+func getDefaultMaxConcurrentRequests() int {
+	n := runtime.GOMAXPROCS(-1)
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	logger.Infof("starting netstorage at storageNodes %s", *storageNodes)
+	startTime := time.Now()
+	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	if len(*storageNodes) == 0 {
+		logger.Fatalf("missing -storageNode arg")
+	}
+	netstorage.InitStorageNodes(*storageNodes)
+	logger.Infof("started netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	if len(*cacheDataPath) > 0 {
+		tmpDataPath := *cacheDataPath + "/tmp"
+		fs.RemoveDirContents(tmpDataPath)
+		netstorage.InitTmpBlocksDir(tmpDataPath)
+		promql.InitRollupResultCache(*cacheDataPath + "/rollupResult")
+	} else {
+		netstorage.InitTmpBlocksDir("")
+		promql.InitRollupResultCache("")
+	}
+	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
+
+	go func() {
+		httpserver.Serve(*httpListenAddr, requestHandler)
+	}()
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+
+	logger.Infof("gracefully shutting down http service at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http service: %s", err)
+	}
+	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
+
+	logger.Infof("shutting down neststorage...")
+	startTime = time.Now()
+	netstorage.Stop()
+	if len(*cacheDataPath) > 0 {
+		promql.StopRollupResultCache()
+	}
+	logger.Infof("successfully stopped netstorage in %.3f seconds", time.Since(startTime).Seconds())
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the vmselect has been stopped")
+}
+
+var concurrencyCh chan struct{}
+
+var (
+	concurrencyLimitReached = metrics.NewCounter(`vm_concurrent_select_limit_reached_total`)
+	concurrencyLimitTimeout = metrics.NewCounter(`vm_concurrent_select_limit_timeout_total`)
+
+	_ = metrics.NewGauge(`vm_concurrent_select_capacity`, func() float64 {
+		return float64(cap(concurrencyCh))
+	})
+	_ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 {
+		return float64(len(concurrencyCh))
+	})
+)
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	startTime := time.Now()
+	// Limit the number of concurrent queries.
+	select {
+	case concurrencyCh <- struct{}{}:
+		defer func() { <-concurrencyCh }()
+	default:
+		// Sleep for a while until giving up. This should resolve short bursts in requests.
+		concurrencyLimitReached.Inc()
+		t := timerpool.Get(*maxQueueDuration)
+		select {
+		case concurrencyCh <- struct{}{}:
+			timerpool.Put(t)
+			defer func() { <-concurrencyCh }()
+		case <-t.C:
+			timerpool.Put(t)
+			concurrencyLimitTimeout.Inc()
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
+					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
+					*maxConcurrentRequests, *maxQueueDuration),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, "%s", err)
+			return true
+		}
+	}
+
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if path == "/internal/resetRollupResultCache" {
+		if len(*resetCacheAuthKey) > 0 && r.FormValue("authKey") != *resetCacheAuthKey {
+			sendPrometheusError(w, r, fmt.Errorf("invalid authKey=%q for %q", r.FormValue("authKey"), path))
+			return true
+		}
+		promql.ResetRollupResultCache()
+		return true
+	}
+
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		httpserver.Errorf(w, "cannot parse path %q: %s", path, err)
+		return true
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, "auth error: %s", err)
+		return true
+	}
+	switch p.Prefix {
+	case "select":
+		return selectHandler(startTime, w, r, p, at)
+	case "delete":
+		return deleteHandler(startTime, w, r, p, at)
+	default:
+		// This is not our link
+		return false
+	}
+}
+
+func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request, p *httpserver.Path, at *auth.Token) bool {
+	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/label/") {
+		s := p.Suffix[len("prometheus/api/v1/label/"):]
+		if strings.HasSuffix(s, "/values") {
+			labelValuesRequests.Inc()
+			labelName := s[:len(s)-len("/values")]
+			httpserver.EnableCORS(w, r)
+			if err := prometheus.LabelValuesHandler(startTime, at, labelName, w, r); err != nil {
+				labelValuesErrors.Inc()
+				sendPrometheusError(w, r, err)
+				return true
+			}
+			return true
+		}
+	}
+
+	switch p.Suffix {
+	case "prometheus/api/v1/query":
+		queryRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.QueryHandler(startTime, at, w, r); err != nil {
+			queryErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/query_range":
+		queryRangeRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.QueryRangeHandler(startTime, at, w, r); err != nil {
+			queryRangeErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/series":
+		seriesRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.SeriesHandler(startTime, at, w, r); err != nil {
+			seriesErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/series/count":
+		seriesCountRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.SeriesCountHandler(startTime, at, w, r); err != nil {
+			seriesCountErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/labels":
+		labelsRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.LabelsHandler(startTime, at, w, r); err != nil {
+			labelsErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/labels/count":
+		labelsCountRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.LabelsCountHandler(startTime, at, w, r); err != nil {
+			labelsCountErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/status/tsdb":
+		tsdbStatusRequests.Inc()
+		if err := prometheus.TSDBStatusHandler(startTime, at, w, r); err != nil {
+			tsdbStatusErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/export":
+		exportRequests.Inc()
+		if err := prometheus.ExportHandler(startTime, at, w, r); err != nil {
+			exportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "prometheus/federate":
+		federateRequests.Inc()
+		if err := prometheus.FederateHandler(startTime, at, w, r); err != nil {
+			federateErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "prometheus/api/v1/rules":
+		// Return dumb placeholder
+		rulesRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
+		return true
+	case "prometheus/api/v1/alerts":
+		// Return dumb placehloder
+		alertsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
+		return true
+	case "prometheus/api/v1/metadata":
+		// Return dumb placeholder
+		metadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
+		return true
+	default:
+		return false
+	}
+}
+
+func deleteHandler(startTime time.Time, w http.ResponseWriter, r *http.Request, p *httpserver.Path, at *auth.Token) bool {
+	switch p.Suffix {
+	case "prometheus/api/v1/admin/tsdb/delete_series":
+		deleteRequests.Inc()
+		if err := prometheus.DeleteHandler(startTime, at, r); err != nil {
+			deleteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	default:
+		return false
+	}
+}
+
+func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
+	logger.Warnf("error in %q: %s", r.RequestURI, err)
+
+	w.Header().Set("Content-Type", "application/json")
+	statusCode := http.StatusUnprocessableEntity
+	if esc, ok := err.(*httpserver.ErrorWithStatusCode); ok {
+		statusCode = esc.StatusCode
+	}
+	w.WriteHeader(statusCode)
+	prometheus.WriteErrorResponse(w, statusCode, err)
+}
+
+var (
+	labelValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)
+	labelValuesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)
+
+	queryRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/query"}`)
+	queryErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/query"}`)
+
+	queryRangeRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/query_range"}`)
+	queryRangeErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/query_range"}`)
+
+	seriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/series"}`)
+	seriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/series"}`)
+
+	seriesCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/series/count"}`)
+	seriesCountErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/series/count"}`)
+
+	labelsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/labels"}`)
+	labelsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/labels"}`)
+
+	labelsCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/labels/count"}`)
+	labelsCountErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/labels/count"}`)
+
+	tsdbStatusRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/status/tsdb"}`)
+	tsdbStatusErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/status/tsdb"}`)
+
+	deleteRequests = metrics.NewCounter(`vm_http_requests_total{path="/delete/{}/prometheus/api/v1/admin/tsdb/delete_series"}`)
+	deleteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/delete/{}/prometheus/api/v1/admin/tsdb/delete_series"}`)
+
+	exportRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/export"}`)
+	exportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/export"}`)
+
+	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/federate"}`)
+	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/federate"}`)
+
+	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/rules"}`)
+	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/alerts"}`)
+	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/metadata"}`)
+)

app/vmselect/netstorage/tmp_blocks_file.go

@@ -0,0 +1,185 @@
+package netstorage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// InitTmpBlocksDir initializes directory to store temporary search results.
+//
+// It stores data in system-defined temporary directory if tmpDirPath is empty.
+func InitTmpBlocksDir(tmpDirPath string) {
+	if len(tmpDirPath) == 0 {
+		tmpDirPath = os.TempDir()
+	}
+	tmpBlocksDir = tmpDirPath + "/searchResults"
+	fs.MustRemoveAll(tmpBlocksDir)
+	if err := fs.MkdirAllIfNotExist(tmpBlocksDir); err != nil {
+		logger.Panicf("FATAL: cannot create %q: %s", tmpBlocksDir, err)
+	}
+}
+
+var tmpBlocksDir string
+
+func maxInmemoryTmpBlocksFile() int {
+	mem := memory.Allowed()
+	maxLen := mem / 1024
+	if maxLen < 64*1024 {
+		return 64 * 1024
+	}
+	if maxLen > 4*1024*1024 {
+		return 4 * 1024 * 1024
+	}
+	return maxLen
+}
+
+var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
+	return float64(maxInmemoryTmpBlocksFile())
+})
+
+type tmpBlocksFile struct {
+	buf []byte
+
+	f *os.File
+	r *fs.ReaderAt
+
+	offset uint64
+}
+
+func getTmpBlocksFile() *tmpBlocksFile {
+	v := tmpBlocksFilePool.Get()
+	if v == nil {
+		return &tmpBlocksFile{
+			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
+		}
+	}
+	return v.(*tmpBlocksFile)
+}
+
+func putTmpBlocksFile(tbf *tmpBlocksFile) {
+	tbf.MustClose()
+	tbf.buf = tbf.buf[:0]
+	tbf.f = nil
+	tbf.r = nil
+	tbf.offset = 0
+	tmpBlocksFilePool.Put(tbf)
+}
+
+var tmpBlocksFilePool sync.Pool
+
+type tmpBlockAddr struct {
+	offset uint64
+	size   int
+}
+
+func (addr tmpBlockAddr) String() string {
+	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
+}
+
+var tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
+
+// WriteBlockData writes b to tbf.
+//
+// It returns errors since the operation may fail on space shortage
+// and this must be handled.
+func (tbf *tmpBlocksFile) WriteBlockData(b []byte) (tmpBlockAddr, error) {
+	var addr tmpBlockAddr
+	addr.offset = tbf.offset
+	addr.size = len(b)
+	tbf.offset += uint64(addr.size)
+	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
+		// Fast path - the data fits tbf.buf
+		tbf.buf = append(tbf.buf, b...)
+		return addr, nil
+	}
+
+	// Slow path: flush the data from tbf.buf to file.
+	if tbf.f == nil {
+		f, err := ioutil.TempFile(tmpBlocksDir, "")
+		if err != nil {
+			return addr, err
+		}
+		tbf.f = f
+		tmpBlocksFilesCreated.Inc()
+	}
+	_, err := tbf.f.Write(tbf.buf)
+	tbf.buf = append(tbf.buf[:0], b...)
+	if err != nil {
+		return addr, fmt.Errorf("cannot write block to %q: %s", tbf.f.Name(), err)
+	}
+	return addr, nil
+}
+
+func (tbf *tmpBlocksFile) Finalize() error {
+	if tbf.f == nil {
+		return nil
+	}
+	fname := tbf.f.Name()
+	if _, err := tbf.f.Write(tbf.buf); err != nil {
+		return fmt.Errorf("cannot write the remaining %d bytes to %q: %s", len(tbf.buf), fname, err)
+	}
+	tbf.buf = tbf.buf[:0]
+	r, err := fs.OpenReaderAt(fname)
+	if err != nil {
+		logger.Panicf("FATAL: cannot open %q: %s", fname, err)
+	}
+	// Hint the OS that the file is read almost sequentiallly.
+	// This should reduce the number of disk seeks, which is important
+	// for HDDs.
+	r.MustFadviseSequentialRead(true)
+	tbf.r = r
+	return nil
+}
+
+func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
+	var buf []byte
+	if tbf.f == nil {
+		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
+	} else {
+		bb := tmpBufPool.Get()
+		defer tmpBufPool.Put(bb)
+		bb.B = bytesutil.Resize(bb.B, addr.size)
+		tbf.r.MustReadAt(bb.B, int64(addr.offset))
+		buf = bb.B
+	}
+	tail, err := storage.UnmarshalBlock(dst, buf)
+	if err != nil {
+		logger.Panicf("FATAL: cannot unmarshal data at %s: %s", addr, err)
+	}
+	if len(tail) > 0 {
+		logger.Panicf("FATAL: unexpected non-empty tail left after unmarshaling data at %s; len(tail)=%d", addr, len(tail))
+	}
+}
+
+var tmpBufPool bytesutil.ByteBufferPool
+
+func (tbf *tmpBlocksFile) MustClose() {
+	if tbf.f == nil {
+		return
+	}
+	if tbf.r != nil {
+		// tbf.r could be nil if Finalize wasn't called.
+		tbf.r.MustClose()
+	}
+	fname := tbf.f.Name()
+
+	// Remove the file at first, then close it.
+	// This way the OS shouldn't try to flush file contents to storage
+	// on close.
+	if err := os.Remove(fname); err != nil {
+		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+	}
+	if err := tbf.f.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+	}
+	tbf.f = nil
+}

app/vmselect/netstorage/tmp_blocks_file_test.go

@@ -0,0 +1,153 @@
+package netstorage
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+func TestMain(m *testing.M) {
+	rand.Seed(time.Now().UnixNano())
+	tmpDir := "TestTmpBlocks"
+	InitTmpBlocksDir(tmpDir)
+	statusCode := m.Run()
+	if err := os.RemoveAll(tmpDir); err != nil {
+		logger.Panicf("cannot remove %q: %s", tmpDir, err)
+	}
+	os.Exit(statusCode)
+}
+
+func TestTmpBlocksFileSerial(t *testing.T) {
+	if err := testTmpBlocksFile(); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+}
+
+func TestTmpBlocksFileConcurrent(t *testing.T) {
+	concurrency := 3
+	ch := make(chan error, concurrency)
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			ch <- testTmpBlocksFile()
+		}()
+	}
+	for i := 0; i < concurrency; i++ {
+		select {
+		case err := <-ch:
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+		case <-time.After(30 * time.Second):
+			t.Fatalf("timeout")
+		}
+	}
+}
+
+func testTmpBlocksFile() error {
+	createBlock := func() *storage.Block {
+		rowsCount := rand.Intn(8000) + 1
+		var timestamps, values []int64
+		ts := int64(rand.Intn(1023434))
+		for i := 0; i < rowsCount; i++ {
+			ts += int64(rand.Intn(1000) + 1)
+			timestamps = append(timestamps, ts)
+			values = append(values, int64(i*i+rand.Intn(20)))
+		}
+		tsid := &storage.TSID{
+			MetricID: 234211,
+		}
+		scale := int16(rand.Intn(123))
+		precisionBits := uint8(rand.Intn(63) + 1)
+		var b storage.Block
+		b.Init(tsid, timestamps, values, scale, precisionBits)
+		_, _, _ = b.MarshalData(0, 0)
+		return &b
+	}
+	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile() / 2, 2 * maxInmemoryTmpBlocksFile()} {
+		err := func() error {
+			tbf := getTmpBlocksFile()
+			defer putTmpBlocksFile(tbf)
+
+			// Write blocks until their summary size exceeds `size`.
+			var addrs []tmpBlockAddr
+			var blocks []*storage.Block
+			bb := tmpBufPool.Get()
+			defer tmpBufPool.Put(bb)
+			for tbf.offset < uint64(size) {
+				b := createBlock()
+				bb.B = storage.MarshalBlock(bb.B[:0], b)
+				addr, err := tbf.WriteBlockData(bb.B)
+				if err != nil {
+					return fmt.Errorf("cannot write block at offset %d: %s", tbf.offset, err)
+				}
+				if addr.offset+uint64(addr.size) != tbf.offset {
+					return fmt.Errorf("unexpected addr=%+v for offset %v", &addr, tbf.offset)
+				}
+				addrs = append(addrs, addr)
+				blocks = append(blocks, b)
+			}
+			if err := tbf.Finalize(); err != nil {
+				return fmt.Errorf("cannot finalize tbf: %s", err)
+			}
+
+			// Read blocks in parallel and verify them
+			concurrency := 2
+			workCh := make(chan int)
+			doneCh := make(chan error)
+			for i := 0; i < concurrency; i++ {
+				go func() {
+					doneCh <- func() error {
+						var b1 storage.Block
+						for idx := range workCh {
+							addr := addrs[idx]
+							b := blocks[idx]
+							if err := b.UnmarshalData(); err != nil {
+								return fmt.Errorf("cannot unmarshal data from the original block: %s", err)
+							}
+							b1.Reset()
+							tbf.MustReadBlockAt(&b1, addr)
+							if err := b1.UnmarshalData(); err != nil {
+								return fmt.Errorf("cannot unmarshal data from tbf: %s", err)
+							}
+							if b1.RowsCount() != b.RowsCount() {
+								return fmt.Errorf("unexpected number of rows in tbf block; got %d; want %d", b1.RowsCount(), b.RowsCount())
+							}
+							if !reflect.DeepEqual(b1.Timestamps(), b.Timestamps()) {
+								return fmt.Errorf("unexpected timestamps; got\n%v\nwant\n%v", b1.Timestamps(), b.Timestamps())
+							}
+							if !reflect.DeepEqual(b1.Values(), b.Values()) {
+								return fmt.Errorf("unexpected values; got\n%v\nwant\n%v", b1.Values(), b.Values())
+							}
+						}
+						return nil
+					}()
+				}()
+			}
+			for i := range addrs {
+				workCh <- i
+			}
+			close(workCh)
+			for i := 0; i < concurrency; i++ {
+				select {
+				case err := <-doneCh:
+					if err != nil {
+						return err
+					}
+				case <-time.After(time.Second):
+					return fmt.Errorf("timeout")
+				}
+			}
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

app/vmselect/prometheus/error_response.qtpl

@@ -0,0 +1,11 @@
+{% stripspace %}
+ErrorResponse generates error response for /api/v1/query.
+See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview
+{% func ErrorResponse(statusCode int, err error) %}
+{
+	"status":"error",
+	"errorType":"{%d statusCode %}",
+	"error": {%q= err.Error() %}
+}
+{% endfunc %}
+{% endstripspace %}

app/vmselect/prometheus/error_response.qtpl.go

@@ -0,0 +1,61 @@
+// Code generated by qtc from "error_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// ErrorResponse generates error response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview
+
+//line app/vmselect/prometheus/error_response.qtpl:4
+package prometheus
+
+//line app/vmselect/prometheus/error_response.qtpl:4
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/error_response.qtpl:4
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/error_response.qtpl:4
+func StreamErrorResponse(qw422016 *qt422016.Writer, statusCode int, err error) {
+//line app/vmselect/prometheus/error_response.qtpl:4
+	qw422016.N().S(`{"status":"error","errorType":"`)
+//line app/vmselect/prometheus/error_response.qtpl:7
+	qw422016.N().D(statusCode)
+//line app/vmselect/prometheus/error_response.qtpl:7
+	qw422016.N().S(`","error":`)
+//line app/vmselect/prometheus/error_response.qtpl:8
+	qw422016.N().Q(err.Error())
+//line app/vmselect/prometheus/error_response.qtpl:8
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/error_response.qtpl:10
+}
+
+//line app/vmselect/prometheus/error_response.qtpl:10
+func WriteErrorResponse(qq422016 qtio422016.Writer, statusCode int, err error) {
+//line app/vmselect/prometheus/error_response.qtpl:10
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/error_response.qtpl:10
+	StreamErrorResponse(qw422016, statusCode, err)
+//line app/vmselect/prometheus/error_response.qtpl:10
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/error_response.qtpl:10
+}
+
+//line app/vmselect/prometheus/error_response.qtpl:10
+func ErrorResponse(statusCode int, err error) string {
+//line app/vmselect/prometheus/error_response.qtpl:10
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/error_response.qtpl:10
+	WriteErrorResponse(qb422016, statusCode, err)
+//line app/vmselect/prometheus/error_response.qtpl:10
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/error_response.qtpl:10
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/error_response.qtpl:10
+	return qs422016
+//line app/vmselect/prometheus/error_response.qtpl:10
+}

app/vmselect/prometheus/export.qtpl

@@ -0,0 +1,96 @@
+{% import (
+	"github.com/valyala/quicktemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+) %}
+
+{% stripspace %}
+
+{% func ExportPrometheusLine(rs *netstorage.Result) %}
+	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+	{% code bb := quicktemplate.AcquireByteBuffer() %}
+	{% code writeprometheusMetricName(bb, &rs.MetricName) %}
+	{% for i, ts := range rs.Timestamps %}
+		{%z= bb.B %}{% space %}
+		{%f= rs.Values[i] %}{% space %}
+		{%dl= ts %}{% newline %}
+	{% endfor %}
+	{% code quicktemplate.ReleaseByteBuffer(bb) %}
+{% endfunc %}
+
+{% func ExportJSONLine(rs *netstorage.Result) %}
+	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+	{
+		"metric":{%= metricNameObject(&rs.MetricName) %},
+		"values":[
+			{% if len(rs.Values) > 0 %}
+				{% code values := rs.Values %}
+				{%f= values[0] %}
+				{% code values = values[1:] %}
+				{% for _, v := range values %}
+					,{%f= v %}
+				{% endfor %}
+			{% endif %}
+		],
+		"timestamps":[
+			{% if len(rs.Timestamps) > 0 %}
+				{% code timestamps := rs.Timestamps %}
+				{%dl= timestamps[0] %}
+				{% code timestamps = timestamps[1:] %}
+				{% for _, ts := range timestamps %}
+					,{%dl= ts %}
+				{% endfor %}
+			{% endif %}
+		]
+	}{% newline %}
+{% endfunc %}
+
+{% func ExportPromAPILine(rs *netstorage.Result) %}
+{
+	"metric": {%= metricNameObject(&rs.MetricName) %},
+	"values": {%= valuesWithTimestamps(rs.Values, rs.Timestamps) %}
+}
+{% endfunc %}
+
+{% func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
+{
+	"status":"success",
+	"data":{
+		"resultType":"matrix",
+		"result":[
+			{% code bb, ok := <-resultsCh %}
+			{% if ok %}
+				{%z= bb.B %}
+				{% code quicktemplate.ReleaseByteBuffer(bb) %}
+				{% for bb := range resultsCh %}
+					,{%z= bb.B %}
+					{% code quicktemplate.ReleaseByteBuffer(bb) %}
+				{% endfor %}
+			{% endif %}
+		]
+	}
+}
+{% endfunc %}
+
+{% func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
+	{% for bb := range resultsCh %}
+		{%z= bb.B %}
+		{% code quicktemplate.ReleaseByteBuffer(bb) %}
+	{% endfor %}
+{% endfunc %}
+
+{% func prometheusMetricName(mn *storage.MetricName) %}
+	{%z= mn.MetricGroup %}
+	{% if len(mn.Tags) > 0 %}
+	{
+		{% code tags := mn.Tags %}
+		{%z= tags[0].Key %}={%qz= tags[0].Value %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%z= tag.Key %}={%qz= tag.Value %}
+		{% endfor %}
+	}
+	{% endif %}
+{% endfunc %}
+{% endstripspace %}

app/vmselect/prometheus/export.qtpl.go

@@ -0,0 +1,385 @@
+// Code generated by qtc from "export.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/export.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/export.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/export.qtpl:9
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/export.qtpl:9
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/export.qtpl:9
+func StreamExportPrometheusLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:10
+	if len(rs.Timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:10
+		return
+//line app/vmselect/prometheus/export.qtpl:10
+	}
+//line app/vmselect/prometheus/export.qtpl:11
+	bb := quicktemplate.AcquireByteBuffer()
+
+//line app/vmselect/prometheus/export.qtpl:12
+	writeprometheusMetricName(bb, &rs.MetricName)
+
+//line app/vmselect/prometheus/export.qtpl:13
+	for i, ts := range rs.Timestamps {
+//line app/vmselect/prometheus/export.qtpl:14
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:14
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/export.qtpl:15
+		qw422016.N().F(rs.Values[i])
+//line app/vmselect/prometheus/export.qtpl:15
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/export.qtpl:16
+		qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:16
+		qw422016.N().S(`
+`)
+//line app/vmselect/prometheus/export.qtpl:17
+	}
+//line app/vmselect/prometheus/export.qtpl:18
+	quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:19
+}
+
+//line app/vmselect/prometheus/export.qtpl:19
+func WriteExportPrometheusLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:19
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:19
+	StreamExportPrometheusLine(qw422016, rs)
+//line app/vmselect/prometheus/export.qtpl:19
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:19
+}
+
+//line app/vmselect/prometheus/export.qtpl:19
+func ExportPrometheusLine(rs *netstorage.Result) string {
+//line app/vmselect/prometheus/export.qtpl:19
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:19
+	WriteExportPrometheusLine(qb422016, rs)
+//line app/vmselect/prometheus/export.qtpl:19
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:19
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:19
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:19
+}
+
+//line app/vmselect/prometheus/export.qtpl:21
+func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:22
+	if len(rs.Timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:22
+		return
+//line app/vmselect/prometheus/export.qtpl:22
+	}
+//line app/vmselect/prometheus/export.qtpl:22
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:24
+	streammetricNameObject(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/export.qtpl:24
+	qw422016.N().S(`,"values":[`)
+//line app/vmselect/prometheus/export.qtpl:26
+	if len(rs.Values) > 0 {
+//line app/vmselect/prometheus/export.qtpl:27
+		values := rs.Values
+
+//line app/vmselect/prometheus/export.qtpl:28
+		qw422016.N().F(values[0])
+//line app/vmselect/prometheus/export.qtpl:29
+		values = values[1:]
+
+//line app/vmselect/prometheus/export.qtpl:30
+		for _, v := range values {
+//line app/vmselect/prometheus/export.qtpl:30
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:31
+			qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:32
+		}
+//line app/vmselect/prometheus/export.qtpl:33
+	}
+//line app/vmselect/prometheus/export.qtpl:33
+	qw422016.N().S(`],"timestamps":[`)
+//line app/vmselect/prometheus/export.qtpl:36
+	if len(rs.Timestamps) > 0 {
+//line app/vmselect/prometheus/export.qtpl:37
+		timestamps := rs.Timestamps
+
+//line app/vmselect/prometheus/export.qtpl:38
+		qw422016.N().DL(timestamps[0])
+//line app/vmselect/prometheus/export.qtpl:39
+		timestamps = timestamps[1:]
+
+//line app/vmselect/prometheus/export.qtpl:40
+		for _, ts := range timestamps {
+//line app/vmselect/prometheus/export.qtpl:40
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:41
+			qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:42
+		}
+//line app/vmselect/prometheus/export.qtpl:43
+	}
+//line app/vmselect/prometheus/export.qtpl:43
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/export.qtpl:45
+	qw422016.N().S(`
+`)
+//line app/vmselect/prometheus/export.qtpl:46
+}
+
+//line app/vmselect/prometheus/export.qtpl:46
+func WriteExportJSONLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:46
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:46
+	StreamExportJSONLine(qw422016, rs)
+//line app/vmselect/prometheus/export.qtpl:46
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:46
+}
+
+//line app/vmselect/prometheus/export.qtpl:46
+func ExportJSONLine(rs *netstorage.Result) string {
+//line app/vmselect/prometheus/export.qtpl:46
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:46
+	WriteExportJSONLine(qb422016, rs)
+//line app/vmselect/prometheus/export.qtpl:46
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:46
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:46
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:46
+}
+
+//line app/vmselect/prometheus/export.qtpl:48
+func StreamExportPromAPILine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:48
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:50
+	streammetricNameObject(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/export.qtpl:50
+	qw422016.N().S(`,"values":`)
+//line app/vmselect/prometheus/export.qtpl:51
+	streamvaluesWithTimestamps(qw422016, rs.Values, rs.Timestamps)
+//line app/vmselect/prometheus/export.qtpl:51
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:53
+}
+
+//line app/vmselect/prometheus/export.qtpl:53
+func WriteExportPromAPILine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/export.qtpl:53
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:53
+	StreamExportPromAPILine(qw422016, rs)
+//line app/vmselect/prometheus/export.qtpl:53
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:53
+}
+
+//line app/vmselect/prometheus/export.qtpl:53
+func ExportPromAPILine(rs *netstorage.Result) string {
+//line app/vmselect/prometheus/export.qtpl:53
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:53
+	WriteExportPromAPILine(qb422016, rs)
+//line app/vmselect/prometheus/export.qtpl:53
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:53
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:53
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:53
+}
+
+//line app/vmselect/prometheus/export.qtpl:55
+func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:55
+	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/export.qtpl:61
+	bb, ok := <-resultsCh
+
+//line app/vmselect/prometheus/export.qtpl:62
+	if ok {
+//line app/vmselect/prometheus/export.qtpl:63
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:64
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:65
+		for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:65
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:66
+			qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:67
+			quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:68
+		}
+//line app/vmselect/prometheus/export.qtpl:69
+	}
+//line app/vmselect/prometheus/export.qtpl:69
+	qw422016.N().S(`]}}`)
+//line app/vmselect/prometheus/export.qtpl:73
+}
+
+//line app/vmselect/prometheus/export.qtpl:73
+func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:73
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:73
+	StreamExportPromAPIResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:73
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:73
+}
+
+//line app/vmselect/prometheus/export.qtpl:73
+func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:73
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:73
+	WriteExportPromAPIResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:73
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:73
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:73
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:73
+}
+
+//line app/vmselect/prometheus/export.qtpl:75
+func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:76
+	for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:77
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:78
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:79
+	}
+//line app/vmselect/prometheus/export.qtpl:80
+}
+
+//line app/vmselect/prometheus/export.qtpl:80
+func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:80
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:80
+	StreamExportStdResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:80
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:80
+}
+
+//line app/vmselect/prometheus/export.qtpl:80
+func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:80
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:80
+	WriteExportStdResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:80
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:80
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:80
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:80
+}
+
+//line app/vmselect/prometheus/export.qtpl:82
+func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:83
+	qw422016.N().Z(mn.MetricGroup)
+//line app/vmselect/prometheus/export.qtpl:84
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/export.qtpl:84
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/export.qtpl:86
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/export.qtpl:87
+		qw422016.N().Z(tags[0].Key)
+//line app/vmselect/prometheus/export.qtpl:87
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:87
+		qw422016.N().QZ(tags[0].Value)
+//line app/vmselect/prometheus/export.qtpl:88
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/export.qtpl:89
+		for i := range tags {
+//line app/vmselect/prometheus/export.qtpl:90
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/export.qtpl:90
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:91
+			qw422016.N().Z(tag.Key)
+//line app/vmselect/prometheus/export.qtpl:91
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:91
+			qw422016.N().QZ(tag.Value)
+//line app/vmselect/prometheus/export.qtpl:92
+		}
+//line app/vmselect/prometheus/export.qtpl:92
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:94
+	}
+//line app/vmselect/prometheus/export.qtpl:95
+}
+
+//line app/vmselect/prometheus/export.qtpl:95
+func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:95
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:95
+	streamprometheusMetricName(qw422016, mn)
+//line app/vmselect/prometheus/export.qtpl:95
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:95
+}
+
+//line app/vmselect/prometheus/export.qtpl:95
+func prometheusMetricName(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/export.qtpl:95
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:95
+	writeprometheusMetricName(qb422016, mn)
+//line app/vmselect/prometheus/export.qtpl:95
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:95
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:95
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:95
+}

app/vmselect/prometheus/federate.qtpl

@@ -0,0 +1,16 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+) %}
+
+{% stripspace %}
+
+// Federate writes rs in /federate format.
+// See https://prometheus.io/docs/prometheus/latest/federation/
+{% func Federate(rs *netstorage.Result) %}
+	{% if len(rs.Timestamps) == 0 || len(rs.Values) == 0 %}{% return %}{% endif %}
+	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
+	{%f= rs.Values[len(rs.Values)-1] %}{% space %}
+	{%dl= rs.Timestamps[len(rs.Timestamps)-1] %}{% newline %}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -0,0 +1,75 @@
+// Code generated by qtc from "federate.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/federate.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/federate.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+)
+
+// Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/
+
+//line app/vmselect/prometheus/federate.qtpl:9
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/federate.qtpl:9
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/federate.qtpl:9
+func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/federate.qtpl:10
+	if len(rs.Timestamps) == 0 || len(rs.Values) == 0 {
+//line app/vmselect/prometheus/federate.qtpl:10
+		return
+//line app/vmselect/prometheus/federate.qtpl:10
+	}
+//line app/vmselect/prometheus/federate.qtpl:11
+	streamprometheusMetricName(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:11
+	qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:12
+	qw422016.N().F(rs.Values[len(rs.Values)-1])
+//line app/vmselect/prometheus/federate.qtpl:12
+	qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:13
+	qw422016.N().DL(rs.Timestamps[len(rs.Timestamps)-1])
+//line app/vmselect/prometheus/federate.qtpl:13
+	qw422016.N().S(`
+`)
+//line app/vmselect/prometheus/federate.qtpl:14
+}
+
+//line app/vmselect/prometheus/federate.qtpl:14
+func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) {
+//line app/vmselect/prometheus/federate.qtpl:14
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:14
+	StreamFederate(qw422016, rs)
+//line app/vmselect/prometheus/federate.qtpl:14
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:14
+}
+
+//line app/vmselect/prometheus/federate.qtpl:14
+func Federate(rs *netstorage.Result) string {
+//line app/vmselect/prometheus/federate.qtpl:14
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:14
+	WriteFederate(qb422016, rs)
+//line app/vmselect/prometheus/federate.qtpl:14
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:14
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:14
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:14
+}

app/vmselect/prometheus/label_values_response.qtpl

@@ -0,0 +1,15 @@
+{% stripspace %}
+LabelValuesResponse generates response for /api/v1/label/<labelName>/values .
+See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
+{% func LabelValuesResponse(labelValues []string) %}
+{
+	"status":"success",
+	"data":[
+		{% for i, labelValue := range labelValues %}
+			{%q= labelValue %}
+			{% if i+1 < len(labelValues) %},{% endif %}
+		{% endfor %}
+	]
+}
+{% endfunc %}
+{% endstripspace %}

app/vmselect/prometheus/label_values_response.qtpl.go

@@ -0,0 +1,67 @@
+// Code generated by qtc from "label_values_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// LabelValuesResponse generates response for /api/v1/label/<labelName>/values .See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
+
+//line app/vmselect/prometheus/label_values_response.qtpl:4
+package prometheus
+
+//line app/vmselect/prometheus/label_values_response.qtpl:4
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/label_values_response.qtpl:4
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/label_values_response.qtpl:4
+func StreamLabelValuesResponse(qw422016 *qt422016.Writer, labelValues []string) {
+//line app/vmselect/prometheus/label_values_response.qtpl:4
+	qw422016.N().S(`{"status":"success","data":[`)
+//line app/vmselect/prometheus/label_values_response.qtpl:8
+	for i, labelValue := range labelValues {
+//line app/vmselect/prometheus/label_values_response.qtpl:9
+		qw422016.N().Q(labelValue)
+//line app/vmselect/prometheus/label_values_response.qtpl:10
+		if i+1 < len(labelValues) {
+//line app/vmselect/prometheus/label_values_response.qtpl:10
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/label_values_response.qtpl:10
+		}
+//line app/vmselect/prometheus/label_values_response.qtpl:11
+	}
+//line app/vmselect/prometheus/label_values_response.qtpl:11
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+}
+
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+func WriteLabelValuesResponse(qq422016 qtio422016.Writer, labelValues []string) {
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	StreamLabelValuesResponse(qw422016, labelValues)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+}
+
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+func LabelValuesResponse(labelValues []string) string {
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	WriteLabelValuesResponse(qb422016, labelValues)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+	return qs422016
+//line app/vmselect/prometheus/label_values_response.qtpl:14
+}

app/vmselect/prometheus/labels_count_response.qtpl

@@ -0,0 +1,17 @@
+{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %}
+
+{% stripspace %}
+LabelsCountResponse generates response for /api/v1/label_entries .
+{% func LabelsCountResponse(labelEntries []storage.TagEntry) %}
+{
+	"status":"success",
+	"data":{
+		{% for i, e := range labelEntries %}
+			{%q= e.Key %}:{%d= len(e.Values) %}
+			{% if i+1 < len(labelEntries) %},{% endif %}
+		{% endfor %}
+	}
+}
+{% endfunc %}
+
+{% endstripspace %}