fix misspelled words (#229)

Continue trying to remove NFS directory on temporary errors for up to a minute.

The previous async removal process breaks in the following case during VictoriaMetrics start

- VictoriaMetrics opens index, finds incomplete merge transactions and starts replaying them.
- The transaction instructs removing old directories for parts, which were already merged into bigger part.
- VictoriaMetrics removes these directories, but their removal is delayed due to NFS errors.
- VictoriaMetrics scans partition directory after all the incomplete merge transactions are finished
  and finds directories, which should be removed, but weren't still removed due to NFS errors.
- VictoriaMetrics panics when it finds unexpected empty directory.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/162

.github/workflows/main.yml

@@ -0,0 +1,45 @@
+name: main
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: Code checkout
+        uses: actions/checkout@v1
+      - name: Dependencies
+        env:
+          GO111MODULE: off
+        run: |
+          go get -v golang.org/x/lint/golint
+          go get -u github.com/kisielk/errcheck
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+            export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+            make check-all
+            git diff --exit-code
+            make test-full
+            make test-pure
+            make test-full-386
+            make victoria-metrics
+            make victoria-metrics-pure
+            make victoria-metrics-arm
+            make victoria-metrics-arm64
+            make vmutils
+            GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+            GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+      - name: Publish coverage
+        uses: codecov/codecov-action@v1.0.0
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+          file: ./coverage.txt
+

.travis.yml

@@ -1,26 +0,0 @@
-language: go
-
-go:
-  - 1.12.x
-
-install: make
-
-env:
-  - GO111MODULE=on
-
-before_install:
-  - GO111MODULE=off go get -v golang.org/x/lint/golint
-  - GO111MODULE=off go get -u github.com/kisielk/errcheck
-
-script:
-  - make check-all
-  - git diff --exit-code
-  - make test-full
-  - make test-pure
-  - make victoria-metrics
-  - make victoria-metrics-pure
-  - make victoria-metrics-arm
-  - make victoria-metrics-arm64
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -1,7 +1,7 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}')))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -19,13 +19,35 @@ include deployment/*/Makefile
 clean:
 	rm -rf bin/*
 
-publish: publish-victoria-metrics
+publish: \
+	publish-victoria-metrics \
+	publish-vmbackup \
+	publish-vmrestore
 
-package: package-victoria-metrics
+package: \
+	package-victoria-metrics \
+	package-vmbackup \
+	package-vmrestore
 
-release: victoria-metrics-prod
+vmutils: \
+	vmbackup \
+	vmrestore
+
+release: \
+	release-victoria-metrics \
+	release-vmutils
+
+release-victoria-metrics: victoria-metrics-prod
 	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod
 
+release-vmutils: \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmbackup-prod vmrestore-prod
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+
 fmt:
 	GO111MODULE=on gofmt -l -w -s ./lib
 	GO111MODULE=on gofmt -l -w -s ./app
@@ -39,13 +61,15 @@ lint: install-golint
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get -u github.com/golang/lint/golint
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
 
 install-errcheck:
 	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
@@ -61,6 +85,9 @@ test-pure:
 test-full:
 	GO111MODULE=on go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
+test-full-386:
+	GO111MODULE=on GOARCH=386 go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
 benchmark:
 	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
 	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
@@ -75,6 +102,12 @@ vendor-update:
 	GO111MODULE=on go mod tidy
 	GO111MODULE=on go mod vendor
 
+app-local:
+	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
 quicktemplate-gen: install-qtc
 	qtc
 
@@ -83,7 +116,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck
+	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck -D structcheck
 
 install-golangci-lint:
 	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint

README.md

@@ -2,7 +2,7 @@
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](http://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
-[![Build Status](https://travis-ci.org/VictoriaMetrics/VictoriaMetrics.svg?branch=master)](https://travis-ci.org/VictoriaMetrics/VictoriaMetrics)
+[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 
 <img alt="Victoria Metrics" src="logo.png">
@@ -21,7 +21,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
 
 * Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
   Additionally, VictoriaMetrics extends PromQL with opt-in [useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
-* Global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
+* Supports global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
 * High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
   and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
   [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
@@ -36,15 +36,17 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   * VictoriaMetrics consists of a single executable without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
   * All the data is stored in a single directory pointed by `-storageDataPath` flag.
-  * Easy backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+  with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) / [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
 * Storage is protected from corruption on unclean shutdown (i.e. hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-* Supports metrics' ingestion and backfilling via the following protocols:
+* Supports metrics' ingestion and [backfilling](#backfilling) via the following protocols:
   * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
   * [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
   * [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
     if `-graphiteListenAddr` is set.
   * [OpenTSDB put message](http://opentsdb.net/docs/build/html/api_telnet/put.html) if `-opentsdbListenAddr` is set.
-* Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars and industrial telemetry.
+  * [HTTP OpenTSDB /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) if `-opentsdbHTTPListenAddr` is set.
+* Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry and various Enterprise workloads.
 * Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 
@@ -86,6 +88,9 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   - [Tuning](#tuning)
   - [Monitoring](#monitoring)
   - [Troubleshooting](#troubleshooting)
+  - [Backfilling](#backfilling)
+  - [Profiling](#profiling)
+- [Integrations](#integrations)
 - [Roadmap](#roadmap)
 - [Contacts](#contacts)
 - [Community and contributions](#community-and-contributions)
@@ -104,11 +109,12 @@ or [docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) wi
 
 The following command-line flags are used the most:
 
-* `-storageDataPath` - path to data directory. VictoriaMetrics stores all the data in this directory.
-* `-retentionPeriod` - retention period in months for the data. Older data is automatically deleted.
+* `-storageDataPath` - path to data directory. VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in current working directory.
+* `-retentionPeriod` - retention period in months for the data. Older data is automatically deleted. Default period is 1 month.
 * `-httpListenAddr` - TCP address to listen to for http requests. By default, it listens port `8428` on all the network interfaces.
 * `-graphiteListenAddr` - TCP and UDP address to listen to for Graphite data. By default, it is disabled.
-* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data. By default, it is disabled.
+* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data over telnet protocol. By default, it is disabled.
+* `-opentsdbHTTPListenAddr` - TCP address to listen to for HTTP OpenTSDB data over `/api/put`. By default, it is disabled.
 
 Pass `-help` to see all the available flags with description and default values.
 
@@ -152,7 +158,7 @@ The label name may be arbitrary - `datacenter` is just an example. The label val
 across Prometheus instances, so those time series may be filtered and grouped by this label.
 
 
-It is recommended upgrading Prometheus to [v2.10.0](https://github.com/prometheus/prometheus/releases) or newer,
+It is recommended upgrading Prometheus to [v2.12.0](https://github.com/prometheus/prometheus/releases) or newer,
 since the previous versions may have issues with `remote_write`.
 
 
@@ -167,7 +173,7 @@ http://<victoriametrics-addr>:8428
 Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
 
 Then build graphs with the created datasource using [Prometheus query language](https://prometheus.io/docs/prometheus/latest/querying/basics/).
-VictoriaMetrics supports native PromQL and [extends it with useful features](ExtendedPromQL).
+VictoriaMetrics supports native PromQL and [extends it with useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
 
 
 ### How to upgrade VictoriaMetrics?
@@ -182,6 +188,9 @@ Follow the following steps during the upgrade:
 2) Wait until the process stops. This can take a few seconds.
 3) Start the upgraded VictoriaMetrics.
 
+Prometheus doesn't drop data during VictoriaMetrics restart.
+See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+
 
 ### How to apply new config to VictoriaMetrics?
 
@@ -191,6 +200,9 @@ VictoriaMetrics must be restarted for applying new config:
 2) Wait until the process stops. This can take a few seconds.
 3) Start VictoriaMetrics with the new config.
 
+Prometheus doesn't drop data during VictoriaMetrics restart.
+See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+
 
 ### How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)?
 
@@ -205,7 +217,8 @@ For instance, put the following lines into `Telegraf` config, so it sends data t
 Do not forget substituting `<victoriametrics-addr>` with the real address where VictoriaMetrics runs.
 
 VictoriaMetrics maps Influx data using the following rules:
-* [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value.
+* [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
+  unless `db` tag exists in the Influx line.
 * Field names are mapped to time series names prefixed with `{measurement}{separator}` value,
   where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag.
   See also `-influxSkipSingleField` command-line flag.
@@ -236,14 +249,14 @@ An arbitrary number of lines delimited by '\n' may be sent in a single request.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
 
 ```
-{"metric":{"__name__":"measurement.field1","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560272508147]}
-{"metric":{"__name__":"measurement.field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1560272508147]}
+{"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560272508147]}
+{"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1560272508147]}
 ```
 
 Note that Influx line protocol expects [timestamps in *nanoseconds* by default](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/#timestamp),
@@ -274,7 +287,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
@@ -294,8 +307,13 @@ or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/mas
 
 ### How to send data from OpenTSDB-compatible agents?
 
+VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
+and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
+
+#### Sending data via `telnet put` protocol
+
 1) Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
-the following command will enable OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
+the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
 
 ```
 /path/to/victoria-metrics-prod -opentsdbListenAddr=:4242
@@ -314,7 +332,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
@@ -324,6 +342,44 @@ The `/api/v1/export` endpoint should return the following response:
 ```
 
 
+#### Sending OpenTSDB data via HTTP `/api/put` requests
+
+1) Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance,
+the following command enables OpenTSDB HTTP server on port `4242`:
+
+```
+/path/to/victoria-metrics-prod -opentsdbHTTPListenAddr=:4242
+```
+
+2) Send data to the given address from OpenTSDB-compatible agents.
+
+Example for writing a single data point:
+
+```
+curl -H 'Content-Type: application/json' -d '{"metric":"x.y.z","value":45.34,"tags":{"t1":"v1","t2":"v2"}}' http://localhost:4242/api/put
+```
+
+Example for writing multiple data points in a single request:
+
+```
+curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
+```
+
+After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+
+```
+curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar'
+```
+
+The `/api/v1/export` endpoint should return the following response:
+
+```
+{"metric":{"__name__":"foo"},"values":[45.34],"timestamps":[1566464846000]}
+{"metric":{"__name__":"bar"},"values":[43],"timestamps":[1566464846000]}
+{"metric":{"__name__":"x.y.z","t1":"v1","t2":"v2"},"values":[45.34],"timestamps":[1566464763000]}
+```
+
+
 ### How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -406,8 +462,8 @@ The page will return the following JSON response:
 ```
 
 Snapshots are created under `<-storageDataPath>/snapshots` directory, where `<-storageDataPath>`
-is the command-line flag value. Snapshots can be archived to backup storage via `cp -L`, `rsync -L`, `scp -r`
-or any similar tool that follows symlinks during copying.
+is the command-line flag value. Snapshots can be archived to backup storage at any time
+with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
 
 The `http://<victoriametrics-addr>:8428/snapshot/list` page contains the list of available snapshots.
 
@@ -419,7 +475,8 @@ Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete_all` in order to
 Steps for restoring from a snapshot:
 1. Stop VictoriaMetrics with `kill -INT`.
 2. Remove the entire contents of the directory pointed by `-storageDataPath` command-line flag.
-3. Copy snapshot contents to the directory pointed by `-storageDataPath`.
+3. Restore snapshot contents from backup with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md)
+   to the directory pointed by `-storageDataPath`.
 4. Start VictoriaMetrics.
 
 
@@ -430,6 +487,9 @@ where `<timeseries_selector_for_delete>` may contain any [time series selector](
 for metrics to delete. After that all the time series matching the given selector are deleted. Storage space for
 the deleted time series isn't freed instantly - it is freed during subsequent merges of data files.
 
+It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
+before actually deleting the metrics.
+
 
 ### How to export time series?
 
@@ -454,7 +514,7 @@ at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for
 
 Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
 `start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. By default, the last point
-on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but can be overridden.
+on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden.
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
 with scrape intervals exceeding `5m`.
 
@@ -470,7 +530,7 @@ A rough estimation of the required resources for ingestion path:
   VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited by `-memory.allowedPercent` flag.
 
 * CPU cores: a CPU core per 300K inserted data points per second. So, ~4 CPU cores are required for processing
-  the insert stream of 1M data points per second. The ingestion rate may be lower for high cardinality data.
+  the insert stream of 1M data points per second. The ingestion rate may be lower for high cardinality data or for time series with high number of labels.
   See [this article](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) for details.
   If you see lower numbers per CPU core, then it is likely active time series info doesn't fit caches,
   so you need more RAM for lowering CPU usage.
@@ -595,6 +655,14 @@ For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<i
 * There is no need in Operating System tuning since VictoriaMetrics is optimized for default OS settings.
   The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
   so Prometheus instances could establish more connections to VictoriaMetrics.
+* The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
+  since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
+  If you plan storing more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
+  then the following options are recommended to pass to `mkfs.ext4`:
+
+```
+mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
+```
 
 
 ### Monitoring
@@ -607,10 +675,8 @@ The most interesting metrics are:
 
 * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
   aka active time series.
-* `vm_rows{type="indexdb"}` - the number of rows in inverted index. Each label in each unique time series adds a single
-  row into the inverted index. An approximate number of time series in the database may be calculated as
-  `vm_rows{type="indexdb"} / (avg_labels_per_series + 1)`, where `avg_labels_per_series` is the average number of labels
-  per each time series.
+* `rate(vm_new_timeseries_created_total[5m])` - time series churn rate.
+* `vm_rows{type="indexdb"}` - the number of rows in inverted index. High value for this number usually mean high churn rate for time series.
 * Sum of `vm_rows{type="storage/big"}` and `vm_rows{type="storage/small"}` - total number of `(timestamp, value)` data points
   in the database.
 * Sum of all the `vm_cache_size_bytes` metrics - the total size of all the caches in the database.
@@ -621,6 +687,9 @@ The most interesting metrics are:
 
 ### Troubleshooting
 
+* It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
+  in tweaking these flag values arises.
+
 * If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
   then it is likely you have too many active time series for the current amount of RAM.
   It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
@@ -637,6 +706,43 @@ The most interesting metrics are:
   of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
   for automatic recovering from such errors.
 
+
+### Backfilling
+
+Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
+
+It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
+historical data with timestamps from the past, since the cache assumes that the data is written with
+the current timestamps. Query cache can be enabled after the backfilling is complete.
+
+
+### Profiling
+
+VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+- Memory profile. It can be collected with the following command:
+```
+curl -s http://<victoria-metrics-host>:8428/debug/pprof/heap > mem.pprof
+```
+
+- CPU profile. It can be collected with the following command:
+```
+curl -s http://<victoria-metrics-host>:8428/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+
+
+## Integrations
+
+* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
+  See [these docs](https://github.com/netdata/netdata#integrations).
+* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
+  See [this example](/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+
+
 ## Roadmap
 
 - [ ] Replication [#118](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/118)
@@ -659,8 +765,8 @@ Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics
 Feel free asking any questions regarding VictoriaMetrics:
 
 - [slack](http://slack.victoriametrics.com/)
-- [telergam-en](https://t.me/VictoriaMetrics_en)
-- [telergam-ru](https://t.me/VictoriaMetrics_ru1)
+- [telegram-en](https://t.me/VictoriaMetrics_en)
+- [telegram-ru](https://t.me/VictoriaMetrics_ru1)
 - [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
 
 

app/victoria-metrics/Makefile

@@ -1,7 +1,7 @@
 # All these commands must run from repository root.
 
 victoria-metrics:
-	GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics ./app/victoria-metrics
+	APP_NAME=victoria-metrics $(MAKE) app-local
 
 victoria-metrics-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker
@@ -32,8 +32,14 @@ victoria-metrics-arm64:
 victoria-metrics-arm64-prod:
 	APP_NAME=victoria-metrics APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker
 
+victoria-metrics-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+
+victoria-metrics-386-prod:
+	APP_NAME=victoria-metrics APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker
+
 victoria-metrics-pure:
-	GO111MODULE=on CGO_ENABLED=0 go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-pure ./app/victoria-metrics
+	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 
 victoria-metrics-pure-prod:
 	APP_NAME=victoria-metrics APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker

app/victoria-metrics/main_test.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net"
@@ -18,26 +19,31 @@ import (
 	"testing"
 	"time"
 
+	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 const (
-	testFixturesDir        = "testdata"
-	testStorageSuffix      = "vm-test-storage"
-	testHTTPListenAddr     = ":7654"
-	testStatsDListenAddr   = ":2003"
-	testOpenTSDBListenAddr = ":4242"
-	testLogLevel           = "INFO"
+	testFixturesDir            = "testdata"
+	testStorageSuffix          = "vm-test-storage"
+	testHTTPListenAddr         = ":7654"
+	testStatsDListenAddr       = ":2003"
+	testOpenTSDBListenAddr     = ":4242"
+	testOpenTSDBHTTPListenAddr = ":4243"
+	testLogLevel               = "INFO"
 )
 
 const (
-	testReadHTTPPath   = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath  = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -50,18 +56,69 @@ var (
 )
 
 type test struct {
-	Name   string `json:"name"`
-	Data   string `json:"data"`
-	Query  string `json:"query"`
-	Result []Row  `json:"result"`
+	Name             string     `json:"name"`
+	Data             []string   `json:"data"`
+	Query            []string   `json:"query"`
+	ResultMetrics    []Metric   `json:"result_metrics"`
+	ResultSeries     Series     `json:"result_series"`
+	ResultQuery      Query      `json:"result_query"`
+	ResultQueryRange QueryRange `json:"result_query_range"`
+	Issue            string     `json:"issue"`
 }
 
-type Row struct {
+type Metric struct {
 	Metric     map[string]string `json:"metric"`
 	Values     []float64         `json:"values"`
 	Timestamps []int64           `json:"timestamps"`
 }
 
+func (r *Metric) UnmarshalJSON(b []byte) error {
+	type plain Metric
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type Series struct {
+	Status string              `json:"status"`
+	Data   []map[string]string `json:"data"`
+}
+type Query struct {
+	Status string    `json:"status"`
+	Data   QueryData `json:"data"`
+}
+type QueryData struct {
+	ResultType string            `json:"resultType"`
+	Result     []QueryDataResult `json:"result"`
+}
+
+type QueryDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Value  []interface{}     `json:"value"`
+}
+
+func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type QueryRange struct {
+	Status string         `json:"status"`
+	Data   QueryRangeData `json:"data"`
+}
+type QueryRangeData struct {
+	ResultType string                 `json:"resultType"`
+	Result     []QueryRangeDataResult `json:"result"`
+}
+
+type QueryRangeDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]interface{}   `json:"values"`
+}
+
+func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryRangeDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
 func TestMain(m *testing.M) {
 	setUp()
 	code := m.Run()
@@ -92,7 +149,7 @@ func setUp() {
 
 func processFlags() {
 	flag.Parse()
-	for _, fs := range []struct {
+	for _, fv := range []struct {
 		flag  string
 		value string
 	}{
@@ -101,10 +158,11 @@ func processFlags() {
 		{flag: "graphiteListenAddr", value: testStatsDListenAddr},
 		{flag: "opentsdbListenAddr", value: testOpenTSDBListenAddr},
 		{flag: "loggerLevel", value: testLogLevel},
+		{flag: "opentsdbHTTPListenAddr", value: testOpenTSDBHTTPListenAddr},
 	} {
 		// panics if flag doesn't exist
-		if err := flag.Lookup(fs.flag).Value.Set(fs.value); err != nil {
-			log.Fatalf("unable to set %q with value %q, err: %v", fs.flag, fs.value, err)
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
 		}
 	}
 }
@@ -121,67 +179,125 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		log.Printf("cannot stop the webservice: %s", err)
+	}
 	vminsert.Stop()
 	vmstorage.Stop()
 	vmselect.Stop()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		log.Fatalf("cannot stop the webservice: %s", err)
-	}
-	os.RemoveAll(storagePath)
+	fs.MustRemoveAll(storagePath)
 }
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
 	time.Sleep(1 * time.Second)
 	vmstorage.Stop()
-
 	// open storage after stop in write
 	vmstorage.InitWithoutMetrics()
 	t.Run("read", testRead)
 }
 
 func testWrite(t *testing.T) {
+	t.Run("prometheus", func(t *testing.T) {
+		for _, test := range readIn("prometheus", t, insertionTime) {
+			s := newSuite(t)
+			r := testutil.WriteRequest{}
+			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
+			data, err := testutil.Compress(r)
+			s.greaterThan(len(r.Timeseries), 0)
+			if err != nil {
+				t.Errorf("error compressing %v %s", r, err)
+				t.Fail()
+			}
+			httpWrite(t, testPromWriteHTTPPath, bytes.NewBuffer(data))
+		}
+	})
+
 	t.Run("influxdb", func(t *testing.T) {
-		for _, test := range readIn("influxdb", t, fmt.Sprintf("%d", insertionTime.UnixNano())) {
+		for _, x := range readIn("influxdb", t, insertionTime) {
+			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
-				httpWrite(t, testWriteHTTPPath, test.Data)
+				httpWrite(t, testWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
 			})
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, test := range readIn("graphite", t, fmt.Sprintf("%d", insertionTime.Unix())) {
+		for _, x := range readIn("graphite", t, insertionTime) {
+			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
-				tcpWrite(t, "127.0.0.1"+testStatsDListenAddr, test.Data)
+				tcpWrite(t, "127.0.0.1"+testStatsDListenAddr, strings.Join(test.Data, "\n"))
 			})
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, test := range readIn("opentsdb", t, fmt.Sprintf("%d", insertionTime.Unix())) {
+		for _, x := range readIn("opentsdb", t, insertionTime) {
+			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
-				tcpWrite(t, "127.0.0.1"+testOpenTSDBListenAddr, test.Data)
+				tcpWrite(t, "127.0.0.1"+testOpenTSDBListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdbhttp", func(t *testing.T) {
+		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				logger.Infof("writing %s", test.Data)
+				httpWrite(t, testOpenTSDBWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
 			})
 		}
 	})
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"graphite", "opentsdb", "influxdb"} {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, test := range readIn(engine, t, fmt.Sprintf("%d", insertionTime.UnixNano())) {
-				test := test
+			for _, x := range readIn(engine, t, insertionTime) {
+				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
-					rowContains(t, httpRead(t, testReadHTTPPath, test.Query), test.Result)
+					for _, q := range test.Query {
+						q = testutil.PopulateTimeTplString(q, insertionTime)
+						if test.Issue != "" {
+							test.Issue = "Regression in " + test.Issue
+						}
+						switch true {
+						case strings.HasPrefix(q, "/api/v1/export"):
+							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
+								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/series"):
+							s := Series{}
+							httpReadStruct(t, testReadHTTPPath, q, &s)
+							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
+								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query_range"):
+							queryResult := QueryRange{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
+								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query"):
+							queryResult := Query{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
+								t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						default:
+							t.Fatalf("unsupported read query %s", q)
+						}
+					}
 				})
 			}
 		})
 	}
 }
 
-func readIn(readFor string, t *testing.T, timeStr string) []test {
+func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	t.Helper()
 	s := newSuite(t)
 	var tt []test
@@ -193,7 +309,9 @@ func readIn(readFor string, t *testing.T, timeStr string) []test {
 		s.noError(err)
 		item := test{}
 		s.noError(json.Unmarshal(b, &item))
-		item.Data = strings.Replace(item.Data, "{TIME}", timeStr, 1)
+		for i := range item.Data {
+			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
+		}
 		tt = append(tt, item)
 		return nil
 	}))
@@ -203,10 +321,10 @@ func readIn(readFor string, t *testing.T, timeStr string) []test {
 	return tt
 }
 
-func httpWrite(t *testing.T, address string, data string) {
+func httpWrite(t *testing.T, address string, r io.Reader) {
 	t.Helper()
 	s := newSuite(t)
-	resp, err := http.Post(address, "", bytes.NewBufferString(data))
+	resp, err := http.Post(address, "", r)
 	s.noError(err)
 	s.noError(resp.Body.Close())
 	s.equalInt(resp.StatusCode, 204)
@@ -223,35 +341,122 @@ func tcpWrite(t *testing.T, address string, data string) {
 	s.equalInt(n, len(data))
 }
 
-func httpRead(t *testing.T, address, query string) []Row {
+func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
 	defer resp.Body.Close()
 	s.equalInt(resp.StatusCode, 200)
-	var rows []Row
+	var rows []Metric
 	for dec := json.NewDecoder(resp.Body); dec.More(); {
-		var row Row
+		var row Metric
 		s.noError(dec.Decode(&row))
 		rows = append(rows, row)
 	}
 	return rows
 }
-
-func rowContains(t *testing.T, rows, contains []Row) {
+func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
-	for _, r := range rows {
-		contains = removeIfFound(r, contains)
-	}
-	if len(contains) > 0 {
-		t.Fatalf("result rows %+v not found in %+v", contains, rows)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer resp.Body.Close()
+	s.equalInt(resp.StatusCode, 200)
+	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
-func removeIfFound(r Row, contains []Row) []Row {
+func checkMetricsResult(got, want []Metric) error {
+	for _, r := range append([]Metric(nil), got...) {
+		want = removeIfFoundMetrics(r, want)
+	}
+	if len(want) > 0 {
+		return fmt.Errorf("exptected metrics %+v not found in %+v", want, got)
+	}
+	return nil
+}
+
+func removeIfFoundMetrics(r Metric, contains []Metric) []Metric {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) &&
+			reflect.DeepEqual(r.Timestamps, item.Timestamps) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkSeriesResult(got, want Series) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	wantData := append([]map[string]string(nil), want.Data...)
+	for _, r := range got.Data {
+		wantData = removeIfFoundSeries(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected seria(s) %+v not found in %+v", wantData, got.Data)
+	}
+	return nil
+}
+
+func removeIfFoundSeries(r map[string]string, contains []map[string]string) []map[string]string {
+	for i, item := range contains {
+		if reflect.DeepEqual(r, item) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryResult(got, want Query) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryRangeResult(got, want QueryRange) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryRangeData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
 	for i, item := range contains {
-		// todo check time
 		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
 			contains[i] = contains[len(contains)-1]
 			return contains[:len(contains)-1]
@@ -279,3 +484,11 @@ func (s *suite) equalInt(a, b int) {
 		s.t.FailNow()
 	}
 }
+
+func (s *suite) greaterThan(a, b int) {
+	s.t.Helper()
+	if a <= b {
+		s.t.Errorf("%d less or equal then %d", a, b)
+		s.t.FailNow()
+	}
+}

app/victoria-metrics/test/parser.go

@@ -0,0 +1,52 @@
+package test
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var (
+	parseTimeExpRegex = regexp.MustCompile(`"?{TIME[^}]*}"?`)
+	extractRegex      = regexp.MustCompile(`"?{([^}]*)}"?`)
+)
+
+// PopulateTimeTplString substitutes {TIME_*} with t in s and returns the result.
+func PopulateTimeTplString(s string, t time.Time) string {
+	return string(PopulateTimeTpl([]byte(s), t))
+}
+
+// PopulateTimeTpl substitutes {TIME_*} with tGlobal in b and returns the result.
+func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
+	return parseTimeExpRegex.ReplaceAllFunc(b, func(repl []byte) []byte {
+		t := tGlobal
+		repl = extractRegex.FindSubmatch(repl)[1]
+		parts := strings.SplitN(string(repl), "-", 2)
+		if len(parts) == 2 {
+			duration, err := time.ParseDuration(strings.TrimSpace(parts[1]))
+			if err != nil {
+				log.Fatalf("error %s parsing duration %s in %s", err, parts[1], repl)
+			}
+			t = t.Add(-duration)
+		}
+		switch strings.TrimSpace(parts[0]) {
+		case `TIME_S`:
+			return []byte(fmt.Sprintf("%d", t.Unix()))
+		case `TIME_MSZ`:
+			return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
+		case `TIME_MS`:
+			return []byte(fmt.Sprintf("%d", timeToMillis(t)))
+		case `TIME_NS`:
+			return []byte(fmt.Sprintf("%d", t.UnixNano()))
+		default:
+			log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
+		}
+		return repl
+	})
+}
+
+func timeToMillis(t time.Time) int64 {
+	return t.UnixNano() / 1e6
+}

app/victoria-metrics/test/parser_test.go

@@ -0,0 +1,24 @@
+package test
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPopulateTimeTplString(t *testing.T) {
+	now, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+	if err != nil {
+		t.Fatalf("unexpected error when parsing time: %s", err)
+	}
+	f := func(s, resultExpected string) {
+		t.Helper()
+		result := PopulateTimeTplString(s, now)
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
+		}
+	}
+	f("", "")
+	f("{TIME_S}", "1136214245")
+	f("now: {TIME_S}, past 30s: {TIME_MS-30s}, now: {TIME_S}", "now: 1136214245, past 30s: 1136214215000, now: 1136214245")
+	f("now: {TIME_MS}, past 30m: {TIME_MSZ-30m}, past 2h: {TIME_NS-2h}", "now: 1136214245000, past 30m: 1136212445000, past 2h: 1136207045000000000")
+}

app/victoria-metrics/test/prom_types.go

@@ -0,0 +1,338 @@
+// +build integration
+
+// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
+package test
+
+import (
+	"encoding/binary"
+	"math"
+	"math/bits"
+)
+
+type WriteRequest struct {
+	Timeseries []TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"`
+}
+
+func (m *WriteRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Timeseries) > 0 {
+		for _, e := range m.Timeseries {
+			l = e.Size()
+			n += 1 + l + sovRemote(uint64(l))
+		}
+	}
+	return n
+}
+func sovRemote(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}
+
+func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Timeseries) > 0 {
+		for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Timeseries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintRemote(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
+	offset -= sovRemote(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+type Sample struct {
+	Value     float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"`
+	Timestamp int64   `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+}
+
+func (m *Sample) Reset() { *m = Sample{} }
+
+// TimeSeries represents samples and labels for a single time series.
+type TimeSeries struct {
+	Labels  []Label  `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+	Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"`
+}
+
+func (m *TimeSeries) Reset() { *m = TimeSeries{} }
+
+type Label struct {
+	Name  string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (m *Label) Reset() { *m = Label{} }
+
+type Labels struct {
+	Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+}
+
+func (m *Labels) Reset() { *m = Labels{} }
+
+func (m *Sample) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.Timestamp != 0 {
+		i = encodeVarintTypes(dAtA, i, uint64(m.Timestamp))
+		i--
+		dAtA[i] = 0x10
+	}
+	if m.Value != 0 {
+		i -= 8
+		binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value))))
+		i--
+		dAtA[i] = 0x9
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *TimeSeries) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Samples) > 0 {
+		for iNdEx := len(m.Samples) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Samples[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *Label) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Label) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Value) > 0 {
+		i -= len(m.Value)
+		copy(dAtA[i:], m.Value)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Value)))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.Name) > 0 {
+		i -= len(m.Name)
+		copy(dAtA[i:], m.Name)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Name)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *Labels) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Labels) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Labels) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
+	offset -= sovTypes(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+func (m *Sample) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	if m.Value != 0 {
+		n += 9
+	}
+	if m.Timestamp != 0 {
+		n += 1 + sovTypes(uint64(m.Timestamp))
+	}
+	return n
+}
+
+func (m *TimeSeries) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	if len(m.Samples) > 0 {
+		for _, e := range m.Samples {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *Label) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	l = len(m.Value)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+func (m *Labels) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+func sovTypes(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}

app/victoria-metrics/test/prom_writter.go

@@ -0,0 +1,13 @@
+// +build integration
+
+package test
+
+import "github.com/golang/snappy"
+
+func Compress(wr WriteRequest) ([]byte, error) {
+	data, err := wr.Marshal()
+	if err != nil {
+		return nil, err
+	}
+	return snappy.Encode(nil, data), nil
+}

app/victoria-metrics/testdata/graphite/basic.json

@@ -1,8 +1,8 @@
 {
   "name": "basic_insertion",
-  "data": "graphite.foo.bar.baz;tag1=value1;tag2=value2 123 {TIME}",
-  "query": "/api/v1/export?match={__name__!=\"\"}",
-  "result": [
-    {"metric":{"__name__":"graphite.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123]}
+  "data": ["graphite.foo.bar.baz;tag1=value1;tag2=value2 123 {TIME_S}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"graphite.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
   ]
 }

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -0,0 +1,16 @@
+{
+  "name": "comparison-not-inf-not-nan",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150",
+  "data": [
+    "not_nan_not_inf;item=x 1 {TIME_S-1m}",
+    "not_nan_not_inf;item=x 1 {TIME_S-2m}",
+    "not_nan_not_inf;item=y 3 {TIME_S-1m}",
+    "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
+  "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"],["{TIME_S}","0.5"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -0,0 +1,24 @@
+{
+  "name": "max_lookback_set",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_set 1 {TIME_S-30s}",
+    "max_lookback_set 2 {TIME_S-60s}",
+    "max_lookback_set 3 {TIME_S-120s}",
+    "max_lookback_set 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_set"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-140s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-110s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-50s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -0,0 +1,32 @@
+{
+  "name": "max_lookback_unset",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_unset 1 {TIME_S-30s}",
+    "max_lookback_unset 2 {TIME_S-60s}",
+    "max_lookback_unset 3 {TIME_S-120s}",
+    "max_lookback_unset 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-140s}","4"],
+	      ["{TIME_S-130s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-110s}","3"],
+	      ["{TIME_S-100s}","3"],
+	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
+	      ["{TIME_S-70s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-50s}","2"],
+	      ["{TIME_S-40s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -0,0 +1,18 @@
+{
+  "name": "not-nan-as-missing-data",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153",
+  "data": [
+    "not_nan_as_missing_data;item=x 2 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=x 1 {TIME_S-1m}",
+    "not_nan_as_missing_data;item=y 4 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
+  ],
+  "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}","3"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -0,0 +1,14 @@
+{
+  "name": "subquery-aggregation",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/184",
+  "data": [
+    "forms_daily_count;item=x 1 {TIME_S-1m}",
+    "forms_daily_count;item=x 2 {TIME_S-2m}",
+    "forms_daily_count;item=y 3 {TIME_S-1m}",
+    "forms_daily_count;item=y 4 {TIME_S-2m}"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","1"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","3"]}]}
+  }
+}

app/victoria-metrics/testdata/influxdb/basic.json

@@ -1,9 +1,9 @@
 {
   "name": "basic_insertion",
-  "data": "measurement,tag1=value1,tag2=value2 field1=1.23,field2=123",
-  "query": "/api/v1/export?match={__name__!=\"\"}",
-  "result": [
-    {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[123]},
-    {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[1.23]}
+  "data": ["measurement,tag1=value1,tag2=value2 field1=1.23,field2=123 {TIME_NS}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[1.23], "timestamps": ["{TIME_MS}"]}
   ]
 }

app/victoria-metrics/testdata/opentsdb/basic.json

@@ -1,8 +1,8 @@
 {
   "name": "basic_insertion",
-  "data": "put openstdb.foo.bar.baz {TIME} 123 tag1=value1 tag2=value2",
-  "query": "/api/v1/export?match={__name__!=\"\"}",
-  "result": [
-    {"metric":{"__name__":"openstdb.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123]}
+  "data": ["put openstdb.foo.bar.baz {TIME_S} 123 tag1=value1 tag2=value2"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"openstdb.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
   ]
 }

app/victoria-metrics/testdata/opentsdbhttp/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["{\"metric\": \"opentsdbhttp.foo\", \"value\": 1001, \"timestamp\": {TIME_S}, \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.foo","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/multi_line.json

@@ -0,0 +1,9 @@
+{
+  "name": "multiline",
+  "data": ["[{\"metric\": \"opentsdbhttp.multiline1\", \"value\": 1001, \"timestamp\": \"{TIME_S}\", \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}, {\"metric\": \"opentsdbhttp.multiline2\", \"value\": 1002, \"timestamp\": {TIME_S}}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.multiline1","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]},
+    {"metric":{"__name__":"opentsdbhttp.multiline2"},"values":[1002], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.bar\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.bar","baz":"qux"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/case-sensitive-regex.json

@@ -0,0 +1,10 @@
+{
+  "name": "case-sensitive-regex",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"sensitiveRegex\"}],\"samples\":[{\"value\":2,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"SensitiveRegex\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={label=~'(?i)sensitiveregex'}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"sensitiveRegex"},"values":[2], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"SensitiveRegex"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/duplicate-label.json

@@ -0,0 +1,9 @@
+{
+  "name": "duplicate_label",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.duplicate_label\"},{\"name\":\"duplicate\",\"value\":\"label\"},{\"name\":\"duplicate\",\"value\":\"label\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.duplicate_label","duplicate":"label"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/match-series.json

@@ -0,0 +1,15 @@
+{
+  "name": "match_series",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"1\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"2\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"3\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"4\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/series?match[]={__name__='MatchSeries'}", "/api/v1/series?match[]={__name__=~'MatchSeries.*'}"],
+  "result_series": {
+    "status": "success",
+    "data": [
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"1","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"2","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"3","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"4","TurbineType":"V112"}
+    ]
+  }
+}

app/vmbackup/Makefile

@@ -0,0 +1,37 @@
+# All these commands must run from repository root.
+
+vmbackup:
+	APP_NAME=vmbackup $(MAKE) app-local
+
+vmbackup-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker
+
+package-vmbackup:
+	APP_NAME=vmbackup $(MAKE) package-via-docker
+
+publish-vmbackup:
+	APP_NAME=vmbackup $(MAKE) publish-via-docker
+
+vmbackup-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+
+vmbackup-arm-prod:
+	APP_NAME=vmbackup APP_SUFFIX='-arm' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm' $(MAKE) app-via-docker
+
+vmbackup-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+
+vmbackup-arm64-prod:
+	APP_NAME=vmbackup APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker
+
+vmbackup-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup
+
+vmbackup-386-prod:
+	APP_NAME=vmbackup APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure
+
+vmbackup-pure-prod:
+	APP_NAME=vmbackup APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker

app/vmbackup/README.md

@@ -0,0 +1,171 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-concurrency` to 1. This should reduce network bandwidth usage.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.

app/vmbackup/deployment/Dockerfile

@@ -0,0 +1,5 @@
+FROM scratch
+COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY bin/vmbackup-prod .
+EXPOSE 8428
+ENTRYPOINT ["/vmbackup-prod"]

app/vmbackup/main.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName    = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	dst             = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+	origin      = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
+	concurrency = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
+)
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	originFS, err := newOriginFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Backup{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+		Origin:      originFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+or local filesystem. Backed up data can be restored with vmrestore.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newSrcFS() (*fslocal.FS, error) {
+	if len(*snapshotName) == 0 {
+		return nil, fmt.Errorf("`-snapshotName` cannot be empty")
+	}
+	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
+
+	// Verify the snapshot exists.
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
+	}
+
+	fs := &fslocal.FS{
+		Dir: snapshotPath,
+	}
+	return fs, nil
+}
+
+func newDstFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*dst)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+	}
+	return fs, nil
+}
+
+func newOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, nil
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+	}
+	return fs, nil
+}

app/vminsert/common/gzip_reader.go

@@ -0,0 +1,30 @@
+package common
+
+import (
+	"compress/gzip"
+	"io"
+	"sync"
+)
+
+// GetGzipReader returns new gzip reader from the pool.
+//
+// Return back the gzip reader when it no longer needed with PutGzipReader.
+func GetGzipReader(r io.Reader) (*gzip.Reader, error) {
+	v := gzipReaderPool.Get()
+	if v == nil {
+		return gzip.NewReader(r)
+	}
+	zr := v.(*gzip.Reader)
+	if err := zr.Reset(r); err != nil {
+		return nil, err
+	}
+	return zr, nil
+}
+
+// PutGzipReader returns back gzip reader obtained via GetGzipReader.
+func PutGzipReader(zr *gzip.Reader) {
+	_ = zr.Close()
+	gzipReaderPool.Put(zr)
+}
+
+var gzipReaderPool sync.Pool

app/vminsert/common/insert_ctx.go

@@ -2,9 +2,11 @@ package common
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -99,7 +101,10 @@ func (ctx *InsertCtx) AddLabel(name, value string) {
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
 	if err := vmstorage.AddRows(ctx.mrs); err != nil {
-		return fmt.Errorf("cannot store metrics: %s", err)
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store metrics: %s", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
 	}
 	return nil
 }

app/vminsert/concurrencylimiter/concurrencylimiter.go

@@ -3,9 +3,11 @@ package concurrencylimiter
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"runtime"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -53,7 +55,10 @@ func Do(f func() error) error {
 	case <-t.C:
 		timerpool.Put(t)
 		concurrencyLimitTimeout.Inc()
-		return fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch))
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch)),
+			StatusCode: http.StatusServiceUnavailable,
+		}
 	}
 }
 

app/vminsert/graphite/parser.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson/fastfloat"
 )
 
@@ -34,13 +36,8 @@ func (rs *Rows) Reset() {
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 //
 // s must be unchanged until rs is in use.
-func (rs *Rows) Unmarshal(s string) error {
-	var err error
-	rs.Rows, rs.tagsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0])
-	if err != nil {
-		return err
-	}
-	return err
+func (rs *Rows) Unmarshal(s string) {
+	rs.Rows, rs.tagsPool = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0])
 }
 
 // Row is a single graphite row.
@@ -83,6 +80,9 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 		tags := tagsPool[tagsStart:]
 		r.Tags = tags[:len(tags):len(tags)]
 	}
+	if len(r.Metric) == 0 {
+		return tagsPool, fmt.Errorf("metric cannot be empty")
+	}
 
 	n = strings.IndexByte(tail, ' ')
 	if n < 0 {
@@ -95,41 +95,46 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 	return tagsPool, nil
 }
 
-func unmarshalRows(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag, error) {
+func unmarshalRows(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag) {
 	for len(s) > 0 {
 		n := strings.IndexByte(s, '\n')
-		if n == 0 {
-			// Skip empty line
-			s = s[1:]
-			continue
-		}
-		if cap(dst) > len(dst) {
-			dst = dst[:len(dst)+1]
-		} else {
-			dst = append(dst, Row{})
-		}
-		r := &dst[len(dst)-1]
 		if n < 0 {
 			// The last line.
-			var err error
-			tagsPool, err = r.unmarshal(s, tagsPool)
-			if err != nil {
-				err = fmt.Errorf("cannot unmarshal Graphite line %q: %s", s, err)
-				return dst, tagsPool, err
-			}
-			return dst, tagsPool, nil
-		}
-		var err error
-		tagsPool, err = r.unmarshal(s[:n], tagsPool)
-		if err != nil {
-			err = fmt.Errorf("cannot unmarshal Graphite line %q: %s", s[:n], err)
-			return dst, tagsPool, err
+			return unmarshalRow(dst, s, tagsPool)
 		}
+		dst, tagsPool = unmarshalRow(dst, s[:n], tagsPool)
 		s = s[n+1:]
 	}
-	return dst, tagsPool, nil
+	return dst, tagsPool
 }
 
+func unmarshalRow(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag) {
+	if len(s) > 0 && s[len(s)-1] == '\r' {
+		s = s[:len(s)-1]
+	}
+	if len(s) == 0 {
+		// Skip empty line
+		return dst, tagsPool
+	}
+
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	var err error
+	tagsPool, err = r.unmarshal(s, tagsPool)
+	if err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal Graphite line %q: %s", s, err)
+		invalidLines.Inc()
+	}
+	return dst, tagsPool
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="graphite"}`)
+
 func unmarshalTags(dst []Tag, s string) ([]Tag, error) {
 	for {
 		if cap(dst) > len(dst) {
@@ -145,12 +150,20 @@ func unmarshalTags(dst []Tag, s string) ([]Tag, error) {
 			if err := tag.unmarshal(s); err != nil {
 				return dst[:len(dst)-1], err
 			}
+			if len(tag.Key) == 0 || len(tag.Value) == 0 {
+				// Skip empty tag
+				dst = dst[:len(dst)-1]
+			}
 			return dst, nil
 		}
 		if err := tag.unmarshal(s[:n]); err != nil {
 			return dst[:len(dst)-1], err
 		}
 		s = s[n+1:]
+		if len(tag.Key) == 0 || len(tag.Value) == 0 {
+			// Skip empty tag
+			dst = dst[:len(dst)-1]
+		}
 	}
 }
 
@@ -172,9 +185,6 @@ func (t *Tag) unmarshal(s string) error {
 		return fmt.Errorf("missing tag value for %q", s)
 	}
 	t.Key = s[:n]
-	if len(t.Key) == 0 {
-		return fmt.Errorf("tag key cannot be empty for %q", s)
-	}
 	t.Value = s[n+1:]
 	return nil
 }

app/vminsert/graphite/parser_test.go

@@ -9,45 +9,42 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
 		}
 
 		// Try again
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
 		}
 	}
 
+	// Missing metric
+	f(" 123 455")
+
 	// Missing value
 	f("aaa")
 
-	// Invalid multiline
-	f("aaa\nbbb 123 34")
-
 	// missing tag
 	f("aa; 12 34")
 
 	// missing tag value
 	f("aa;bb 23 34")
-	f("aa;=dsd 234 45")
 }
 
 func TestRowsUnmarshalSuccess(t *testing.T) {
 	f := func(s string, rowsExpected *Rows) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
 
 		// Try unmarshaling again
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
@@ -60,7 +57,9 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 
 	// Empty line
 	f("", &Rows{})
+	f("\r", &Rows{})
 	f("\n\n", &Rows{})
+	f("\n\r\n", &Rows{})
 
 	// Single line
 	f("foobar -123.456 789", &Rows{
@@ -86,6 +85,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		}},
 	})
 
+	// Timestamp bigger than 1<<31
+	f("aaa 1123 429496729600", &Rows{
+		Rows: []Row{{
+			Metric:    "aaa",
+			Value:     1123,
+			Timestamp: 429496729600,
+		}},
+	})
+
 	// Tags
 	f("foo;bar=baz 1 2", &Rows{
 		Rows: []Row{{
@@ -98,7 +106,8 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			Timestamp: 2,
 		}},
 	})
-	f("foo;bar=baz;aa=;x=y 1 2", &Rows{
+	// Empty tags
+	f("foo;bar=baz;aa=;x=y;=z 1 2", &Rows{
 		Rows: []Row{{
 			Metric: "foo",
 			Tags: []Tag{
@@ -106,10 +115,6 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 					Key:   "bar",
 					Value: "baz",
 				},
-				{
-					Key:   "aa",
-					Value: "",
-				},
 				{
 					Key:   "x",
 					Value: "y",
@@ -139,4 +144,20 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			},
 		},
 	})
+
+	// Multi lines with invalid line
+	f("foo 0.3 2\naaa\nbar.baz 0.34 43\n", &Rows{
+		Rows: []Row{
+			{
+				Metric:    "foo",
+				Value:     0.3,
+				Timestamp: 2,
+			},
+			{
+				Metric:    "bar.baz",
+				Value:     0.34,
+				Timestamp: 43,
+			},
+		},
+	})
 }

app/vminsert/graphite/parser_timing_test.go

@@ -16,8 +16,9 @@ cpu.usage_irq 0.34432 1234556768
 	b.RunParallel(func(pb *testing.PB) {
 		var rows Rows
 		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
+			rows.Unmarshal(s)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of rows unmarshaled: got %d; want 4", len(rows.Rows)))
 			}
 		}
 	})

app/vminsert/graphite/request_handler.go

@@ -85,11 +85,7 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 			return false
 		}
 	}
-	if err := ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf)); err != nil {
-		graphiteUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal graphite plaintext protocol data with size %d: %s", len(ctx.reqBuf), err)
-		return false
-	}
+	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
 
 	// Fill missing timestamps with the current timestamp rounded to seconds.
 	currentTimestamp := time.Now().Unix()
@@ -136,9 +132,8 @@ func (ctx *pushCtx) reset() {
 }
 
 var (
-	graphiteReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
-	graphiteReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
-	graphiteUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="graphite"}`)
+	graphiteReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
+	graphiteReadErrors = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
 )
 
 func getPushCtx() *pushCtx {

app/vminsert/influx/parser.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson/fastfloat"
 )
 
@@ -41,13 +43,8 @@ func (rs *Rows) Reset() {
 // See https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/
 //
 // s must be unchanged until rs is in use.
-func (rs *Rows) Unmarshal(s string) error {
-	var err error
-	rs.Rows, rs.tagsPool, rs.fieldsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0], rs.fieldsPool[:0])
-	if err != nil {
-		return err
-	}
-	return err
+func (rs *Rows) Unmarshal(s string) {
+	rs.Rows, rs.tagsPool, rs.fieldsPool = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0], rs.fieldsPool[:0])
 }
 
 // Row is a single influx row.
@@ -65,9 +62,8 @@ func (r *Row) reset() {
 	r.Timestamp = 0
 }
 
-func (r *Row) unmarshal(s string, tagsPool []Tag, fieldsPool []Field) ([]Tag, []Field, error) {
+func (r *Row) unmarshal(s string, tagsPool []Tag, fieldsPool []Field, noEscapeChars bool) ([]Tag, []Field, error) {
 	r.reset()
-	noEscapeChars := strings.IndexByte(s, '\\') < 0
 	n := nextUnescapedChar(s, ' ', noEscapeChars)
 	if n < 0 {
 		return tagsPool, fieldsPool, fmt.Errorf("cannot find Whitespace I in %q", s)
@@ -141,9 +137,6 @@ func (tag *Tag) unmarshal(s string, noEscapeChars bool) error {
 		return fmt.Errorf("missing tag value for %q", s)
 	}
 	tag.Key = unescapeTagValue(s[:n], noEscapeChars)
-	if len(tag.Key) == 0 {
-		return fmt.Errorf("tag key cannot be empty")
-	}
 	tag.Value = unescapeTagValue(s[n+1:], noEscapeChars)
 	return nil
 }
@@ -177,41 +170,51 @@ func (f *Field) unmarshal(s string, noEscapeChars, hasQuotedFields bool) error {
 	return nil
 }
 
-func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]Row, []Tag, []Field, error) {
+func unmarshalRows(dst []Row, s string, tagsPool []Tag, fieldsPool []Field) ([]Row, []Tag, []Field) {
+	noEscapeChars := strings.IndexByte(s, '\\') < 0
 	for len(s) > 0 {
 		n := strings.IndexByte(s, '\n')
-		if n == 0 {
-			// Skip empty line
-			s = s[1:]
-			continue
-		}
-		if cap(dst) > len(dst) {
-			dst = dst[:len(dst)+1]
-		} else {
-			dst = append(dst, Row{})
-		}
-		r := &dst[len(dst)-1]
 		if n < 0 {
 			// The last line.
-			var err error
-			tagsPool, fieldsPool, err = r.unmarshal(s, tagsPool, fieldsPool)
-			if err != nil {
-				err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s, err)
-				return dst, tagsPool, fieldsPool, err
-			}
-			return dst, tagsPool, fieldsPool, nil
-		}
-		var err error
-		tagsPool, fieldsPool, err = r.unmarshal(s[:n], tagsPool, fieldsPool)
-		if err != nil {
-			err = fmt.Errorf("cannot unmarshal Influx line %q: %s", s[:n], err)
-			return dst, tagsPool, fieldsPool, err
+			return unmarshalRow(dst, s, tagsPool, fieldsPool, noEscapeChars)
 		}
+		dst, tagsPool, fieldsPool = unmarshalRow(dst, s[:n], tagsPool, fieldsPool, noEscapeChars)
 		s = s[n+1:]
 	}
-	return dst, tagsPool, fieldsPool, nil
+	return dst, tagsPool, fieldsPool
 }
 
+func unmarshalRow(dst []Row, s string, tagsPool []Tag, fieldsPool []Field, noEscapeChars bool) ([]Row, []Tag, []Field) {
+	if len(s) > 0 && s[len(s)-1] == '\r' {
+		s = s[:len(s)-1]
+	}
+	if len(s) == 0 {
+		// Skip empty line
+		return dst, tagsPool, fieldsPool
+	}
+	if s[0] == '#' {
+		// Skip comment
+		return dst, tagsPool, fieldsPool
+	}
+
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	var err error
+	tagsPool, fieldsPool, err = r.unmarshal(s, tagsPool, fieldsPool, noEscapeChars)
+	if err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal Influx line %q: %s; skipping it", s, err)
+		invalidLines.Inc()
+	}
+	return dst, tagsPool, fieldsPool
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="influx"}`)
+
 func unmarshalTags(dst []Tag, s string, noEscapeChars bool) ([]Tag, error) {
 	for {
 		if cap(dst) > len(dst) {
@@ -223,14 +226,22 @@ func unmarshalTags(dst []Tag, s string, noEscapeChars bool) ([]Tag, error) {
 		n := nextUnescapedChar(s, ',', noEscapeChars)
 		if n < 0 {
 			if err := tag.unmarshal(s, noEscapeChars); err != nil {
-				return dst, err
+				return dst[:len(dst)-1], err
+			}
+			if len(tag.Key) == 0 || len(tag.Value) == 0 {
+				// Skip empty tag
+				dst = dst[:len(dst)-1]
 			}
 			return dst, nil
 		}
 		if err := tag.unmarshal(s[:n], noEscapeChars); err != nil {
-			return dst, err
+			return dst[:len(dst)-1], err
 		}
 		s = s[n+1:]
+		if len(tag.Key) == 0 || len(tag.Value) == 0 {
+			// Skip empty tag
+			dst = dst[:len(dst)-1]
+		}
 	}
 }
 

app/vminsert/influx/parser_test.go

@@ -74,13 +74,15 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
 		}
 
 		// Try again
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
 		}
 	}
 
@@ -94,12 +96,8 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	// Missing tag value
 	f("foo,bar")
 	f("foo,bar baz")
-	f("foo,bar= baz")
 	f("foo,bar=123, 123")
 
-	// Missing tag name
-	f("foo,=bar baz=234")
-
 	// Missing field value
 	f("foo bar")
 	f("foo bar=")
@@ -122,17 +120,13 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 	f := func(s string, rowsExpected *Rows) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
 
 		// Try unmarshaling again
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
@@ -146,6 +140,12 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 	// Empty line
 	f("", &Rows{})
 	f("\n\n", &Rows{})
+	f("\n\r\n", &Rows{})
+
+	// Comment
+	f("\n# foobar\n", &Rows{})
+	f("#foobar baz", &Rows{})
+	f("#foobar baz\n#sss", &Rows{})
 
 	// Minimal line without tags and timestamp
 	f("foo bar=123", &Rows{
@@ -157,6 +157,15 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			}},
 		}},
 	})
+	f("# comment\nfoo bar=123\r\n#comment2 sdsf dsf", &Rows{
+		Rows: []Row{{
+			Measurement: "foo",
+			Fields: []Field{{
+				Key:   "bar",
+				Value: 123,
+			}},
+		}},
+	})
 	f("foo bar=123\n", &Rows{
 		Rows: []Row{{
 			Measurement: "foo",
@@ -216,7 +225,7 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 	})
 
 	// Line with empty tag values
-	f("foo,tag1=xyz,tagN=,tag2=43as bar=123", &Rows{
+	f("foo,tag1=xyz,tagN=,tag2=43as,=xxx bar=123", &Rows{
 		Rows: []Row{{
 			Measurement: "foo",
 			Tags: []Tag{
@@ -224,10 +233,6 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 					Key:   "tag1",
 					Value: "xyz",
 				},
-				{
-					Key:   "tagN",
-					Value: "",
-				},
 				{
 					Key:   "tag2",
 					Value: "43as",
@@ -309,11 +314,11 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 	})
 
 	// Escape chars
-	f(`fo\,bar\=baz,x\==\\a\,\=\q\  \\\a\=\,=4.34`, &Rows{
+	f(`fo\,bar\=baz,x\=\b=\\a\,\=\q\  \\\a\=\,=4.34`, &Rows{
 		Rows: []Row{{
 			Measurement: `fo,bar=baz`,
 			Tags: []Tag{{
-				Key:   `x=`,
+				Key:   `x=\b`,
 				Value: `\a,=\q `,
 			}},
 			Fields: []Field{{
@@ -348,6 +353,34 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			},
 		},
 	})
+
+	// Multiple lines with invalid line in the middle.
+	f("foo,tag=xyz field=1.23 48934\n"+
+		"invalid line\n"+
+		"bar x=-1i\n\n", &Rows{
+		Rows: []Row{
+			{
+				Measurement: "foo",
+				Tags: []Tag{{
+					Key:   "tag",
+					Value: "xyz",
+				}},
+				Fields: []Field{{
+					Key:   "field",
+					Value: 1.23,
+				}},
+				Timestamp: 48934,
+			},
+			{
+				Measurement: "bar",
+				Fields: []Field{{
+					Key:   "x",
+					Value: -1,
+				}},
+			},
+		},
+	})
+
 	// No newline after the second line.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/82
 	f("foo,tag=xyz field=1.23 48934\n"+
@@ -374,4 +407,24 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			},
 		},
 	})
+
+	f("x,y=z,g=p:\\ \\ 5432\\,\\ gp\\ mon\\ [lol]\\ con10\\ cmd5\\ SELECT f=1", &Rows{
+		Rows: []Row{{
+			Measurement: "x",
+			Tags: []Tag{
+				{
+					Key:   "y",
+					Value: "z",
+				},
+				{
+					Key:   "g",
+					Value: "p:  5432, gp mon [lol] con10 cmd5 SELECT",
+				},
+			},
+			Fields: []Field{{
+				Key:   "f",
+				Value: 1,
+			}},
+		}},
+	})
 }

app/vminsert/influx/parser_timing_test.go

@@ -6,14 +6,19 @@ import (
 )
 
 func BenchmarkRowsUnmarshal(b *testing.B) {
-	s := `cpu usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 1234556768`
+	s := `cpu usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 1234556768
+cpu usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 123455676344
+aaa usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 123455676344
+bbb usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 123455676344
+`
 	b.SetBytes(int64(len(s)))
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		var rows Rows
 		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
+			rows.Unmarshal(s)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of rows parsed; got %d; want 4", len(rows.Rows)))
 			}
 		}
 	})

app/vminsert/influx/request_handler.go

@@ -1,7 +1,6 @@
 package influx
 
 import (
-	"compress/gzip"
 	"flag"
 	"fmt"
 	"io"
@@ -41,11 +40,11 @@ func insertHandlerInternal(req *http.Request) error {
 
 	r := req.Body
 	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := getGzipReader(r)
+		zr, err := common.GetGzipReader(r)
 		if err != nil {
 			return fmt.Errorf("cannot read gzipped influx line protocol data: %s", err)
 		}
-		defer putGzipReader(zr)
+		defer common.PutGzipReader(zr)
 		r = zr
 	}
 
@@ -91,11 +90,17 @@ func (ctx *pushCtx) InsertRows(db string) error {
 	for i := range rows {
 		r := &rows[i]
 		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("db", db)
+		hasDBLabel := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
 			ic.AddLabel(tag.Key, tag.Value)
 		}
+		if len(db) > 0 && !hasDBLabel {
+			ic.AddLabel("db", db)
+		}
 		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
 		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
 		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
@@ -120,25 +125,6 @@ func (ctx *pushCtx) InsertRows(db string) error {
 	return ic.FlushBufs()
 }
 
-func getGzipReader(r io.Reader) (*gzip.Reader, error) {
-	v := gzipReaderPool.Get()
-	if v == nil {
-		return gzip.NewReader(r)
-	}
-	zr := v.(*gzip.Reader)
-	if err := zr.Reset(r); err != nil {
-		return nil, err
-	}
-	return zr, nil
-}
-
-func putGzipReader(zr *gzip.Reader) {
-	_ = zr.Close()
-	gzipReaderPool.Put(zr)
-}
-
-var gzipReaderPool sync.Pool
-
 func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
 	if ctx.err != nil {
 		return false
@@ -151,11 +137,7 @@ func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
 		}
 		return false
 	}
-	if err := ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf)); err != nil {
-		influxUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal influx line protocol data with size %d: %s", len(ctx.reqBuf), err)
-		return false
-	}
+	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
 
 	// Adjust timestamps according to tsMultiplier
 	currentTs := time.Now().UnixNano() / 1e6
@@ -184,9 +166,8 @@ func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
 }
 
 var (
-	influxReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
-	influxReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
-	influxUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="influx"}`)
+	influxReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
+	influxReadErrors = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
 )
 
 type pushCtx struct {

app/vminsert/main.go

@@ -10,19 +10,25 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	graphiteListenAddr   = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	opentsdbListenAddr   = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty")
-	maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
+	graphiteListenAddr     = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr     = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	maxInsertRequestSize   = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
 )
 
 // Init initializes vminsert.
 func Init() {
+	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+
 	concurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
 		go graphite.Serve(*graphiteListenAddr)
@@ -30,6 +36,9 @@ func Init() {
 	if len(*opentsdbListenAddr) > 0 {
 		go opentsdb.Serve(*opentsdbListenAddr)
 	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		go opentsdbhttp.Serve(*opentsdbHTTPListenAddr, int64(*maxInsertRequestSize))
+	}
 }
 
 // Stop stops vminsert.
@@ -40,6 +49,9 @@ func Stop() {
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdb.Stop()
 	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttp.Stop()
+	}
 }
 
 // RequestHandler is a handler for Prometheus remote storage write API

app/vminsert/opentsdb/parser.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson/fastfloat"
 )
 
@@ -34,13 +36,8 @@ func (rs *Rows) Reset() {
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 //
 // s must be unchanged until rs is in use.
-func (rs *Rows) Unmarshal(s string) error {
-	var err error
-	rs.Rows, rs.tagsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0])
-	if err != nil {
-		return err
-	}
-	return err
+func (rs *Rows) Unmarshal(s string) {
+	rs.Rows, rs.tagsPool = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0])
 }
 
 // Row is a single OpenTSDB row.
@@ -69,6 +66,9 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 		return tagsPool, fmt.Errorf("cannot find whitespace between metric and timestamp in %q", s)
 	}
 	r.Metric = s[:n]
+	if len(r.Metric) == 0 {
+		return tagsPool, fmt.Errorf("metric cannot be empty")
+	}
 	tail := s[n+1:]
 	n = strings.IndexByte(tail, ' ')
 	if n < 0 {
@@ -92,41 +92,46 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 	return tagsPool, nil
 }
 
-func unmarshalRows(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag, error) {
+func unmarshalRows(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag) {
 	for len(s) > 0 {
 		n := strings.IndexByte(s, '\n')
-		if n == 0 {
-			// Skip empty line
-			s = s[1:]
-			continue
-		}
-		if cap(dst) > len(dst) {
-			dst = dst[:len(dst)+1]
-		} else {
-			dst = append(dst, Row{})
-		}
-		r := &dst[len(dst)-1]
 		if n < 0 {
 			// The last line.
-			var err error
-			tagsPool, err = r.unmarshal(s, tagsPool)
-			if err != nil {
-				err = fmt.Errorf("cannot unmarshal OpenTSDB line %q: %s", s, err)
-				return dst, tagsPool, err
-			}
-			return dst, tagsPool, nil
-		}
-		var err error
-		tagsPool, err = r.unmarshal(s[:n], tagsPool)
-		if err != nil {
-			err = fmt.Errorf("cannot unmarshal OpenTSDB line %q: %s", s[:n], err)
-			return dst, tagsPool, err
+			return unmarshalRow(dst, s, tagsPool)
 		}
+		dst, tagsPool = unmarshalRow(dst, s[:n], tagsPool)
 		s = s[n+1:]
 	}
-	return dst, tagsPool, nil
+	return dst, tagsPool
 }
 
+func unmarshalRow(dst []Row, s string, tagsPool []Tag) ([]Row, []Tag) {
+	if len(s) > 0 && s[len(s)-1] == '\r' {
+		s = s[:len(s)-1]
+	}
+	if len(s) == 0 {
+		// Skip empty line
+		return dst, tagsPool
+	}
+
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	var err error
+	tagsPool, err = r.unmarshal(s, tagsPool)
+	if err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal OpenTSDB line %q: %s", s, err)
+		invalidLines.Inc()
+	}
+	return dst, tagsPool
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="opentsdb"}`)
+
 func unmarshalTags(dst []Tag, s string) ([]Tag, error) {
 	for {
 		if cap(dst) > len(dst) {
@@ -142,12 +147,20 @@ func unmarshalTags(dst []Tag, s string) ([]Tag, error) {
 			if err := tag.unmarshal(s); err != nil {
 				return dst[:len(dst)-1], err
 			}
+			if len(tag.Key) == 0 || len(tag.Value) == 0 {
+				// Skip empty tag
+				dst = dst[:len(dst)-1]
+			}
 			return dst, nil
 		}
 		if err := tag.unmarshal(s[:n]); err != nil {
 			return dst[:len(dst)-1], err
 		}
 		s = s[n+1:]
+		if len(tag.Key) == 0 || len(tag.Value) == 0 {
+			// Skip empty tag
+			dst = dst[:len(dst)-1]
+		}
 	}
 }
 
@@ -169,9 +182,6 @@ func (t *Tag) unmarshal(s string) error {
 		return fmt.Errorf("missing tag value for %q", s)
 	}
 	t.Key = s[:n]
-	if len(t.Key) == 0 {
-		return fmt.Errorf("tag key cannot be empty for %q", s)
-	}
 	t.Value = s[n+1:]
 	return nil
 }

app/vminsert/opentsdb/parser_test.go

@@ -9,19 +9,24 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
 		}
 
 		// Try again
-		if err := rows.Unmarshal(s); err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
 		}
 	}
 
 	// Missing put prefix
 	f("xx")
 
+	// Missing metric
+	f("put  111 34")
+
 	// Missing timestamp
 	f("put aaa")
 
@@ -42,26 +47,19 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 
 	// Invalid tag
 	f("put aaa 123 4.5 foo")
-	f("put aaa 123 4.5 =")
-	f("put aaa 123 4.5 =foo")
-	f("put aaa 123 4.5 =foo a=b")
 }
 
 func TestRowsUnmarshalSuccess(t *testing.T) {
 	f := func(s string, rowsExpected *Rows) {
 		t.Helper()
 		var rows Rows
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
 
 		// Try unmarshaling again
-		if err := rows.Unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal %q: %s", s, err)
-		}
+		rows.Unmarshal(s)
 		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
 			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
 		}
@@ -74,7 +72,9 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 
 	// Empty line
 	f("", &Rows{})
+	f("\r", &Rows{})
 	f("\n\n", &Rows{})
+	f("\n\r\n", &Rows{})
 
 	// Single line
 	f("put foobar 789 -123.456 a=b", &Rows{
@@ -88,17 +88,13 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			}},
 		}},
 	})
-	// Empty tag value
-	f("put foobar 789 -123.456 a= b=c", &Rows{
+	// Empty tag
+	f("put foobar 789 -123.456 a= b=c =d", &Rows{
 		Rows: []Row{{
 			Metric:    "foobar",
 			Value:     -123.456,
 			Timestamp: 789,
 			Tags: []Tag{
-				{
-					Key:   "a",
-					Value: "",
-				},
 				{
 					Key:   "b",
 					Value: "c",
@@ -200,4 +196,27 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			},
 		},
 	})
+	// Multi lines with invalid line
+	f("put foo 2 0.3 a=b\naaa bbb\nput bar.baz 43 0.34 a=b\n", &Rows{
+		Rows: []Row{
+			{
+				Metric:    "foo",
+				Value:     0.3,
+				Timestamp: 2,
+				Tags: []Tag{{
+					Key:   "a",
+					Value: "b",
+				}},
+			},
+			{
+				Metric:    "bar.baz",
+				Value:     0.34,
+				Timestamp: 43,
+				Tags: []Tag{{
+					Key:   "a",
+					Value: "b",
+				}},
+			},
+		},
+	})
 }

app/vminsert/opentsdb/parser_timing_test.go

@@ -6,18 +6,19 @@ import (
 )
 
 func BenchmarkRowsUnmarshal(b *testing.B) {
-	s := `cpu.usage_user 1234556768 1.23 a=b
-cpu.usage_system 1234556768 23.344 a=b
-cpu.usage_iowait 1234556769 3.3443 a=b
-cpu.usage_irq 1234556768 0.34432 a=b
+	s := `put cpu.usage_user 1234556768 1.23 a=b
+put cpu.usage_system 1234556768 23.344 a=b
+put cpu.usage_iowait 1234556769 3.3443 a=b
+put cpu.usage_irq 1234556768 0.34432 a=b
 `
 	b.SetBytes(int64(len(s)))
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		var rows Rows
 		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
+			rows.Unmarshal(s)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of parsed rows; got %d; want 4", len(rows.Rows)))
 			}
 		}
 	})

app/vminsert/opentsdb/request_handler.go

@@ -85,15 +85,21 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 			return false
 		}
 	}
-	if err := ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf)); err != nil {
-		opentsdbUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal OpenTSDB put protocol data with size %d: %s", len(ctx.reqBuf), err)
-		return false
+	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
+
+	// Fill in missing timestamps
+	currentTimestamp := time.Now().Unix()
+	rows := ctx.Rows.Rows
+	for i := range rows {
+		r := &rows[i]
+		if r.Timestamp == 0 {
+			r.Timestamp = currentTimestamp
+		}
 	}
 
 	// Convert timestamps from seconds to milliseconds
-	for i := range ctx.Rows.Rows {
-		ctx.Rows.Rows[i].Timestamp *= 1e3
+	for i := range rows {
+		rows[i].Timestamp *= 1e3
 	}
 	return true
 }
@@ -125,9 +131,8 @@ func (ctx *pushCtx) reset() {
 }
 
 var (
-	opentsdbReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
-	opentsdbReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
-	opentsdbUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb"}`)
+	opentsdbReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
+	opentsdbReadErrors = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
 )
 
 func getPushCtx() *pushCtx {

app/vminsert/opentsdbhttp/parser.go

@@ -0,0 +1,198 @@
+package opentsdbhttp
+
+import (
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+	"github.com/valyala/fastjson/fastfloat"
+)
+
+// Rows contains parsed OpenTSDB rows.
+type Rows struct {
+	Rows []Row
+
+	tagsPool []Tag
+}
+
+// Reset resets rs.
+func (rs *Rows) Reset() {
+	// Release references to objects, so they can be GC'ed.
+	for i := range rs.Rows {
+		rs.Rows[i].reset()
+	}
+	rs.Rows = rs.Rows[:0]
+
+	for i := range rs.tagsPool {
+		rs.tagsPool[i].reset()
+	}
+	rs.tagsPool = rs.tagsPool[:0]
+}
+
+// Unmarshal unmarshals OpenTSDB rows from av.
+//
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+//
+// s must be unchanged until rs is in use.
+func (rs *Rows) Unmarshal(av *fastjson.Value) {
+	rs.Rows, rs.tagsPool = unmarshalRows(rs.Rows[:0], av, rs.tagsPool[:0])
+}
+
+// Row is a single OpenTSDB row.
+type Row struct {
+	Metric    string
+	Tags      []Tag
+	Value     float64
+	Timestamp int64
+}
+
+func (r *Row) reset() {
+	r.Metric = ""
+	r.Tags = nil
+	r.Value = 0
+	r.Timestamp = 0
+}
+
+func (r *Row) unmarshal(o *fastjson.Value, tagsPool []Tag) ([]Tag, error) {
+	r.reset()
+	m := o.GetStringBytes("metric")
+	if len(m) == 0 {
+		return tagsPool, fmt.Errorf("missing `metric` in %s", o)
+	}
+	r.Metric = bytesutil.ToUnsafeString(m)
+
+	rawTs := o.Get("timestamp")
+	if rawTs != nil {
+		ts, err := getFloat64(rawTs)
+		if err != nil {
+			return tagsPool, fmt.Errorf("invalid `timestamp` in %s: %s", o, err)
+		}
+		r.Timestamp = int64(ts)
+	} else {
+		// Allow missing timestamp. It is automatically populated
+		// with the current time in this case.
+		r.Timestamp = 0
+	}
+
+	rawV := o.Get("value")
+	if rawV == nil {
+		return tagsPool, fmt.Errorf("missing `value` in %s", o)
+	}
+	v, err := getFloat64(rawV)
+	if err != nil {
+		return tagsPool, fmt.Errorf("invalid `value` in %s: %s", o, err)
+	}
+	r.Value = v
+
+	vt := o.Get("tags")
+	if vt == nil {
+		// Allow empty tags.
+		return tagsPool, nil
+	}
+	rawTags, err := vt.Object()
+	if err != nil {
+		return tagsPool, fmt.Errorf("invalid `tags` in %s: %s", o, err)
+	}
+
+	tagsStart := len(tagsPool)
+	tagsPool, err = unmarshalTags(tagsPool, rawTags)
+	if err != nil {
+		return tagsPool, fmt.Errorf("cannot parse tags %s: %s", rawTags, err)
+	}
+	tags := tagsPool[tagsStart:]
+	r.Tags = tags[:len(tags):len(tags)]
+	return tagsPool, nil
+}
+
+func getFloat64(v *fastjson.Value) (float64, error) {
+	switch v.Type() {
+	case fastjson.TypeNumber:
+		return v.Float64()
+	case fastjson.TypeString:
+		vStr, _ := v.StringBytes()
+		vFloat := fastfloat.ParseBestEffort(bytesutil.ToUnsafeString(vStr))
+		if vFloat == 0 && string(vStr) != "0" && string(vStr) != "0.0" {
+			return 0, fmt.Errorf("invalid float64 value: %q", vStr)
+		}
+		return vFloat, nil
+	default:
+		return 0, fmt.Errorf("value doesn't contain float64; it contains %s", v.Type())
+	}
+}
+
+func unmarshalRows(dst []Row, av *fastjson.Value, tagsPool []Tag) ([]Row, []Tag) {
+	switch av.Type() {
+	case fastjson.TypeObject:
+		return unmarshalRow(dst, av, tagsPool)
+	case fastjson.TypeArray:
+		a, _ := av.Array()
+		for _, o := range a {
+			dst, tagsPool = unmarshalRow(dst, o, tagsPool)
+		}
+		return dst, tagsPool
+	default:
+		logger.Errorf("OpenTSDB JSON must be either object or array; got %s; body=%s", av.Type(), av)
+		invalidLines.Inc()
+		return dst, tagsPool
+	}
+}
+
+func unmarshalRow(dst []Row, o *fastjson.Value, tagsPool []Tag) ([]Row, []Tag) {
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	var err error
+	tagsPool, err = r.unmarshal(o, tagsPool)
+	if err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal OpenTSDB object %s: %s", o, err)
+		invalidLines.Inc()
+	}
+	return dst, tagsPool
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="opentsdb-http"}`)
+
+func unmarshalTags(dst []Tag, o *fastjson.Object) ([]Tag, error) {
+	var err error
+	o.Visit(func(k []byte, v *fastjson.Value) {
+		if v.Type() != fastjson.TypeString {
+			err = fmt.Errorf("tag value must be string; got %s; value=%s", v.Type(), v)
+			return
+		}
+		if len(k) == 0 {
+			// Skip empty tags
+			return
+		}
+		vStr, _ := v.StringBytes()
+		if len(vStr) == 0 {
+			// Skip empty tags
+			return
+		}
+		if cap(dst) > len(dst) {
+			dst = dst[:len(dst)+1]
+		} else {
+			dst = append(dst, Tag{})
+		}
+		tag := &dst[len(dst)-1]
+		tag.Key = bytesutil.ToUnsafeString(k)
+		tag.Value = bytesutil.ToUnsafeString(vStr)
+	})
+	return dst, err
+}
+
+// Tag is an OpenTSDB tag.
+type Tag struct {
+	Key   string
+	Value string
+}
+
+func (t *Tag) reset() {
+	t.Key = ""
+	t.Value = ""
+}

app/vminsert/opentsdbhttp/parser_test.go

@@ -0,0 +1,246 @@
+package opentsdbhttp
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRowsUnmarshalFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		var rows Rows
+		p := parserPool.Get()
+		defer parserPool.Put(p)
+		v, err := p.Parse(s)
+		if err != nil {
+			// Expected JSON parser error
+			return
+		}
+		// Verify OpenTSDB body parsing error
+		rows.Unmarshal(v)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
+		}
+		// Try again
+		rows.Unmarshal(v)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0", len(rows.Rows))
+		}
+	}
+
+	// invalid json
+	f("{g")
+
+	// Invalid json type
+	f(`1`)
+	f(`"foo"`)
+	f(`[1,2]`)
+	f(`null`)
+
+	// Incomplete object
+	f(`{}`)
+	f(`{"metric": "aaa"}`)
+	f(`{"metric": "aaa", "timestamp": 1122}`)
+	f(`{"metric": "aaa", "timestamp": "tststs"}`)
+	f(`{"timestamp": 1122, "value": 33}`)
+	f(`{"value": 33}`)
+	f(`{"value": 33, "tags": {"fooo":"bar"}}`)
+
+	// Invalid value
+	f(`{"metric": "aaa", "timestamp": 1122, "value": "0.0.0"}`)
+
+	// Invalid metric type
+	f(`{"metric": "", "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`)
+	f(`{"metric": ["aaa"], "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`)
+	f(`{"metric": {"aaa":1}, "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`)
+	f(`{"metric": 1, "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`)
+
+	// Invalid timestamp type
+	f(`{"metric": "aaa", "timestamp": "foobar", "value": 0.45, "tags": {"foo": "bar"}}`)
+	f(`{"metric": "aaa", "timestamp": [1,2], "value": 0.45, "tags": {"foo": "bar"}}`)
+	f(`{"metric": "aaa", "timestamp": {"a":1}, "value": 0.45, "tags": {"foo": "bar"}}`)
+
+	// Invalid value type
+	f(`{"metric": "aaa", "timestamp": 1122, "value": [0,1], "tags": {"foo":"bar"}}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": {"a":1}, "tags": {"foo":"bar"}}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": "foobar", "tags": {"foo":"bar"}}`)
+
+	// Invalid tags type
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": 1}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": [1,2]}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": "foo"}`)
+
+	// Invalid tag value type
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": ["bar"]}}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": {"bar":"baz"}}}`)
+	f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": 1}}`)
+
+	// Invalid multiline
+	f(`[{"metric": "aaa", "timestamp": 1122, "value": "trt", "tags":{"foo":"bar"}}, {"metric": "aaa", "timestamp": [1122], "value": 111}]`)
+}
+
+func TestRowsUnmarshalSuccess(t *testing.T) {
+	f := func(s string, rowsExpected *Rows) {
+		t.Helper()
+		var rows Rows
+
+		p := parserPool.Get()
+		defer parserPool.Put(p)
+		v, err := p.Parse(s)
+		if err != nil {
+			t.Fatalf("cannot parse json %s: %s", s, err)
+		}
+		rows.Unmarshal(v)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		// Try unmarshaling again
+		rows.Unmarshal(v)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		rows.Reset()
+		if len(rows.Rows) != 0 {
+			t.Fatalf("non-empty rows after reset: %+v", rows.Rows)
+		}
+	}
+
+	// Normal line
+	f(`{"metric": "foobar", "timestamp": 789, "value": -123.456, "tags": {"a":"b"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{{
+				Key:   "a",
+				Value: "b",
+			}},
+		}},
+	})
+	// Timestamp as string
+	f(`{"metric": "foobar", "timestamp": "1789", "value": -123.456, "tags": {"a":"b"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 1789,
+			Tags: []Tag{{
+				Key:   "a",
+				Value: "b",
+			}},
+		}},
+	})
+	// Timestamp as float64 (it is truncated to integer)
+	f(`{"metric": "foobar", "timestamp": 17.89, "value": -123.456, "tags": {"a":"b"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 17,
+			Tags: []Tag{{
+				Key:   "a",
+				Value: "b",
+			}},
+		}},
+	})
+	// Empty tags
+	f(`{"metric": "foobar", "timestamp": 789, "value": -123.456, "tags": {}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags:      nil,
+		}},
+	})
+	// Missing tags
+	f(`{"metric": "foobar", "timestamp": 789, "value": -123.456}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags:      nil,
+		}},
+	})
+	// Empty tag value
+	f(`{"metric": "foobar", "timestamp": 123, "value": -123.456, "tags": {"a":"", "b":"c", "": "d"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 123,
+			Tags: []Tag{
+				{
+					Key:   "b",
+					Value: "c",
+				},
+			},
+		}},
+	})
+	// Value as string
+	f(`{"metric": "foobar", "timestamp": 789, "value": "-12.456", "tags": {"a":"b"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -12.456,
+			Timestamp: 789,
+			Tags: []Tag{{
+				Key:   "a",
+				Value: "b",
+			}},
+		}},
+	})
+	// Missing timestamp
+	f(`{"metric": "foobar", "value": "-12.456", "tags": {"a":"b"}}`, &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -12.456,
+			Timestamp: 0,
+			Tags: []Tag{{
+				Key:   "a",
+				Value: "b",
+			}},
+		}},
+	})
+
+	// Multiple tags
+	f(`{"metric": "foo", "value": 1, "timestamp": 2, "tags": {"bar":"baz", "x": "y"}}`, &Rows{
+		Rows: []Row{{
+			Metric: "foo",
+			Tags: []Tag{
+				{
+					Key:   "bar",
+					Value: "baz",
+				},
+				{
+					Key:   "x",
+					Value: "y",
+				},
+			},
+			Value:     1,
+			Timestamp: 2,
+		}},
+	})
+
+	// Multi lines
+	f(`[{"metric": "foo", "value": "0.3", "timestamp": 2, "tags": {"a":"b"}},
+{"metric": "bar.baz", "value": 0.34, "timestamp": 43, "tags": {"a":"b"}}]`, &Rows{
+		Rows: []Row{
+			{
+				Metric:    "foo",
+				Value:     0.3,
+				Timestamp: 2,
+				Tags: []Tag{{
+					Key:   "a",
+					Value: "b",
+				}},
+			},
+			{
+				Metric:    "bar.baz",
+				Value:     0.34,
+				Timestamp: 43,
+				Tags: []Tag{{
+					Key:   "a",
+					Value: "b",
+				}},
+			},
+		},
+	})
+}

app/vminsert/opentsdbhttp/parser_timing_test.go

@@ -0,0 +1,33 @@
+package opentsdbhttp
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/valyala/fastjson"
+)
+
+func BenchmarkRowsUnmarshal(b *testing.B) {
+	s := `[{"metric": "cpu.usage_user", "timestamp": 1234556768, "value": 1.23, "tags": {"a":"b", "x": "y"}},
+{"metric": "cpu.usage_system", "timestamp": 1234556768, "value": 23.344, "tags": {"a":"b"}},
+{"metric": "cpu.usage_iowait", "timestamp": 1234556769, "value":3.3443, "tags": {"a":"b"}},
+{"metric": "cpu.usage_irq", "timestamp": 1234556768, "value": 0.34432, "tags": {"a":"b"}}
+]
+`
+	b.SetBytes(int64(len(s)))
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		var rows Rows
+		var p fastjson.Parser
+		for pb.Next() {
+			v, err := p.Parse(s)
+			if err != nil {
+				panic(fmt.Errorf("cannot parse %q: %s", s, err))
+			}
+			rows.Unmarshal(v)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of rows unmarshaled; got %d; want 4", len(rows.Rows)))
+			}
+		}
+	})
+}

app/vminsert/opentsdbhttp/request_handler.go

@@ -0,0 +1,150 @@
+package opentsdbhttp
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb-http"}`)
+	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb-http"}`)
+
+	opentsdbReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb-http"}`)
+	opentsdbReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb-http"}`)
+	opentsdbUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb-http"}`)
+)
+
+// insertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func insertHandler(req *http.Request, maxSize int64) error {
+	return concurrencylimiter.Do(func() error {
+		return insertHandlerInternal(req, maxSize)
+	})
+}
+
+func insertHandlerInternal(req *http.Request, maxSize int64) error {
+	opentsdbReadCalls.Inc()
+
+	r := req.Body
+	if req.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			opentsdbReadErrors.Inc()
+			return fmt.Errorf("cannot read gzipped http protocol data: %s", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
+	}
+
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	// Read the request in ctx.reqBuf
+	lr := io.LimitReader(r, maxSize+1)
+	reqLen, err := ctx.reqBuf.ReadFrom(lr)
+	if err != nil {
+		opentsdbReadErrors.Inc()
+		return fmt.Errorf("cannot read HTTP OpenTSDB request: %s", err)
+	}
+	if reqLen > maxSize {
+		opentsdbReadErrors.Inc()
+		return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed %d bytes", maxSize)
+	}
+
+	// Unmarshal the request to ctx.Rows
+	p := parserPool.Get()
+	defer parserPool.Put(p)
+	v, err := p.ParseBytes(ctx.reqBuf.B)
+	if err != nil {
+		opentsdbUnmarshalErrors.Inc()
+		return fmt.Errorf("cannot parse HTTP OpenTSDB json: %s", err)
+	}
+	ctx.Rows.Unmarshal(v)
+
+	// Fill in missing timestamps
+	currentTimestamp := time.Now().Unix()
+	rows := ctx.Rows.Rows
+	for i := range rows {
+		r := &rows[i]
+		if r.Timestamp == 0 {
+			r.Timestamp = currentTimestamp
+		}
+	}
+
+	// Convert timestamps in seconds to milliseconds if needed.
+	// See http://opentsdb.net/docs/javadoc/net/opentsdb/core/Const.html#SECOND_MASK
+	for i := range rows {
+		r := &rows[i]
+		if r.Timestamp&secondMask == 0 {
+			r.Timestamp *= 1e3
+		}
+	}
+
+	// Insert ctx.Rows to db.
+	ic := &ctx.Common
+	ic.Reset(len(rows))
+	for i := range rows {
+		r := &rows[i]
+		ic.Labels = ic.Labels[:0]
+		ic.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ic.AddLabel(tag.Key, tag.Value)
+		}
+		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+	}
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ic.FlushBufs()
+}
+
+const secondMask int64 = 0x7FFFFFFF00000000
+
+var parserPool fastjson.ParserPool
+
+type pushCtx struct {
+	Rows   Rows
+	Common common.InsertCtx
+
+	reqBuf bytesutil.ByteBuffer
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Rows.Reset()
+	ctx.Common.Reset(0)
+	ctx.reqBuf.Reset()
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/opentsdbhttp/server.go

@@ -0,0 +1,70 @@
+package opentsdbhttp
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	writeRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/put", protocol="opentsdb-http"}`)
+	writeErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/put", protocol="opentsdb-http"}`)
+)
+
+var (
+	httpServer     *http.Server
+	httpAddr       string
+	maxRequestSize int64
+)
+
+// Serve starts HTTP OpenTSDB server on the given addr.
+func Serve(addr string, maxReqSize int64) {
+	logger.Infof("starting HTTP OpenTSDB server at %q", addr)
+	httpAddr = addr
+	maxRequestSize = maxReqSize
+	httpServer = &http.Server{
+		Addr:         addr,
+		Handler:      http.HandlerFunc(requestHandler),
+		ReadTimeout:  30 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+	go func() {
+		err := httpServer.ListenAndServe()
+		if err == http.ErrServerClosed {
+			return
+		}
+		if err != nil {
+			logger.Fatalf("error serving HTTP OpenTSDB: %s", err)
+		}
+	}()
+}
+
+// requestHandler handles HTTP OpenTSDB insert request.
+func requestHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/api/put":
+		writeRequests.Inc()
+		if err := insertHandler(r, maxRequestSize); err != nil {
+			writeErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	default:
+		httpserver.Errorf(w, "unexpected path requested on HTTP OpenTSDB server: %q", r.URL.Path)
+	}
+}
+
+// Stop stops HTTP OpenTSDB server.
+func Stop() {
+	logger.Infof("stopping HTTP OpenTSDB server at %q...", httpAddr)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := httpServer.Shutdown(ctx); err != nil {
+		logger.Fatalf("cannot close HTTP OpenTSDB server: %s", err)
+	}
+}

app/vmrestore/Makefile

@@ -0,0 +1,37 @@
+# All these commands must run from repository root.
+
+vmrestore:
+	APP_NAME=vmrestore $(MAKE) app-local
+
+vmrestore-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker
+
+package-vmrestore:
+	APP_NAME=vmrestore $(MAKE) package-via-docker
+
+publish-vmrestore:
+	APP_NAME=vmrestore $(MAKE) publish-via-docker
+
+vmrestore-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm ./app/vmrestore
+
+vmrestore-arm-prod:
+	APP_NAME=vmrestore APP_SUFFIX='-arm' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm' $(MAKE) app-via-docker
+
+vmrestore-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm64 ./app/vmrestore
+
+vmrestore-arm64-prod:
+	APP_NAME=vmrestore APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker
+
+vmrestore-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-386 ./app/vmrestore
+
+vmrestore-386-prod:
+	APP_NAME=vmrestore APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker
+
+vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) app-local-pure
+
+vmrestore-pure-prod:
+	APP_NAME=vmrestore APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker

app/vmrestore/README.md

@@ -0,0 +1,78 @@
+## vmrestore
+
+`vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
+
+Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+when restarting `vmrestore` with the same args.
+
+
+### Usage
+
+VictoriaMetrics must be stopped during the restore process.
+
+```
+vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+
+```
+
+* `<bucket>` is [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets) name.
+* `<path/to/backup>` is the path to backup made with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) on GCS bucket.
+* `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
+  to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
+
+The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup.
+
+
+### Advanced usage
+
+Run `vmrestore -help` in order to see all the available options:
+
+```
+vmrestore restores VictoriaMetrics data from backups made by vmbackup.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md .
+
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -src string
+    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -storageDataPath string
+    	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmrestore` from the root folder of the repository.
+   It builds `vmrestore` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmrestore-prod` from the root folder of the repository.
+   It builds `vmrestore-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.

app/vmrestore/deployment/Dockerfile

@@ -0,0 +1,5 @@
+FROM scratch
+COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY bin/vmrestore-prod .
+EXPOSE 8428
+ENTRYPOINT ["/vmrestore-prod"]

app/vmrestore/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	src = flag.String("src", "", "Source path with backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
+		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
+	concurrency = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
+)
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Restore{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot restore from backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmrestore restores VictoriaMetrics data from backups made by vmbackup.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newDstFS() (*fslocal.FS, error) {
+	if len(*storageDataPath) == 0 {
+		return nil, fmt.Errorf("`-storageDataPath` cannot be empty")
+	}
+	fs := &fslocal.FS{
+		Dir: *storageDataPath,
+	}
+	return fs, nil
+}
+
+func newSrcFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*src)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %s", *src, err)
+	}
+	return fs, nil
+}

app/vmselect/main.go

@@ -2,6 +2,7 @@ package vmselect
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"runtime"
 	"strings"
@@ -70,7 +71,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		case <-t.C:
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()
-			httpserver.Errorf(w, "cannot handle more than %d concurrent requests", cap(concurrencyCh))
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("cannot handle more than %d concurrent requests", cap(concurrencyCh)),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, "%s", err)
 			return true
 		}
 	}
@@ -185,7 +190,10 @@ func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	logger.Errorf("error in %q: %s", r.URL.Path, err)
 
 	w.Header().Set("Content-Type", "application/json")
-	statusCode := 422
+	statusCode := http.StatusUnprocessableEntity
+	if esc, ok := err.(*httpserver.ErrorWithStatusCode); ok {
+		statusCode = esc.StatusCode
+	}
 	w.WriteHeader(statusCode)
 	prometheus.WriteErrorResponse(w, statusCode, err)
 }

app/vmselect/netstorage/fadvise_darwin.go

@@ -4,6 +4,6 @@ import (
 	"os"
 )
 
-func mustFadviseRandomRead(f *os.File) {
+func mustFadviseSequentialRead(f *os.File) {
 	// Do nothing :)
 }

app/vmselect/netstorage/fadvise_freebsd.go

@@ -0,0 +1,15 @@
+package netstorage
+
+import (
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"golang.org/x/sys/unix"
+)
+
+func mustFadviseSequentialRead(f *os.File) {
+	fd := int(f.Fd())
+	if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_SEQUENTIAL|unix.FADV_WILLNEED); err != nil {
+		logger.Panicf("FATAL: error returned from unix.Fadvise(SEQUENTIAL|WILLNEED): %s", err)
+	}
+}

app/vmselect/netstorage/fadvise_linux.go

@@ -7,9 +7,9 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func mustFadviseRandomRead(f *os.File) {
+func mustFadviseSequentialRead(f *os.File) {
 	fd := int(f.Fd())
-	if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_RANDOM|unix.FADV_WILLNEED); err != nil {
-		logger.Panicf("FATAL: error returned from unix.Fadvise(RANDOM|WILLNEED): %s", err)
+	if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_SEQUENTIAL|unix.FADV_WILLNEED); err != nil {
+		logger.Panicf("FATAL: error returned from unix.Fadvise(SEQUENTIAL|WILLNEED): %s", err)
 	}
 }

app/vmselect/netstorage/netstorage.go

@@ -19,9 +19,9 @@ import (
 )
 
 var (
-	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 10e3, "The maximum number of tag keys returned per search")
-	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 10e3, "The maximum number of tag values returned per search")
-	maxMetricsPerSearch   = flag.Int("search.maxUniqueTimeseries", 100e3, "The maximum number of unique time series each search can scan")
+	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search")
+	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search")
+	maxMetricsPerSearch   = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
 )
 
 // Result is a single timeseries result.
@@ -484,9 +484,12 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 	tbf := getTmpBlocksFile()
 	m := make(map[string][]tmpBlockAddr)
 	blocksRead := 0
+	bb := tmpBufPool.Get()
+	defer tmpBufPool.Put(bb)
 	for sr.NextMetricBlock() {
 		blocksRead++
-		addr, err := tbf.WriteBlock(sr.MetricBlock.Block)
+		bb.B = storage.MarshalBlock(bb.B[:0], sr.MetricBlock.Block)
+		addr, err := tbf.WriteBlockData(bb.B)
 		if err != nil {
 			putTmpBlocksFile(tbf)
 			return nil, fmt.Errorf("cannot write data block #%d to temporary blocks file: %s", blocksRead, err)
@@ -520,6 +523,15 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 		pts.metricName = metricName
 		pts.addrs = addrs
 	}
+
+	// Sort rss.packedTimeseries by the first addr offset in order
+	// to reduce the number of disk seeks during unpacking in RunParallel.
+	// In this case tmpBlocksFile must be read almost sequentially.
+	sort.Slice(rss.packedTimeseries, func(i, j int) bool {
+		pts := rss.packedTimeseries
+		return pts[i].addrs[0].offset < pts[j].addrs[0].offset
+	})
+
 	return &rss, nil
 }
 

app/vmselect/netstorage/tmp_blocks_file.go

@@ -1,7 +1,6 @@
 package netstorage
 
 import (
-	"bufio"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -10,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,13 +30,23 @@ func InitTmpBlocksDir(tmpDirPath string) {
 
 var tmpBlocksDir string
 
-const maxInmemoryTmpBlocksFile = 512 * 1024
+func maxInmemoryTmpBlocksFile() int {
+	mem := memory.Allowed()
+	maxLen := mem / 1024
+	if maxLen < 64*1024 {
+		return 64 * 1024
+	}
+	return maxLen
+}
+
+var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
+	return float64(maxInmemoryTmpBlocksFile())
+})
 
 type tmpBlocksFile struct {
 	buf []byte
 
-	f  *os.File
-	bw *bufio.Writer
+	f *os.File
 
 	offset uint64
 }
@@ -44,7 +54,9 @@ type tmpBlocksFile struct {
 func getTmpBlocksFile() *tmpBlocksFile {
 	v := tmpBlocksFilePool.Get()
 	if v == nil {
-		return &tmpBlocksFile{}
+		return &tmpBlocksFile{
+			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
+		}
 	}
 	return v.(*tmpBlocksFile)
 }
@@ -53,7 +65,6 @@ func putTmpBlocksFile(tbf *tmpBlocksFile) {
 	tbf.MustClose()
 	tbf.buf = tbf.buf[:0]
 	tbf.f = nil
-	tbf.bw = nil
 	tbf.offset = 0
 	tmpBlocksFilePool.Put(tbf)
 }
@@ -69,51 +80,34 @@ func (addr tmpBlockAddr) String() string {
 	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
 }
 
-func getBufioWriter(f *os.File) *bufio.Writer {
-	v := bufioWriterPool.Get()
-	if v == nil {
-		return bufio.NewWriterSize(f, maxInmemoryTmpBlocksFile*2)
-	}
-	bw := v.(*bufio.Writer)
-	bw.Reset(f)
-	return bw
-}
-
-func putBufioWriter(bw *bufio.Writer) {
-	bufioWriterPool.Put(bw)
-}
-
-var bufioWriterPool sync.Pool
-
 var tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
 
-// WriteBlock writes b to tbf.
+// WriteBlockData writes b to tbf.
 //
 // It returns errors since the operation may fail on space shortage
 // and this must be handled.
-func (tbf *tmpBlocksFile) WriteBlock(b *storage.Block) (tmpBlockAddr, error) {
+func (tbf *tmpBlocksFile) WriteBlockData(b []byte) (tmpBlockAddr, error) {
 	var addr tmpBlockAddr
 	addr.offset = tbf.offset
-
-	tbfBufLen := len(tbf.buf)
-	tbf.buf = storage.MarshalBlock(tbf.buf, b)
-	addr.size = len(tbf.buf) - tbfBufLen
+	addr.size = len(b)
 	tbf.offset += uint64(addr.size)
-	if tbf.offset <= maxInmemoryTmpBlocksFile {
+	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
+		// Fast path - the data fits tbf.buf
+		tbf.buf = append(tbf.buf, b...)
 		return addr, nil
 	}
 
+	// Slow path: flush the data from tbf.buf to file.
 	if tbf.f == nil {
 		f, err := ioutil.TempFile(tmpBlocksDir, "")
 		if err != nil {
 			return addr, err
 		}
 		tbf.f = f
-		tbf.bw = getBufioWriter(f)
 		tmpBlocksFilesCreated.Inc()
 	}
-	_, err := tbf.bw.Write(tbf.buf)
-	tbf.buf = tbf.buf[:0]
+	_, err := tbf.f.Write(tbf.buf)
+	tbf.buf = append(tbf.buf[:0], b...)
 	if err != nil {
 		return addr, fmt.Errorf("cannot write block to %q: %s", tbf.f.Name(), err)
 	}
@@ -124,15 +118,18 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	if tbf.f == nil {
 		return nil
 	}
-
-	err := tbf.bw.Flush()
-	putBufioWriter(tbf.bw)
-	tbf.bw = nil
+	if _, err := tbf.f.Write(tbf.buf); err != nil {
+		return fmt.Errorf("cannot flush the remaining %d bytes to tmpBlocksFile: %s", len(tbf.buf), err)
+	}
+	tbf.buf = tbf.buf[:0]
 	if _, err := tbf.f.Seek(0, 0); err != nil {
 		logger.Panicf("FATAL: cannot seek to the start of file: %s", err)
 	}
-	mustFadviseRandomRead(tbf.f)
-	return err
+	// Hint the OS that the file is read almost sequentiallly.
+	// This should reduce the number of disk seeks, which is important
+	// for HDDs.
+	mustFadviseSequentialRead(tbf.f)
+	return nil
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
@@ -167,10 +164,6 @@ func (tbf *tmpBlocksFile) MustClose() {
 	if tbf.f == nil {
 		return
 	}
-	if tbf.bw != nil {
-		putBufioWriter(tbf.bw)
-		tbf.bw = nil
-	}
 	fname := tbf.f.Name()
 
 	// Remove the file at first, then close it.

app/vmselect/netstorage/tmp_blocks_file_test.go

@@ -30,7 +30,7 @@ func TestTmpBlocksFileSerial(t *testing.T) {
 }
 
 func TestTmpBlocksFileConcurrent(t *testing.T) {
-	concurrency := 4
+	concurrency := 3
 	ch := make(chan error, concurrency)
 	for i := 0; i < concurrency; i++ {
 		go func() {
@@ -69,7 +69,7 @@ func testTmpBlocksFile() error {
 		_, _, _ = b.MarshalData(0, 0)
 		return &b
 	}
-	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile / 2, 2 * maxInmemoryTmpBlocksFile} {
+	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile() / 2, 2 * maxInmemoryTmpBlocksFile()} {
 		err := func() error {
 			tbf := getTmpBlocksFile()
 			defer putTmpBlocksFile(tbf)
@@ -77,9 +77,12 @@ func testTmpBlocksFile() error {
 			// Write blocks until their summary size exceeds `size`.
 			var addrs []tmpBlockAddr
 			var blocks []*storage.Block
+			bb := tmpBufPool.Get()
+			defer tmpBufPool.Put(bb)
 			for tbf.offset < uint64(size) {
 				b := createBlock()
-				addr, err := tbf.WriteBlock(b)
+				bb.B = storage.MarshalBlock(bb.B[:0], b)
+				addr, err := tbf.WriteBlockData(bb.B)
 				if err != nil {
 					return fmt.Errorf("cannot write block at offset %d: %s", tbf.offset, err)
 				}
@@ -94,7 +97,7 @@ func testTmpBlocksFile() error {
 			}
 
 			// Read blocks in parallel and verify them
-			concurrency := 3
+			concurrency := 2
 			workCh := make(chan int)
 			doneCh := make(chan error)
 			for i := 0; i < concurrency; i++ {

app/vmselect/prometheus/export.qtpl

@@ -13,7 +13,7 @@
 	{% for i, ts := range rs.Timestamps %}
 		{%z= bb.B %}{% space %}
 		{%f= rs.Values[i] %}{% space %}
-		{%d= int(ts) %}{% newline %}
+		{%dl= ts %}{% newline %}
 	{% endfor %}
 	{% code quicktemplate.ReleaseByteBuffer(bb) %}
 {% endfunc %}
@@ -35,10 +35,10 @@
 		"timestamps":[
 			{% if len(rs.Timestamps) > 0 %}
 				{% code timestamps := rs.Timestamps %}
-				{%d= int(timestamps[0]) %}
+				{%dl= timestamps[0] %}
 				{% code timestamps = timestamps[1:] %}
 				{% for _, ts := range timestamps %}
-					,{%d= int(ts) %}
+					,{%dl= ts %}
 				{% endfor %}
 			{% endif %}
 		]

app/vmselect/prometheus/export.qtpl.go

@@ -49,7 +49,7 @@ func StreamExportPrometheusLine(qw422016 *qt422016.Writer, rs *netstorage.Result
 //line app/vmselect/prometheus/export.qtpl:15
 		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:16
-		qw422016.N().D(int(ts))
+		qw422016.N().DL(ts)
 //line app/vmselect/prometheus/export.qtpl:16
 		qw422016.N().S(`
 `)
@@ -129,7 +129,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 		timestamps := rs.Timestamps
 
 //line app/vmselect/prometheus/export.qtpl:38
-		qw422016.N().D(int(timestamps[0]))
+		qw422016.N().DL(timestamps[0])
 //line app/vmselect/prometheus/export.qtpl:39
 		timestamps = timestamps[1:]
 
@@ -138,7 +138,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 //line app/vmselect/prometheus/export.qtpl:40
 			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:41
-			qw422016.N().D(int(ts))
+			qw422016.N().DL(ts)
 //line app/vmselect/prometheus/export.qtpl:42
 		}
 //line app/vmselect/prometheus/export.qtpl:43

app/vmselect/prometheus/federate.qtpl

@@ -10,7 +10,7 @@
 	{% if len(rs.Timestamps) == 0 || len(rs.Values) == 0 %}{% return %}{% endif %}
 	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
 	{%f= rs.Values[len(rs.Values)-1] %}{% space %}
-	{%d= int(rs.Timestamps[len(rs.Timestamps)-1]) %}{% newline %}
+	{%dl= rs.Timestamps[len(rs.Timestamps)-1] %}{% newline %}
 {% endfunc %}
 
 {% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -41,7 +41,7 @@ func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 //line app/vmselect/prometheus/federate.qtpl:12
 	qw422016.N().S(` `)
 //line app/vmselect/prometheus/federate.qtpl:13
-	qw422016.N().D(int(rs.Timestamps[len(rs.Timestamps)-1]))
+	qw422016.N().DL(rs.Timestamps[len(rs.Timestamps)-1])
 //line app/vmselect/prometheus/federate.qtpl:13
 	qw422016.N().S(`
 `)

app/vmselect/prometheus/prometheus.go

@@ -21,17 +21,17 @@ import (
 )
 
 var (
+	latencyOffset = flag.Duration("search.latencyOffset", time.Second*60, "The time when data points become visible in query results after the colection. "+
+		"Too small value can result in incomplete last points for query results")
 	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum time for search query execution")
 	maxQueryLen      = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
+	maxLookback      = flag.Duration("search.maxLookback", 0, "Synonim to `-search.lookback-delta` from Prometheus. "+
+		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via `max_lookback` arg")
 )
 
 // Default step used if not set.
 const defaultStep = 5 * 60 * 1000
 
-// Latency for data processing pipeline, i.e. the time between data is ignested
-// into the system and the time it becomes visible to search.
-const latencyOffset = 60 * 1000
-
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
@@ -43,11 +43,14 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 	if len(matches) == 0 {
 		return fmt.Errorf("missing `match[]` arg")
 	}
-	maxLookback, err := getDuration(r, "max_lookback", defaultStep)
+	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
 	}
-	start, err := getTime(r, "start", ct-maxLookback)
+	if lookbackDelta <= 0 {
+		lookbackDelta = defaultStep
+	}
+	start, err := getTime(r, "start", ct-lookbackDelta)
 	if err != nil {
 		return err
 	}
@@ -463,17 +466,22 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	step, err := getDuration(r, "step", latencyOffset)
+	queryOffset := getLatencyOffsetMilliseconds()
+	step, err := getDuration(r, "step", queryOffset)
 	if err != nil {
 		return err
 	}
 	deadline := getDeadline(r)
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return err
+	}
 
 	if len(query) > *maxQueryLen {
 		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
 	}
-	if ct-start < latencyOffset {
-		start -= latencyOffset
+	if ct-start < queryOffset {
+		start -= queryOffset
 	}
 	if childQuery, windowStr, offsetStr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
 		var window int64
@@ -503,10 +511,11 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	}
 
 	ec := promql.EvalConfig{
-		Start:    start,
-		End:      start,
-		Step:     step,
-		Deadline: deadline,
+		Start:         start,
+		End:           start,
+		Step:          step,
+		Deadline:      deadline,
+		LookbackDelta: lookbackDelta,
 	}
 	result, err := promql.Exec(&ec, query, true)
 	if err != nil {
@@ -546,6 +555,10 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 	}
 	deadline := getDeadline(r)
 	mayCache := !getBool(r, "nocache")
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return err
+	}
 
 	// Validate input args.
 	if len(query) > *maxQueryLen {
@@ -557,29 +570,68 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 	if err := promql.ValidateMaxPointsPerTimeseries(start, end, step); err != nil {
 		return err
 	}
-	start, end = promql.AdjustStartEnd(start, end, step)
+	if mayCache {
+		start, end = promql.AdjustStartEnd(start, end, step)
+	}
 
 	ec := promql.EvalConfig{
-		Start:    start,
-		End:      end,
-		Step:     step,
-		Deadline: deadline,
-		MayCache: mayCache,
+		Start:         start,
+		End:           end,
+		Step:          step,
+		Deadline:      deadline,
+		MayCache:      mayCache,
+		LookbackDelta: lookbackDelta,
 	}
 	result, err := promql.Exec(&ec, query, false)
 	if err != nil {
 		return fmt.Errorf("cannot execute %q: %s", query, err)
 	}
-	if ct-end < latencyOffset {
+	queryOffset := getLatencyOffsetMilliseconds()
+	if ct-end < queryOffset {
 		result = adjustLastPoints(result)
 	}
 
+	// Remove NaN values as Prometheus does.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153
+	removeNaNValuesInplace(result)
+
 	w.Header().Set("Content-Type", "application/json")
 	WriteQueryRangeResponse(w, result)
 	queryRangeDuration.UpdateDuration(startTime)
 	return nil
 }
 
+func removeNaNValuesInplace(tss []netstorage.Result) {
+	for i := range tss {
+		ts := &tss[i]
+		hasNaNs := false
+		for _, v := range ts.Values {
+			if math.IsNaN(v) {
+				hasNaNs = true
+				break
+			}
+		}
+		if !hasNaNs {
+			// Fast path: nothing to remove.
+			continue
+		}
+
+		// Slow path: remove NaNs.
+		srcTimestamps := ts.Timestamps
+		dstValues := ts.Values[:0]
+		dstTimestamps := ts.Timestamps[:0]
+		for j, v := range ts.Values {
+			if math.IsNaN(v) {
+				continue
+			}
+			dstValues = append(dstValues, v)
+			dstTimestamps = append(dstTimestamps, srcTimestamps[j])
+		}
+		ts.Values = dstValues
+		ts.Timestamps = dstTimestamps
+	}
+}
+
 var queryRangeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/query_range"}`)
 
 // adjustLastPoints substitutes the last point values with the previous
@@ -689,6 +741,11 @@ func getDuration(r *http.Request, argKey string, defaultValue int64) (int64, err
 
 const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
 
+func getMaxLookback(r *http.Request) (int64, error) {
+	d := int64(*maxLookback / time.Millisecond)
+	return getDuration(r, "max_lookback", d)
+}
+
 func getDeadline(r *http.Request) netstorage.Deadline {
 	d, err := getDuration(r, "timeout", 0)
 	if err != nil {
@@ -727,3 +784,11 @@ func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error)
 	}
 	return tagFilterss, nil
 }
+
+func getLatencyOffsetMilliseconds() int64 {
+	d := int64(*latencyOffset / time.Millisecond)
+	if d <= 1000 {
+		d = 1000
+	}
+	return d
+}

app/vmselect/prometheus/prometheus_test.go

@@ -2,11 +2,48 @@ package prometheus
 
 import (
 	"fmt"
+	"math"
 	"net/http"
 	"net/url"
+	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 )
 
+func TestRemoveNaNValuesInplace(t *testing.T) {
+	f := func(tss []netstorage.Result, tssExpected []netstorage.Result) {
+		t.Helper()
+		removeNaNValuesInplace(tss)
+		if !reflect.DeepEqual(tss, tssExpected) {
+			t.Fatalf("unexpected result; got %v; want %v", tss, tssExpected)
+		}
+	}
+
+	nan := math.NaN()
+
+	f(nil, nil)
+	f([]netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 3},
+		},
+		{
+			Timestamps: []int64{100, 200, 300, 400},
+			Values:     []float64{nan, nan, 3, nan},
+		},
+	}, []netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 3},
+		},
+		{
+			Timestamps: []int64{300},
+			Values:     []float64{3},
+		},
+	})
+}
+
 func TestGetTimeSuccess(t *testing.T) {
 	f := func(s string, timestampExpected int64) {
 		t.Helper()

app/vmselect/prometheus/series_count_response.qtpl

@@ -3,7 +3,7 @@ SeriesCountResponse generates response for /api/v1/series/count .
 {% func SeriesCountResponse(n uint64) %}
 {
 	"status":"success",
-	"data":[{%d int(n) %}]
+	"data":[{%dl int64(n) %}]
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/series_count_response.qtpl.go

@@ -24,7 +24,7 @@ func StreamSeriesCountResponse(qw422016 *qt422016.Writer, n uint64) {
 //line app/vmselect/prometheus/series_count_response.qtpl:3
 	qw422016.N().S(`{"status":"success","data":[`)
 //line app/vmselect/prometheus/series_count_response.qtpl:6
-	qw422016.N().D(int(n))
+	qw422016.N().DL(int64(n))
 //line app/vmselect/prometheus/series_count_response.qtpl:6
 	qw422016.N().S(`]}`)
 //line app/vmselect/prometheus/series_count_response.qtpl:8

app/vmselect/promql/aggr.go

@@ -353,6 +353,25 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// Remove dstLabel from grouping like Prometheus does.
+	modifier := &afa.ae.Modifier
+	switch strings.ToLower(modifier.Op) {
+	case "without":
+		modifier.Args = append(modifier.Args, dstLabel)
+	case "by":
+		dstArgs := modifier.Args[:0]
+		for _, arg := range modifier.Args {
+			if arg == dstLabel {
+				continue
+			}
+			dstArgs = append(dstArgs, arg)
+		}
+		modifier.Args = dstArgs
+	default:
+		// Do nothing
+	}
+
 	afe := func(tss []*timeseries) []*timeseries {
 		m := make(map[float64]bool)
 		for _, ts := range tss {

app/vmselect/promql/aggr_incremental_test.go

@@ -179,7 +179,8 @@ func compareValues(vs1, vs2 []float64) error {
 			}
 			continue
 		}
-		if v1 != v2 {
+		eps := math.Abs(v1 - v2)
+		if eps > 1e-14 {
 			return fmt.Errorf("unexpected value; got %v; want %v", v1, v2)
 		}
 	}

app/vmselect/promql/arch_386.go

@@ -0,0 +1,3 @@
+package promql
+
+const maxByteSliceLen = 1<<31 - 1

app/vmselect/promql/binary_op.go

@@ -292,24 +292,14 @@ func adjustBinaryOpTags(be *binaryOpExpr, left, right []*timeseries) ([]*timeser
 	}
 
 	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
-	ensureOneX := func(side string, tss []*timeseries) error {
-		if len(tss) == 0 {
-			logger.Panicf("BUG: tss must contain at least one value")
-		}
-		if len(tss) == 1 {
-			return nil
-		}
-		if mergeNonOverlappingTimeseries(tss) {
-			return nil
-		}
-		return fmt.Errorf(`duplicate timeseries on the %s side of %s %s: %s and %s`, side, be.Op, be.GroupModifier.AppendString(nil),
-			stringMetricTags(&tss[0].MetricName), stringMetricTags(&tss[1].MetricName))
-	}
-
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
-	joinTags := be.JoinModifier.Args
+	groupOp := strings.ToLower(be.GroupModifier.Op)
+	if len(groupOp) == 0 {
+		groupOp = "ignoring"
+	}
+	groupTags := be.GroupModifier.Args
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
 		if len(tssRight) == 0 {
@@ -317,37 +307,38 @@ func adjustBinaryOpTags(be *binaryOpExpr, left, right []*timeseries) ([]*timeser
 		}
 		switch joinOp {
 		case "group_left":
-			if err := ensureOneX("right", tssRight); err != nil {
+			var err error
+			rvsLeft, rvsRight, err = groupJoin("right", be, rvsLeft, rvsRight, tssLeft, tssRight)
+			if err != nil {
 				return nil, nil, nil, err
 			}
-			src := tssRight[0]
-			for _, ts := range tssLeft {
-				ts.MetricName.AddMissingTags(joinTags, &src.MetricName)
-				rvsLeft = append(rvsLeft, ts)
-				rvsRight = append(rvsRight, src)
-			}
 		case "group_right":
-			if err := ensureOneX("left", tssLeft); err != nil {
+			var err error
+			rvsRight, rvsLeft, err = groupJoin("left", be, rvsRight, rvsLeft, tssRight, tssLeft)
+			if err != nil {
 				return nil, nil, nil, err
 			}
-			src := tssLeft[0]
-			for _, ts := range tssRight {
-				ts.MetricName.AddMissingTags(joinTags, &src.MetricName)
-				rvsLeft = append(rvsLeft, src)
-				rvsRight = append(rvsRight, ts)
-			}
 		case "":
-			if err := ensureOneX("left", tssLeft); err != nil {
+			if err := ensureSingleTimeseries("left", be, tssLeft); err != nil {
 				return nil, nil, nil, err
 			}
-			if err := ensureOneX("right", tssRight); err != nil {
+			if err := ensureSingleTimeseries("right", be, tssRight); err != nil {
 				return nil, nil, nil, err
 			}
-			resetMetricGroupIfRequired(be, tssLeft[0])
-			rvsLeft = append(rvsLeft, tssLeft[0])
+			tsLeft := tssLeft[0]
+			resetMetricGroupIfRequired(be, tsLeft)
+			switch groupOp {
+			case "on":
+				tsLeft.MetricName.RemoveTagsOn(groupTags)
+			case "ignoring":
+				tsLeft.MetricName.RemoveTagsIgnoring(groupTags)
+			default:
+				logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
+			}
+			rvsLeft = append(rvsLeft, tsLeft)
 			rvsRight = append(rvsRight, tssRight[0])
 		default:
-			return nil, nil, nil, fmt.Errorf(`unexpected join modifier %q`, joinOp)
+			logger.Panicf("BUG: unexpected join modifier %q", joinOp)
 		}
 	}
 	dst := rvsLeft
@@ -357,6 +348,90 @@ func adjustBinaryOpTags(be *binaryOpExpr, left, right []*timeseries) ([]*timeser
 	return rvsLeft, rvsRight, dst, nil
 }
 
+func ensureSingleTimeseries(side string, be *binaryOpExpr, tss []*timeseries) error {
+	if len(tss) == 0 {
+		logger.Panicf("BUG: tss must contain at least one value")
+	}
+	for len(tss) > 1 {
+		if !mergeNonOverlappingTimeseries(tss[0], tss[len(tss)-1]) {
+			return fmt.Errorf(`duplicate time series on the %s side of %s %s: %s and %s`, side, be.Op, be.GroupModifier.AppendString(nil),
+				stringMetricTags(&tss[0].MetricName), stringMetricTags(&tss[len(tss)-1].MetricName))
+		}
+		tss = tss[:len(tss)-1]
+	}
+	return nil
+}
+
+func groupJoin(singleTimeseriesSide string, be *binaryOpExpr, rvsLeft, rvsRight, tssLeft, tssRight []*timeseries) ([]*timeseries, []*timeseries, error) {
+	joinTags := be.JoinModifier.Args
+	var m map[string]*timeseries
+	for _, tsLeft := range tssLeft {
+		resetMetricGroupIfRequired(be, tsLeft)
+		if len(tssRight) == 1 {
+			// Easy case - right part contains only a single matching time series.
+			tsLeft.MetricName.AddMissingTags(joinTags, &tssRight[0].MetricName)
+			rvsLeft = append(rvsLeft, tsLeft)
+			rvsRight = append(rvsRight, tssRight[0])
+			continue
+		}
+
+		// Hard case - right part contains multiple matching time series.
+		// Verify it doesn't result in duplicate MetricName values after adding missing tags.
+		if m == nil {
+			m = make(map[string]*timeseries, len(tssRight))
+		} else {
+			for k := range m {
+				delete(m, k)
+			}
+		}
+		bb := bbPool.Get()
+		for _, tsRight := range tssRight {
+			var tsCopy timeseries
+			tsCopy.CopyFromShallowTimestamps(tsLeft)
+			tsCopy.MetricName.AddMissingTags(joinTags, &tsRight.MetricName)
+			bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
+			if tsExisting := m[string(bb.B)]; tsExisting != nil {
+				// Try merging tsExisting with tsRight if they don't overlap.
+				if mergeNonOverlappingTimeseries(tsExisting, tsRight) {
+					continue
+				}
+				return nil, nil, fmt.Errorf("duplicate time series on the %s side of `%s %s %s`: %s and %s",
+					singleTimeseriesSide, be.Op, be.GroupModifier.AppendString(nil), be.JoinModifier.AppendString(nil),
+					stringMetricTags(&tsExisting.MetricName), stringMetricTags(&tsRight.MetricName))
+			}
+			m[string(bb.B)] = tsRight
+			rvsLeft = append(rvsLeft, &tsCopy)
+			rvsRight = append(rvsRight, tsRight)
+		}
+		bbPool.Put(bb)
+	}
+	return rvsLeft, rvsRight, nil
+}
+
+func mergeNonOverlappingTimeseries(dst, src *timeseries) bool {
+	// Verify whether the time series can be merged.
+	srcValues := src.Values
+	dstValues := dst.Values
+	_ = dstValues[len(srcValues)-1]
+	for i, v := range srcValues {
+		if math.IsNaN(v) {
+			continue
+		}
+		if !math.IsNaN(dstValues[i]) {
+			return false
+		}
+	}
+
+	// Time series can be merged. Merge them.
+	for i, v := range srcValues {
+		if math.IsNaN(v) {
+			continue
+		}
+		dstValues[i] = v
+	}
+	return true
+}
+
 func resetMetricGroupIfRequired(be *binaryOpExpr, ts *timeseries) {
 	if isBinaryOpCmp(be.Op) && !be.Bool {
 		// Do not reset MetricGroup for non-boolean `compare` binary ops like Prometheus does.
@@ -533,26 +608,3 @@ func isScalar(arg []*timeseries) bool {
 	}
 	return len(mn.Tags) == 0
 }
-
-func mergeNonOverlappingTimeseries(tss []*timeseries) bool {
-	if len(tss) < 2 {
-		logger.Panicf("BUG: expecting at least two timeseries. Got %d", len(tss))
-	}
-
-	// Check whether time series in tss overlap.
-	var dst timeseries
-	dst.CopyFromShallowTimestamps(tss[0])
-	dstValues := dst.Values
-	for _, ts := range tss[1:] {
-		for i, value := range ts.Values {
-			if math.IsNaN(dstValues[i]) {
-				dstValues[i] = value
-			} else if !math.IsNaN(value) {
-				// Time series overlap.
-				return false
-			}
-		}
-	}
-	tss[0].CopyFromShallowTimestamps(&dst)
-	return true
-}

app/vmselect/promql/eval.go

@@ -70,6 +70,9 @@ type EvalConfig struct {
 
 	MayCache bool
 
+	// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
+	LookbackDelta int64
+
 	timestamps     []int64
 	timestampsOnce sync.Once
 }
@@ -82,6 +85,7 @@ func newEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.Step = src.Step
 	ec.Deadline = src.Deadline
 	ec.MayCache = src.MayCache
+	ec.LookbackDelta = src.LookbackDelta
 
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
@@ -290,10 +294,10 @@ func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFu
 		return fe, nrf
 	}
 	if re, ok := e.(*rollupExpr); ok {
-		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() {
+		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
 			return nil, nil
 		}
-		// e = rollupExpr(metricExpr)
+		// e = metricExpr[d]
 		fe := &funcExpr{
 			Name: "default_rollup",
 			Args: []expr{re},
@@ -315,15 +319,17 @@ func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFu
 		if me.IsEmpty() {
 			return nil, nil
 		}
+		// e = rollupFunc(metricExpr)
 		return &funcExpr{
 			Name: fe.Name,
 			Args: []expr{me},
 		}, nrf
 	}
 	if re, ok := arg.(*rollupExpr); ok {
-		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() {
+		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
 			return nil, nil
 		}
+		// e = rollupFunc(metricExpr[d])
 		return fe, nrf
 	}
 	return nil, nil
@@ -368,8 +374,8 @@ func getRollupExprArg(arg expr) *rollupExpr {
 			Expr: arg,
 		}
 	}
-	if len(re.Step) == 0 && !re.InheritStep {
-		// Return standard rollup if it doesn't set step.
+	if !re.ForSubquery() {
+		// Return standard rollup if it doesn't contain subquery.
 		return re
 	}
 	me, ok := re.Expr.(*metricExpr)
@@ -463,7 +469,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	}
 
 	sharedTimestamps := getTimestamps(ec.Start, ec.End, ec.Step)
-	preFunc, rcs := getRollupConfigs(name, rf, ec.Start, ec.End, ec.Step, window, sharedTimestamps)
+	preFunc, rcs := getRollupConfigs(name, rf, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
 	var tssLock sync.Mutex
 	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
@@ -584,12 +590,23 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 		return tss, nil
 	}
 	sharedTimestamps := getTimestamps(start, ec.End, ec.Step)
-	preFunc, rcs := getRollupConfigs(name, rf, start, ec.End, ec.Step, window, sharedTimestamps)
+	preFunc, rcs := getRollupConfigs(name, rf, start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.
 	pointsPerTimeseries := 1 + (ec.End-ec.Start)/ec.Step
-	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(rssLen*len(rcs)))
+	timeseriesLen := rssLen
+	if iafc != nil {
+		// Incremental aggregates require hold only GOMAXPROCS timeseries in memory.
+		timeseriesLen = runtime.GOMAXPROCS(-1)
+		if iafc.ae.Modifier.Op != "" {
+			// Increase the number of timeseries for non-empty group list: `aggr() by (something)`,
+			// since each group can have own set of time series in memory.
+			// Estimate the number of such groups is lower than 100 :)
+			timeseriesLen *= 100
+		}
+	}
+	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs)))
 	rollupMemorySize := mulNoOverflow(rollupPoints, 16)
 	rml := getRollupMemoryLimiter()
 	if !rml.Get(uint64(rollupMemorySize)) {
@@ -687,7 +704,8 @@ func doRollupForTimeseries(rc *rollupConfig, tsDst *timeseries, mnSrc *storage.M
 	tsDst.denyReuse = true
 }
 
-func getRollupConfigs(name string, rf rollupFunc, start, end, step, window int64, sharedTimestamps []int64) (func(values []float64, timestamps []int64), []*rollupConfig) {
+func getRollupConfigs(name string, rf rollupFunc, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) (
+	func(values []float64, timestamps []int64), []*rollupConfig) {
 	preFunc := func(values []float64, timestamps []int64) {}
 	if rollupFuncsRemoveCounterResets[name] {
 		preFunc = func(values []float64, timestamps []int64) {
@@ -703,6 +721,7 @@ func getRollupConfigs(name string, rf rollupFunc, start, end, step, window int64
 			Step:            step,
 			Window:          window,
 			MayAdjustWindow: rollupFuncsMayAdjustWindow[name],
+			LookbackDelta:   lookbackDelta,
 			Timestamps:      sharedTimestamps,
 		}
 	}

app/vmselect/promql/exec.go

@@ -105,14 +105,14 @@ func maySortResults(e expr, tss []*timeseries) bool {
 func timeseriesToResult(tss []*timeseries, maySort bool) ([]netstorage.Result, error) {
 	tss = removeNaNs(tss)
 	result := make([]netstorage.Result, len(tss))
-	m := make(map[string]bool)
+	m := make(map[string]struct{}, len(tss))
 	bb := bbPool.Get()
 	for i, ts := range tss {
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		if m[string(bb.B)] {
+		if _, ok := m[string(bb.B)]; ok {
 			return nil, fmt.Errorf(`duplicate output timeseries: %s%s`, ts.MetricName.MetricGroup, stringMetricName(&ts.MetricName))
 		}
-		m[string(bb.B)] = true
+		m[string(bb.B)] = struct{}{}
 
 		rs := &result[i]
 		rs.MetricNameMarshaled = append(rs.MetricNameMarshaled[:0], bb.B...)
@@ -194,11 +194,14 @@ type parseCacheValue struct {
 }
 
 type parseCache struct {
-	m  map[string]*parseCacheValue
-	mu sync.RWMutex
+	// Move atomic counters to the top of struct for 8-byte alignment on 32-bit arch.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
 
 	requests uint64
 	misses   uint64
+
+	m  map[string]*parseCacheValue
+	mu sync.RWMutex
 }
 
 func (pc *parseCache) Requests() uint64 {

app/vmselect/promql/exec_test.go

@@ -369,6 +369,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("timestamp(time()>=1600)", func(t *testing.T) {
+		t.Parallel()
+		q := `timestamp(time()>=1600)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("time()/100", func(t *testing.T) {
 		t.Parallel()
 		q := `time()/100`
@@ -1302,6 +1313,44 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_value()`, func(t *testing.T) {
+		t.Parallel()
+		q := `with (
+			x = (
+				label_set(time(), "foo", "123.456", "__name__", "aaa"),
+				label_set(-time(), "foo", "bar", "__name__", "bbb"),
+				label_set(-time(), "__name__", "bxs"),
+				label_set(-time(), "foo", "45", "bar", "xs"),
+			)
+		)
+		sort(x + label_value(x, "foo"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{-955, -1155, -1355, -1555, -1755, -1955},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("bar"),
+				Value: []byte("xs"),
+			},
+			{
+				Key:   []byte("foo"),
+				Value: []byte("45"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1123.456, 1323.456, 1523.456, 1723.456, 1923.456, 2123.456},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("123.456"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`label_transform(mismatch)`, func(t *testing.T) {
 		t.Parallel()
 		q := `label_transform(time(), "__name__", "foobar", "xx")`
@@ -1788,10 +1837,6 @@ func TestExecSuccess(t *testing.T) {
 			Timestamps: timestampsExpected,
 		}
 		r.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("aa"),
-				Value: []byte("bb"),
-			},
 			{
 				Key:   []byte("foo"),
 				Value: []byte("bar"),
@@ -1813,17 +1858,60 @@ func TestExecSuccess(t *testing.T) {
 				Key:   []byte("foo"),
 				Value: []byte("bar"),
 			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector * on(foo) group_left(additional_tag) duplicate_timeseries_differ_by_additional_tag`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(label_set(time()/10, "foo", "bar", "xx", "yy", "__name__", "qwert") + on(foo) group_left(op) (
+			label_set(time() < 1400, "foo", "bar", "op", "le"),
+			label_set(time() >= 1400, "foo", "bar", "op", "ge"),
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1100, 1320, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("op"),
+				Value: []byte("le"),
+			},
 			{
 				Key:   []byte("xx"),
 				Value: []byte("yy"),
 			},
 		}
-		resultExpected := []netstorage.Result{r}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1540, 1760, 1980, 2200},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("op"),
+				Value: []byte("ge"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
-	t.Run(`vector * on(foo) group_left() duplicate_timeseries`, func(t *testing.T) {
+	t.Run(`vector * on(foo) duplicate_nonoverlapping_timeseries`, func(t *testing.T) {
 		t.Parallel()
-		q := `label_set(time()/10, "foo", "bar") + on(foo) group_left() (
+		q := `label_set(time()/10, "foo", "bar", "xx", "yy", "__name__", "qwert") + on(foo) (
 			label_set(time() < 1400, "foo", "bar", "op", "le"),
 			label_set(time() >= 1400, "foo", "bar", "op", "ge"),
 		)`
@@ -1832,13 +1920,105 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1100, 1320, 1540, 1760, 1980, 2200},
 			Timestamps: timestampsExpected,
 		}
-		r1.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+		}
 		resultExpected := []netstorage.Result{r1}
 		f(q, resultExpected)
 	})
+	t.Run(`vector * on(foo) group_left() duplicate_nonoverlapping_timeseries`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time()/10, "foo", "bar", "xx", "yy", "__name__", "qwert") + on(foo) group_left() (
+			label_set(time() < 1400, "foo", "bar", "op", "le"),
+			label_set(time() >= 1400, "foo", "bar", "op", "ge"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1100, 1320, 1540, 1760, 1980, 2200},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`vector * on(foo) group_left(__name__)`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time()/10, "foo", "bar", "xx", "yy", "__name__", "qwert") + on(foo) group_left(__name__)
+			label_set(time(), "foo", "bar", "__name__", "aaa")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1100, 1320, 1540, 1760, 1980, 2200},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("aaa")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`vector * on(foo) group_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(label_set(time()/10, "foo", "bar", "xx", "yy", "__name__", "qwert") + on(foo) group_right(xx) (
+			label_set(time(), "foo", "bar", "__name__", "aaa"),
+			label_set(time()+3, "foo", "bar", "__name__", "yyy","ppp", "123"),
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1100, 1320, 1540, 1760, 1980, 2200},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1103, 1323, 1543, 1763, 1983, 2203},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("ppp"),
+				Value: []byte("123"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`vector * on() group_left scalar`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_desc((label_set(time(), "foo", "bar") or label_set(10, "foo", "qwert")) * on() group_left 2)`
@@ -1933,10 +2113,6 @@ func TestExecSuccess(t *testing.T) {
 			Timestamps: timestampsExpected,
 		}
 		r.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("t1"),
-				Value: []byte("v123"),
-			},
 			{
 				Key:   []byte("t2"),
 				Value: []byte("v3"),
@@ -2042,10 +2218,6 @@ func TestExecSuccess(t *testing.T) {
 			Timestamps: timestampsExpected,
 		}
 		r.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("t1"),
-				Value: []byte("v123"),
-			},
 			{
 				Key:   []byte("t2"),
 				Value: []byte("v3"),
@@ -2160,21 +2332,78 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`histogram_quantile(negative-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(histogram_quantile(0.6,
+		q := `histogram_quantile(0.6,
 			label_set(90, "foo", "bar", "le", "10")
 			or label_set(-100, "foo", "bar", "le", "30")
 			or label_set(300, "foo", "bar", "le", "+Inf")
-		))`
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{30, 30, 30, 30, 30, 30},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0.6,
+			label_set(90, "foo", "bar", "le", "10")
+			or label_set(NaN, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{30, 30, 30, 30, 30, 30},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0.2,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(zero-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0.6,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(0, "foo", "bar", "le", "30")
+			or label_set(0, "foo", "bar", "le", "+Inf")
+		)`
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
 	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(histogram_quantile(0.6,
-			label_set(90, "foo", "bar", "le", "10")
-			or label_set(NaN, "foo", "bar", "le", "30")
-			or label_set(300, "foo", "bar", "le", "+Inf")
-		))`
+		q := `histogram_quantile(0.6,
+			label_set(nan, "foo", "bar", "le", "10")
+			or label_set(nan, "foo", "bar", "le", "30")
+			or label_set(nan, "foo", "bar", "le", "+Inf")
+		)`
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
@@ -2453,6 +2682,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`increases_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `increases_over_time(rand(0)[200s:10s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 9, 9, 12, 9, 8},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`decreases_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `decreases_over_time(rand(0)[200s:10s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{9, 11, 11, 8, 11, 12},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`limitk(-1)`, func(t *testing.T) {
 		t.Parallel()
 		q := `limitk(-1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
@@ -3305,7 +3556,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r4 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.85, 0.94, 0.97, 0.93, 0.98, 0.92},
+			Values:     []float64{0.9, 0.94, 0.97, 0.93, 0.98, 0.92},
 			Timestamps: timestampsExpected,
 		}
 		r4.MetricName.Tags = []storage.Tag{{
@@ -3353,7 +3604,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(rollup(time()[:50s]))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{850, 1050, 1250, 1450, 1650, 1850},
+			Values:     []float64{800, 1000, 1200, 1400, 1600, 1800},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -3459,6 +3710,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`lag()`, func(t *testing.T) {
+		t.Parallel()
+		q := `lag(time()[60s:17s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 10, 6, 2, 15, 11},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`()`, func(t *testing.T) {
 		t.Parallel()
 		q := `()`
@@ -3723,6 +3985,107 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6}
 		f(q, resultExpected)
 	})
+	t.Run(`count_values by (xxx)`, func(t *testing.T) {
+		t.Parallel()
+		q := `count_values("xxx", label_set(10, "foo", "bar", "xxx", "aaa") or label_set(floor(time()/600), "foo", "bar", "baz", "xx")) by (xxx)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, nan, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1, 1, 1, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("2"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, nan, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("3"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("10"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
+	t.Run(`count_values without (baz)`, func(t *testing.T) {
+		t.Parallel()
+		q := `count_values("xxx", label_set(floor(time()/600), "foo", "bar")) without (baz)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, nan, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1, 1, 1, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("2"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, nan, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("3"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 }
 
 func TestExecError(t *testing.T) {
@@ -3823,6 +4186,8 @@ func TestExecError(t *testing.T) {
 	f(`alias()`)
 	f(`alias(1)`)
 	f(`alias(1, "foo", "bar")`)
+	f(`lifetime()`)
+	f(`lag()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)

app/vmselect/promql/parser.go

@@ -1550,6 +1550,10 @@ type rollupExpr struct {
 	InheritStep bool
 }
 
+func (re *rollupExpr) ForSubquery() bool {
+	return len(re.Step) > 0 || re.InheritStep
+}
+
 func (re *rollupExpr) AppendString(dst []byte) []byte {
 	needParens := func() bool {
 		if _, ok := re.Expr.(*rollupExpr); ok {

app/vmselect/promql/regexp_cache.go

@@ -51,11 +51,14 @@ type regexpCacheValue struct {
 }
 
 type regexpCache struct {
-	m  map[string]*regexpCacheValue
-	mu sync.RWMutex
+	// Move atomic counters to the top of struct for 8-byte alignment on 32-bit arch.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
 
 	requests uint64
 	misses   uint64
+
+	m  map[string]*regexpCacheValue
+	mu sync.RWMutex
 }
 
 func (rc *regexpCache) Requests() uint64 {

app/vmselect/promql/rollup.go

@@ -38,21 +38,24 @@ var rollupFuncs = map[string]newRollupFunc{
 	"stdvar_over_time":   newRollupFuncOneArg(rollupStdvar),
 
 	// Additional rollup funcs.
-	"sum2_over_time":     newRollupFuncOneArg(rollupSum2),
-	"geomean_over_time":  newRollupFuncOneArg(rollupGeomean),
-	"first_over_time":    newRollupFuncOneArg(rollupFirst),
-	"last_over_time":     newRollupFuncOneArg(rollupLast),
-	"distinct_over_time": newRollupFuncOneArg(rollupDistinct),
-	"integrate":          newRollupFuncOneArg(rollupIntegrate),
-	"ideriv":             newRollupFuncOneArg(rollupIderiv),
-	"lifetime":           newRollupFuncOneArg(rollupLifetime),
-	"scrape_interval":    newRollupFuncOneArg(rollupScrapeInterval),
-	"rollup":             newRollupFuncOneArg(rollupFake),
-	"rollup_rate":        newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_deriv":       newRollupFuncOneArg(rollupFake),
-	"rollup_delta":       newRollupFuncOneArg(rollupFake),
-	"rollup_increase":    newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_candlestick": newRollupFuncOneArg(rollupFake),
+	"sum2_over_time":      newRollupFuncOneArg(rollupSum2),
+	"geomean_over_time":   newRollupFuncOneArg(rollupGeomean),
+	"first_over_time":     newRollupFuncOneArg(rollupFirst),
+	"last_over_time":      newRollupFuncOneArg(rollupLast),
+	"distinct_over_time":  newRollupFuncOneArg(rollupDistinct),
+	"increases_over_time": newRollupFuncOneArg(rollupIncreases),
+	"decreases_over_time": newRollupFuncOneArg(rollupDecreases),
+	"integrate":           newRollupFuncOneArg(rollupIntegrate),
+	"ideriv":              newRollupFuncOneArg(rollupIderiv),
+	"lifetime":            newRollupFuncOneArg(rollupLifetime),
+	"lag":                 newRollupFuncOneArg(rollupLag),
+	"scrape_interval":     newRollupFuncOneArg(rollupScrapeInterval),
+	"rollup":              newRollupFuncOneArg(rollupFake),
+	"rollup_rate":         newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_deriv":        newRollupFuncOneArg(rollupFake),
+	"rollup_delta":        newRollupFuncOneArg(rollupFake),
+	"rollup_increase":     newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_candlestick":  newRollupFuncOneArg(rollupFake),
 }
 
 var rollupFuncsMayAdjustWindow = map[string]bool{
@@ -111,8 +114,9 @@ type rollupFuncArg struct {
 	values        []float64
 	timestamps    []int64
 
-	idx  int
-	step int64
+	currTimestamp int64
+	idx           int
+	step          int64
 }
 
 func (rfa *rollupFuncArg) reset() {
@@ -120,6 +124,7 @@ func (rfa *rollupFuncArg) reset() {
 	rfa.prevTimestamp = 0
 	rfa.values = nil
 	rfa.timestamps = nil
+	rfa.currTimestamp = 0
 	rfa.idx = 0
 	rfa.step = 0
 }
@@ -147,6 +152,9 @@ type rollupConfig struct {
 	MayAdjustWindow bool
 
 	Timestamps []int64
+
+	// LoookbackDelta is the analog to `-query.lookback-delta` from Prometheus world.
+	LookbackDelta int64
 }
 
 var (
@@ -184,6 +192,9 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
 	maxPrevInterval := getMaxPrevInterval(timestamps)
+	if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta {
+		maxPrevInterval = rc.LookbackDelta
+	}
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
@@ -197,29 +208,28 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 
 	i := 0
 	j := 0
+	ni := 0
+	nj := 0
 	for _, tEnd := range rc.Timestamps {
 		tStart := tEnd - window
-		n := sort.Search(len(timestamps)-i, func(n int) bool {
-			return timestamps[i+n] > tStart
-		})
-		i += n
+		ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni)
+		i += ni
 		if j < i {
 			j = i
 		}
-		n = sort.Search(len(timestamps)-j, func(n int) bool {
-			return timestamps[j+n] > tEnd
-		})
-		j += n
+		nj = seekFirstTimestampIdxAfter(timestamps[j:], tEnd, nj)
+		j += nj
 
 		rfa.prevValue = nan
 		rfa.prevTimestamp = tStart - maxPrevInterval
-		if i > 0 && timestamps[i-1] > rfa.prevTimestamp {
+		if i < len(timestamps) && i > 0 && timestamps[i-1] > rfa.prevTimestamp {
 			rfa.prevValue = values[i-1]
 			rfa.prevTimestamp = timestamps[i-1]
 		}
 
 		rfa.values = values[i:j]
 		rfa.timestamps = timestamps[i:j]
+		rfa.currTimestamp = tEnd
 		value := rc.Func(rfa)
 		rfa.idx++
 		dstValues = append(dstValues, value)
@@ -229,6 +239,46 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 	return dstValues
 }
 
+func seekFirstTimestampIdxAfter(timestamps []int64, seekTimestamp int64, nHint int) int {
+	if len(timestamps) == 0 || timestamps[0] > seekTimestamp {
+		return 0
+	}
+	startIdx := nHint - 2
+	if startIdx < 0 {
+		startIdx = 0
+	}
+	if startIdx >= len(timestamps) {
+		startIdx = len(timestamps) - 1
+	}
+	endIdx := nHint + 2
+	if endIdx > len(timestamps) {
+		endIdx = len(timestamps)
+	}
+	if startIdx > 0 && timestamps[startIdx] <= seekTimestamp {
+		timestamps = timestamps[startIdx:]
+		endIdx -= startIdx
+	} else {
+		startIdx = 0
+	}
+	if endIdx < len(timestamps) && timestamps[endIdx] > seekTimestamp {
+		timestamps = timestamps[:endIdx]
+	}
+	if len(timestamps) < 16 {
+		// Fast path: the number of timestamps to search is small, so scan them all.
+		for i, timestamp := range timestamps {
+			if timestamp > seekTimestamp {
+				return startIdx + i
+			}
+		}
+		return startIdx + len(timestamps)
+	}
+	// Slow path: too big len(timestamps), so use binary search.
+	i := sort.Search(len(timestamps), func(n int) bool {
+		return n >= 0 && n < len(timestamps) && timestamps[n] > seekTimestamp
+	})
+	return startIdx + i
+}
+
 func getMaxPrevInterval(timestamps []int64) int64 {
 	if len(timestamps) < 2 {
 		return int64(maxSilenceInterval)
@@ -493,11 +543,14 @@ func rollupAvg(rfa *rollupFuncArg) float64 {
 func rollupMin(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
+	minValue := rfa.prevValue
 	values := rfa.values
-	if len(values) == 0 {
-		return rfa.prevValue
+	if math.IsNaN(minValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		minValue = values[0]
 	}
-	minValue := values[0]
 	for _, v := range values {
 		if v < minValue {
 			minValue = v
@@ -509,11 +562,14 @@ func rollupMin(rfa *rollupFuncArg) float64 {
 func rollupMax(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
+	maxValue := rfa.prevValue
 	values := rfa.values
-	if len(values) == 0 {
-		return rfa.prevValue
+	if math.IsNaN(maxValue) {
+		if len(values) == 0 {
+			return nan
+		}
+		maxValue = values[0]
 	}
-	maxValue := values[0]
 	for _, v := range values {
 		if v > maxValue {
 			maxValue = v
@@ -527,7 +583,10 @@ func rollupSum(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		return rfa.prevValue
+		if math.IsNaN(rfa.prevValue) {
+			return nan
+		}
+		return 0
 	}
 	var sum float64
 	for _, v := range values {
@@ -744,6 +803,18 @@ func rollupLifetime(rfa *rollupFuncArg) float64 {
 	return float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) * 1e-3
 }
 
+func rollupLag(rfa *rollupFuncArg) float64 {
+	// Calculate the duration between the current timestamp and the last data point.
+	timestamps := rfa.timestamps
+	if len(timestamps) == 0 {
+		if math.IsNaN(rfa.prevValue) {
+			return nan
+		}
+		return float64(rfa.currTimestamp-rfa.prevTimestamp) * 1e-3
+	}
+	return float64(rfa.currTimestamp-timestamps[len(timestamps)-1]) * 1e-3
+}
+
 func rollupScrapeInterval(rfa *rollupFuncArg) float64 {
 	// Calculate the average interval between data points.
 	timestamps := rfa.timestamps
@@ -782,6 +853,37 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 	return float64(n)
 }
 
+func rollupIncreases(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	if len(values) == 0 {
+		if math.IsNaN(rfa.prevValue) {
+			return nan
+		}
+		return 0
+	}
+	prevValue := rfa.prevValue
+	if math.IsNaN(prevValue) {
+		prevValue = values[0]
+		values = values[1:]
+	}
+	if len(values) == 0 {
+		return 0
+	}
+	n := 0
+	for _, v := range values {
+		if v > prevValue {
+			n++
+		}
+		prevValue = v
+	}
+	return float64(n)
+}
+
+// `decreases_over_time` logic is the same as `resets` logic.
+var rollupDecreases = rollupResets
+
 func rollupResets(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.

app/vmselect/promql/rollup_result_cache.go

@@ -53,8 +53,7 @@ func InitRollupResultCache(cachePath string) {
 		c = workingsetcache.New(cacheSize, time.Hour)
 	}
 	if *disableCache {
-		c.Stop()
-		c = nil
+		c.Reset()
 	}
 
 	stats := &fastcache.Stats{}
@@ -99,10 +98,8 @@ func InitRollupResultCache(cachePath string) {
 // StopRollupResultCache closes the rollupResult cache.
 func StopRollupResultCache() {
 	if len(rollupResultCachePath) == 0 {
-		if !*disableCache {
-			rollupResultCacheV.c.Stop()
-			rollupResultCacheV.c = nil
-		}
+		rollupResultCacheV.c.Stop()
+		rollupResultCacheV.c = nil
 		return
 	}
 	logger.Infof("saving rollupResult cache to %q...", rollupResultCachePath)

app/vmselect/promql/rollup_test.go

@@ -182,7 +182,8 @@ func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpecte
 				t.Fatalf("unexpected value; got %v; want %v", v, vExpected)
 			}
 		} else {
-			if v != vExpected {
+			eps := math.Abs(v - vExpected)
+			if eps > 1e-14 {
 				t.Fatalf("unexpected value; got %v; want %v", v, vExpected)
 			}
 		}
@@ -293,6 +294,8 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("integrate", 61.0275)
 	f("distinct_over_time", 8)
 	f("ideriv", 0)
+	f("decreases_over_time", 5)
+	f("increases_over_time", 5)
 }
 
 func TestRollupNewRollupFuncError(t *testing.T) {
@@ -358,7 +361,7 @@ func TestRollupNoWindowNoPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{2, 0, 0, 0, 0, 0, 0, 0}
+		valuesExpected := []float64{2, 0, 0, 0, nan, nan, nan, nan}
 		timestampsExpected := []int64{120, 124, 128, 132, 136, 140, 144, 148}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -389,7 +392,7 @@ func TestRollupWindowNoPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{34, 34, 34, nan}
+		valuesExpected := []float64{nan, nan, nan, nan}
 		timestampsExpected := []int64{161, 171, 181, 191}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -420,7 +423,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{12, 44, 34, 34}
+		valuesExpected := []float64{12, 44, 34, nan}
 		timestampsExpected := []int64{100, 120, 140, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -465,7 +468,7 @@ func TestRollupWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{44, 34, 34, 34}
+		valuesExpected := []float64{44, 34, 34, nan}
 		timestampsExpected := []int64{100, 120, 140, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -479,12 +482,57 @@ func TestRollupWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 54, 44, 34}
+		valuesExpected := []float64{nan, 54, 44, nan}
 		timestampsExpected := []int64{0, 50, 100, 150}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
 
+func TestRollupFuncsLookbackDelta(t *testing.T) {
+	t.Run("1", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:          rollupFirst,
+			Start:         80,
+			End:           140,
+			Step:          10,
+			LookbackDelta: 1,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{99, 12, 44, nan, 32, 34, nan}
+		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	t.Run("7", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:          rollupFirst,
+			Start:         80,
+			End:           140,
+			Step:          10,
+			LookbackDelta: 7,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{99, 12, 44, 44, 32, 34, nan}
+		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	t.Run("0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:          rollupFirst,
+			Start:         80,
+			End:           140,
+			Step:          10,
+			LookbackDelta: 0,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{34, 12, 12, 44, 44, 34, nan}
+		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}
+
 func TestRollupFuncsNoWindow(t *testing.T) {
 	t.Run("first", func(t *testing.T) {
 		rc := rollupConfig{
@@ -524,7 +572,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 12, 32, 34}
+		valuesExpected := []float64{nan, 21, 12, 12, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -584,7 +632,21 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{10, 50, 90, 130}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
-	t.Run("lifetime", func(t *testing.T) {
+	t.Run("lag", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:   rollupLag,
+			Start:  0,
+			End:    160,
+			Step:   40,
+			Window: 0,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
+		timestampsExpected := []int64{0, 40, 80, 120, 160}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	t.Run("lifetime_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupLifetime,
 			Start:  0,
@@ -598,7 +660,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
-	t.Run("lifetime", func(t *testing.T) {
+	t.Run("lifetime_2", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupLifetime,
 			Start:  0,
@@ -612,7 +674,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
-	t.Run("scrape_interval", func(t *testing.T) {
+	t.Run("scrape_interval_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupScrapeInterval,
 			Start:  0,
@@ -626,7 +688,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
-	t.Run("scrape_interval", func(t *testing.T) {
+	t.Run("scrape_interval_2", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupScrapeInterval,
 			Start:  0,
@@ -752,7 +814,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
-	t.Run("distinct", func(t *testing.T) {
+	t.Run("distinct_over_time_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupDistinct,
 			Start:  0,
@@ -766,6 +828,20 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("distinct_over_time_2", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:   rollupDistinct,
+			Start:  0,
+			End:    160,
+			Step:   40,
+			Window: 80,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{nan, 4, 7, 6, 3}
+		timestampsExpected := []int64{0, 40, 80, 120, 160}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
 }
 
 func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExpected []float64, timestampsExpected []int64) {
@@ -796,7 +872,7 @@ func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExp
 			}
 			continue
 		}
-		if v != vExpected {
+		if math.Abs(v-vExpected) > 1e-15 {
 			t.Fatalf("unexpected value at values[%d]; got %f; want %f\nvalues=\n%v\nvaluesExpected=\n%v",
 				i, v, vExpected, values, valuesExpected)
 		}

app/vmselect/promql/transform.go

@@ -63,6 +63,7 @@ var transformFuncs = map[string]transformFunc{
 	"label_copy":         transformLabelCopy,
 	"label_move":         transformLabelMove,
 	"label_transform":    transformLabelTransform,
+	"label_value":        transformLabelValue,
 	"union":              transformUnion,
 	"":                   transformUnion, // empty func is a synonim to union
 	"keep_last_value":    transformKeepLastValue,
@@ -308,8 +309,16 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	bbPool.Put(bb)
 
 	// Calculate quantile for each group in m
-	lastNonInf := func(xss []x) float64 {
-		for len(xss) > 0 && math.IsInf(xss[len(xss)-1].le, 0) {
+
+	lastNonInf := func(i int, xss []x) float64 {
+		for len(xss) > 0 {
+			xsLast := xss[len(xss)-1]
+			if xsLast.ts.Values[i] == 0 {
+				return nan
+			}
+			if !math.IsInf(xsLast.le, 0) {
+				break
+			}
 			xss = xss[:len(xss)-1]
 		}
 		if len(xss) == 0 {
@@ -318,27 +327,38 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 		return xss[len(xss)-1].le
 	}
 	quantile := func(i int, phis []float64, xss []x) float64 {
-		vPrev := float64(0)
-		lePrev := float64(0)
 		phi := phis[i]
 		if math.IsNaN(phi) {
 			return nan
 		}
-		// Verify for broken buckets with NaN or negative values.
+		// Fix broken buckets.
+		// They are already sorted by le, so their values must be in ascending order,
+		// since the next bucket value includes all the previous buckets.
+		vPrev := float64(0)
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
-			if math.IsNaN(v) || v < 0 {
-				// Broken bucket.
-				return nan
+			if math.IsNaN(v) || v < vPrev {
+				xs.ts.Values[i] = vPrev
+			} else {
+				vPrev = v
 			}
 		}
+		if len(xss) == 0 {
+			return nan
+		}
 		if phi < 0 {
 			return -inf
 		}
 		if phi > 1 {
 			return inf
 		}
-		vReq := xss[len(xss)-1].ts.Values[i] * phi
+		vLast := xss[len(xss)-1].ts.Values[i]
+		if vLast == 0 {
+			return nan
+		}
+		vReq := vLast * phi
+		vPrev = 0
+		lePrev := float64(0)
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
 			le := xs.le
@@ -348,16 +368,16 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 				continue
 			}
 			if math.IsInf(le, 0) {
-				return lastNonInf(xss)
+				return lastNonInf(i, xss)
 			}
 			if v == vPrev {
 				return lePrev
 			}
 			return lePrev + (le-lePrev)*(vReq-vPrev)/(v-vPrev)
 		}
-		return lastNonInf(xss)
+		return lastNonInf(i, xss)
 	}
-	var rvs []*timeseries
+	rvs := make([]*timeseries, 0, len(m))
 	for _, xss := range m {
 		sort.Slice(xss, func(i, j int) bool {
 			return xss[i].le < xss[j].le
@@ -881,6 +901,33 @@ func labelReplace(tss []*timeseries, srcLabel string, r *regexp.Regexp, dstLabel
 	return tss, nil
 }
 
+func transformLabelValue(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	labelName, err := getString(args[1], 1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get label name: %s", err)
+	}
+	rvs := args[0]
+	for _, ts := range rvs {
+		ts.MetricName.ResetMetricGroup()
+		labelValue := ts.MetricName.GetTagValue(labelName)
+		v, err := strconv.ParseFloat(string(labelValue), 64)
+		if err != nil {
+			v = nan
+		}
+		values := ts.Values
+		for i := range values {
+			values[i] = v
+		}
+	}
+	// Do not remove timeseries with only NaN values, so `default` could be applied to them:
+	// label_value(q, "label") default 123
+	return rvs, nil
+}
+
 func transformLn(v float64) float64 {
 	return math.Log(v)
 }
@@ -1074,7 +1121,10 @@ func transformTimestamp(tfa *transformFuncArg) ([]*timeseries, error) {
 		ts.MetricName.ResetMetricGroup()
 		values := ts.Values
 		for i, t := range ts.Timestamps {
-			values[i] = float64(t) / 1e3
+			v := values[i]
+			if !math.IsNaN(v) {
+				values[i] = float64(t) / 1e3
+			}
 		}
 	}
 	return rvs, nil

app/vmstorage/main.go

@@ -24,6 +24,9 @@ var (
 
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
+
+	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
+	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
 )
 
 // Init initializes vmstorage.
@@ -39,6 +42,10 @@ func InitWithoutMetrics() {
 	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
+
+	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
+	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
+
 	logger.Infof("opening storage at %q with retention period %d months", *DataPath, *retentionPeriod)
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
@@ -298,6 +305,9 @@ func registerStorageMetrics() {
 		return float64(idbm().PartsRefCount)
 	})
 
+	metrics.NewGauge(`vm_new_timeseries_created_total`, func() float64 {
+		return float64(idbm().NewTimeseriesCreated)
+	})
 	metrics.NewGauge(`vm_missing_tsids_for_metric_id_total`, func() float64 {
 		return float64(idbm().MissingTSIDsForMetricID)
 	})
@@ -313,6 +323,12 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_date_metric_ids_search_hits_total`, func() float64 {
 		return float64(idbm().DateMetricIDsSearchHits)
 	})
+	metrics.NewGauge(`vm_index_blocks_with_metric_ids_processed_total`, func() float64 {
+		return float64(idbm().IndexBlocksWithMetricIDsProcessed)
+	})
+	metrics.NewGauge(`vm_index_blocks_with_metric_ids_incorrect_order_total`, func() float64 {
+		return float64(idbm().IndexBlocksWithMetricIDsIncorrectOrder)
+	})
 
 	metrics.NewGauge(`vm_assisted_merges_total{type="storage/small"}`, func() float64 {
 		return float64(tm().SmallAssistedMerges)
@@ -391,6 +407,36 @@ func registerStorageMetrics() {
 		return float64(idbm().ItemsCount)
 	})
 
+	metrics.NewGauge(`vm_recent_hour_inverted_index_entries`, func() float64 {
+		return float64(m().RecentHourInvertedIndexSize)
+	})
+	metrics.NewGauge(`vm_recent_hour_inverted_index_unique_tag_pairs`, func() float64 {
+		return float64(m().RecentHourInvertedIndexUniqueTagPairsSize)
+	})
+	metrics.NewGauge(`vm_recent_hour_inverted_index_pending_metric_ids`, func() float64 {
+		return float64(m().RecentHourInvertedIndexPendingMetricIDsSize)
+	})
+	metrics.NewGauge(`vm_recent_hour_inverted_index_search_calls_total`, func() float64 {
+		return float64(idbm().RecentHourInvertedIndexSearchCalls)
+	})
+	metrics.NewGauge(`vm_recent_hour_inverted_index_search_hits_total`, func() float64 {
+		return float64(idbm().RecentHourInvertedIndexSearchHits)
+	})
+
+	metrics.NewGauge(`vm_date_range_search_calls_total`, func() float64 {
+		return float64(idbm().DateRangeSearchCalls)
+	})
+	metrics.NewGauge(`vm_date_range_hits_total`, func() float64 {
+		return float64(idbm().DateRangeSearchHits)
+	})
+
+	metrics.NewGauge(`vm_date_metric_id_cache_syncs_total`, func() float64 {
+		return float64(m().DateMetricIDCacheSyncsCount)
+	})
+	metrics.NewGauge(`vm_date_metric_id_cache_resets_total`, func() float64 {
+		return float64(m().DateMetricIDCacheResetsCount)
+	})
+
 	metrics.NewGauge(`vm_cache_entries{type="storage/tsid"}`, func() float64 {
 		return float64(m().TSIDCacheSize)
 	})
@@ -437,9 +483,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_size_bytes{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheSizeBytes)
 	})
-	metrics.NewGauge(`vm_cache_size_bytes{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheSizeBytes)
-	})
 	metrics.NewGauge(`vm_cache_size_bytes{type="indexdb/tagFilters"}`, func() float64 {
 		return float64(idbm().TagCacheSizeBytes)
 	})
@@ -456,9 +499,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_requests_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheRequests)
 	})
-	metrics.NewGauge(`vm_cache_requests_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheRequests)
-	})
 	metrics.NewGauge(`vm_cache_requests_total{type="storage/bigIndexBlocks"}`, func() float64 {
 		return float64(tm().BigIndexBlocksCacheRequests)
 	})
@@ -490,9 +530,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_misses_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheMisses)
 	})
-	metrics.NewGauge(`vm_cache_misses_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheMisses)
-	})
 	metrics.NewGauge(`vm_cache_misses_total{type="storage/bigIndexBlocks"}`, func() float64 {
 		return float64(tm().BigIndexBlocksCacheMisses)
 	})
@@ -525,7 +562,4 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_collisions_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheCollisions)
 	})
-	metrics.NewGauge(`vm_cache_collisions_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheCollisions)
-	})
 }

deployment/docker/Makefile

@@ -1,5 +1,5 @@
 DOCKER_NAMESPACE := victoriametrics
-BUILDER_IMAGE := local/builder:go1.12.9
+BUILDER_IMAGE := local/builder:go1.13.3
 CERTS_IMAGE := local/certs:1.0.2
 
 package-certs:
@@ -21,7 +21,8 @@ app-via-docker: package-certs package-builder
 		--env GO111MODULE=on \
 		$(DOCKER_OPTS) \
 		$(BUILDER_IMAGE) \
-		go build $(RACE) -mod=vendor -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' -o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME)
+		go build $(RACE) -mod=vendor -trimpath -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' \
+			-o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME)
 
 package-via-docker:
 	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)') || (\

deployment/docker/builder/Dockerfile

@@ -1,2 +1,2 @@
-FROM golang:1.12.9
+FROM golang:1.13.3
 STOPSIGNAL SIGINT

deployment/docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.5'
 services:
   prometheus:
     container_name: prometheus
-    image: prom/prometheus:v2.10.0
+    image: prom/prometheus:v2.12.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -35,7 +35,7 @@ services:
     restart: always
   grafana:
     container_name: grafana
-    image: grafana/grafana:6.2.1
+    image: grafana/grafana:6.4.4
     entrypoint: >
       /bin/sh -c "
       cd /var/lib/grafana &&

deployment/docker/provisioning/datasources/datasource.yml

@@ -5,10 +5,10 @@ datasources:
       type: prometheus
       access: proxy
       url: http://prometheus:9090
-      isDefault: false
+      isDefault: true
 
     - name: VictoriaMetrics
       type: prometheus
       access: proxy
       url: http://victoriametrics:8428
-      isDefault: true
+      isDefault: false

go.mod

@@ -1,18 +1,28 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
 require (
-	github.com/VictoriaMetrics/fastcache v1.5.1
-	github.com/VictoriaMetrics/metrics v1.7.1
-	github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18
+	cloud.google.com/go v0.47.0 // indirect
+	cloud.google.com/go/storage v1.2.1
+	github.com/VictoriaMetrics/fastcache v1.5.2
+	github.com/VictoriaMetrics/metrics v1.7.2
+	github.com/aws/aws-sdk-go v1.25.29
+	github.com/cespare/xxhash/v2 v2.1.0
+	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
 	github.com/golang/snappy v0.0.1
-	github.com/google/go-cmp v0.3.0 // indirect
-	github.com/klauspost/compress v1.7.5
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/klauspost/compress v1.9.1
 	github.com/valyala/fastjson v1.4.1
-	github.com/valyala/gozstd v1.6.0
+	github.com/valyala/fastrand v1.0.0
+	github.com/valyala/gozstd v1.6.2
 	github.com/valyala/histogram v1.0.1
-	github.com/valyala/quicktemplate v1.1.1
-	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa
+	github.com/valyala/quicktemplate v1.3.1
+	go.opencensus.io v0.22.1 // indirect
+	golang.org/x/net v0.0.0-20191105084925-a882066a44e0 // indirect
+	golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd
+	golang.org/x/tools v0.0.0-20191107010934-f79515f33823 // indirect
+	google.golang.org/api v0.13.0
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/grpc v1.25.0 // indirect
 )
 
 go 1.12

go.sum

@@ -1,36 +1,98 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.47.0 h1:1JUtpcY9E7+eTospEwWS2QXP3DEn7poB3E2j0jN74mM=
+cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU=
+cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.2.1 h1:3G66kIUChoxgxJPzwpWroEnXjfbLuZgbCNAHIvN/sKY=
+cloud.google.com/go/storage v1.2.1/go.mod h1:kpwTAahUQmhyVVGgLWQh2GdyPDZSA3UJDjMm/fDV2oQ=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
-github.com/VictoriaMetrics/fastcache v1.5.1 h1:qHgHjyoNFV7jgucU8QZUuU4gcdhfs8QW1kw68OD2Lag=
-github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
-github.com/VictoriaMetrics/metrics v1.7.1 h1:g2qrY6Upn8rvlvR40cGHFY0crwi4hpqF0n9vJMNsCSg=
-github.com/VictoriaMetrics/metrics v1.7.1/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
+github.com/VictoriaMetrics/fastcache v1.5.2 h1:Erd8iIuBAL9kke8JzM4+WxkKuFkHh3ktwLanJvDgR44=
+github.com/VictoriaMetrics/fastcache v1.5.2/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
+github.com/VictoriaMetrics/metrics v1.7.2 h1:PzC0SEo5lbbNK7xaYwclCCdoaIGRmXOfflIMF3LpSW4=
+github.com/VictoriaMetrics/metrics v1.7.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/aws/aws-sdk-go v1.25.29 h1:XMAoVra1O2paI3ZgiougWydq6zGt6sLzWn4qb7oVbBY=
+github.com/aws/aws-sdk-go v1.25.29/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0=
 github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.7.5 h1:NMapGoDIKPKpk2hpcgAU6XHfsREHG2p8PIg7C3f/jpI=
-github.com/klauspost/compress v1.7.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.1 h1:TWy0o9J9c6LK9C8t7Msh6IAJNXbsU/nvKLTQUU5HdaY=
+github.com/klauspost/compress v1.9.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -41,13 +103,136 @@ github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/y
 github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
-github.com/valyala/gozstd v1.6.0 h1:34qKK75C6Dx9zof2JqUiunfJQ87Up6vTHXABWDyCH+g=
-github.com/valyala/gozstd v1.6.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/valyala/gozstd v1.6.2 h1:MgBfNm0I8IKm51LUTTKfO9vi4BtmoH7kBXeUvgaiZVU=
+github.com/valyala/gozstd v1.6.2/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg=
 github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto=
-github.com/valyala/quicktemplate v1.1.1 h1:C58y/wN0FMTi2PR0n3onltemfFabany53j7M6SDDB8k=
-github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/quicktemplate v1.3.1 h1:V9Ixd/ONuoT6C1ipx8XR2dNGSDgIVnvT4ezZ38ZWllU=
+github.com/valyala/quicktemplate v1.3.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
+go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191105084925-a882066a44e0 h1:QPlSTtPE2k6PZPasQUbzuK3p9JbS+vMXYVto8g/yrsg=
+golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII=
+golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y=
+golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 h1:UXl+Zk3jqqcbEVV7ace5lrt4YdA4tXiz3f/KbmD29Vo=
+google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.0 h1:ItERT+UbGdX+s4u+nQNlVM/Q7cbmf7icKfvzbWqVtq0=
+google.golang.org/grpc v1.25.0/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

lib/backup/actions/backup.go

@@ -0,0 +1,167 @@
+package actions
+
+import (
+	"fmt"
+	"io"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Backup performs backup according to the provided settings.
+//
+// Note that the backup works only for VictoriaMetrics snapshots
+// made via `/snapshot/create`. It works improperly on mutable files.
+type Backup struct {
+	// Concurrency is the number of concurrent workers during the backup.
+	// Concurrency=1 by default.
+	Concurrency int
+
+	// Src is backup source
+	Src *fslocal.FS
+
+	// Dst is backup destination.
+	//
+	// If dst contains the previous backup data, then incremental backup
+	// is made, i.e. only the changed data is uploaded.
+	//
+	// If dst points to empty dir, then full backup is made.
+	// Origin can be set to the previous backup in order to reduce backup duration
+	// and reduce network bandwidth usage.
+	Dst common.RemoteFS
+
+	// Origin is optional origin for speeding up full backup if Dst points
+	// to empty dir.
+	Origin common.OriginFS
+}
+
+// Run runs b with the provided settings.
+func (b *Backup) Run() error {
+	startTime := time.Now()
+
+	concurrency := b.Concurrency
+	src := b.Src
+	dst := b.Dst
+	origin := b.Origin
+
+	if origin == nil {
+		origin = &fsnil.FS{}
+	}
+
+	logger.Infof("starting backup from %s to %s using origin %s", src, dst, origin)
+
+	logger.Infof("obtaining list of parts at %s", src)
+	srcParts, err := src.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list src parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", dst)
+	dstParts, err := dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", origin)
+	originParts, err := origin.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list origin parts: %s", err)
+	}
+
+	backupSize := getPartsSize(srcParts)
+
+	partsToDelete := common.PartsDifference(dstParts, srcParts)
+	deleteSize := getPartsSize(partsToDelete)
+	if len(partsToDelete) > 0 {
+		logger.Infof("deleting %d parts from %s", len(partsToDelete), dst)
+		deletedParts := uint64(0)
+		err = runParallel(concurrency, partsToDelete, func(p common.Part) error {
+			logger.Infof("deleting %s from %s", &p, dst)
+			if err := dst.DeletePart(p); err != nil {
+				return fmt.Errorf("cannot delete %s from %s: %s", &p, dst, err)
+			}
+			atomic.AddUint64(&deletedParts, 1)
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&deletedParts)
+			logger.Infof("deleted %d out of %d parts from %s in %s", n, len(partsToDelete), dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+		if err := dst.RemoveEmptyDirs(); err != nil {
+			return fmt.Errorf("cannot remove empty directories at %s: %s", dst, err)
+		}
+	}
+
+	partsToCopy := common.PartsDifference(srcParts, dstParts)
+	originCopyParts := common.PartsIntersect(originParts, partsToCopy)
+	copySize := getPartsSize(originCopyParts)
+	if len(originCopyParts) > 0 {
+		logger.Infof("server-side copying %d parts from %s to %s", len(originCopyParts), origin, dst)
+		copiedParts := uint64(0)
+		err = runParallel(concurrency, originCopyParts, func(p common.Part) error {
+			logger.Infof("server-side copying %s from %s to %s", &p, origin, dst)
+			if err := dst.CopyPart(origin, p); err != nil {
+				return fmt.Errorf("cannot copy %s from %s to %s: %s", &p, origin, dst, err)
+			}
+			atomic.AddUint64(&copiedParts, 1)
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&copiedParts)
+			logger.Infof("server-side copied %d out of %d parts from %s to %s in %s", n, len(originCopyParts), origin, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	srcCopyParts := common.PartsDifference(partsToCopy, originParts)
+	uploadSize := getPartsSize(srcCopyParts)
+	if len(srcCopyParts) > 0 {
+		logger.Infof("uploading %d parts from %s to %s", len(srcCopyParts), src, dst)
+		bytesUploaded := uint64(0)
+		err = runParallel(concurrency, srcCopyParts, func(p common.Part) error {
+			logger.Infof("uploading %s from %s to %s", &p, src, dst)
+			rc, err := src.NewReadCloser(p)
+			if err != nil {
+				return fmt.Errorf("cannot create reader for %s from %s: %s", &p, src, err)
+			}
+			sr := &statReader{
+				r:         rc,
+				bytesRead: &bytesUploaded,
+			}
+			if err := dst.UploadPart(p, sr); err != nil {
+				return fmt.Errorf("cannot upload %s to %s: %s", &p, dst, err)
+			}
+			if err = rc.Close(); err != nil {
+				return fmt.Errorf("cannot close reader for %s from %s: %s", &p, src, err)
+			}
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&bytesUploaded)
+			logger.Infof("uploaded %d out of %d bytes from %s to %s in %s", n, uploadSize, src, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("backed up %d bytes in %s; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
+		backupSize, time.Since(startTime), deleteSize, copySize, uploadSize)
+
+	return nil
+}
+
+type statReader struct {
+	r         io.Reader
+	bytesRead *uint64
+}
+
+func (sr *statReader) Read(p []byte) (int, error) {
+	n, err := sr.r.Read(p)
+	atomic.AddUint64(sr.bytesRead, uint64(n))
+	return n, err
+}

lib/backup/actions/restore.go

@@ -0,0 +1,170 @@
+package actions
+
+import (
+	"fmt"
+	"io"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Restore restores data according to the provided settings.
+//
+// Note that the restore works only for VictoriaMetrics backups made from snapshots.
+// It works improperly on mutable files.
+type Restore struct {
+	// Concurrency is the number of concurrent workers to run during restore.
+	// Concurrency=1 is used by default.
+	Concurrency int
+
+	// Src is the source containing backed up data.
+	Src common.RemoteFS
+
+	// Dst is destination to restore the data.
+	//
+	// If dst points to existing directory, then incremental restore is performed,
+	// i.e. only new data is downloaded from src.
+	Dst *fslocal.FS
+}
+
+// Run runs r with the provided settings.
+func (r *Restore) Run() error {
+	startTime := time.Now()
+
+	concurrency := r.Concurrency
+	src := r.Src
+	dst := r.Dst
+	logger.Infof("starting restore from %s to %s", src, dst)
+
+	logger.Infof("obtaining list of parts at %s", src)
+	srcParts, err := src.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list src parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", dst)
+	dstParts, err := dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts: %s", err)
+	}
+
+	backupSize := getPartsSize(srcParts)
+
+	// Validate srcParts. They must cover the whole files.
+	common.SortParts(srcParts)
+	offset := uint64(0)
+	var pOld common.Part
+	var path string
+	for _, p := range srcParts {
+		if p.Path != path {
+			if offset != pOld.FileSize {
+				return fmt.Errorf("invalid size for %q; got %d; want %d", path, offset, pOld.FileSize)
+			}
+			pOld = p
+			path = p.Path
+			offset = 0
+		}
+		if p.Offset < offset {
+			return fmt.Errorf("there is an overlap in %d bytes between %s and %s", offset-p.Offset, &pOld, &p)
+		}
+		if p.Offset > offset {
+			if offset == 0 {
+				return fmt.Errorf("there is a gap in %d bytes from file start to %s", p.Offset, &p)
+			}
+			return fmt.Errorf("there is a gap in %d bytes between %s and %s", p.Offset-offset, &pOld, &p)
+		}
+		if p.Size != p.ActualSize {
+			return fmt.Errorf("invalid size for %s; got %d; want %d", &p, p.ActualSize, p.Size)
+		}
+		offset += p.Size
+	}
+
+	partsToDelete := common.PartsDifference(dstParts, srcParts)
+	deleteSize := uint64(0)
+	if len(partsToDelete) > 0 {
+		// Fully remove local file if certain parts from the remote part are missing.
+		pathsToDelete := make(map[string]bool)
+		for _, p := range partsToDelete {
+			pathsToDelete[p.Path] = true
+		}
+		logger.Infof("deleting %d files from %s", len(pathsToDelete), dst)
+		for path := range pathsToDelete {
+			logger.Infof("deleting %s from %s", path, dst)
+			size, err := dst.DeletePath(path)
+			if err != nil {
+				return fmt.Errorf("cannot delete %s from %s: %s", path, dst, err)
+			}
+			deleteSize += size
+		}
+		if err != nil {
+			return err
+		}
+		if err := dst.RemoveEmptyDirs(); err != nil {
+			return fmt.Errorf("cannot remove empty directories at %s: %s", dst, err)
+		}
+	}
+
+	// Re-read dstParts, since additional parts may be removed on the previous step.
+	dstParts, err = dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts after the deletion: %s", err)
+	}
+
+	partsToCopy := common.PartsDifference(srcParts, dstParts)
+	downloadSize := getPartsSize(partsToCopy)
+	if len(partsToCopy) > 0 {
+		perPath := make(map[string][]common.Part)
+		for _, p := range partsToCopy {
+			parts := perPath[p.Path]
+			parts = append(parts, p)
+			perPath[p.Path] = parts
+		}
+		logger.Infof("downloading %d parts from %s to %s", len(partsToCopy), src, dst)
+		bytesDownloaded := uint64(0)
+		err = runParallelPerPath(concurrency, perPath, func(parts []common.Part) error {
+			// Sort partsToCopy in order to properly grow file size during downloading.
+			common.SortParts(parts)
+			for _, p := range parts {
+				logger.Infof("downloading %s from %s to %s", &p, src, dst)
+				wc, err := dst.NewWriteCloser(p)
+				if err != nil {
+					return fmt.Errorf("cannot create writer for %q to %s: %s", &p, dst, err)
+				}
+				sw := &statWriter{
+					w:            wc,
+					bytesWritten: &bytesDownloaded,
+				}
+				if err := src.DownloadPart(p, sw); err != nil {
+					return fmt.Errorf("cannot download %s to %s: %s", &p, dst, err)
+				}
+				if err := wc.Close(); err != nil {
+					return fmt.Errorf("cannot close reader fro %s from %s: %s", &p, src, err)
+				}
+			}
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&bytesDownloaded)
+			logger.Infof("downloaded %d out of %d bytes from %s to %s in %s", n, downloadSize, src, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("restored %d bytes from backup in %s; deleted %d bytes; downloaded %d bytes", backupSize, time.Since(startTime), deleteSize, downloadSize)
+
+	return nil
+}
+
+type statWriter struct {
+	w            io.Writer
+	bytesWritten *uint64
+}
+
+func (sw *statWriter) Write(p []byte) (int, error) {
+	n, err := sw.w.Write(p)
+	atomic.AddUint64(sw.bytesWritten, uint64(n))
+	return n, err
+}

lib/backup/actions/util.go

@@ -0,0 +1,231 @@
+package actions
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
+)
+
+var (
+	credsFilePath = flag.String("credsFilePath", "", "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.\n"+
+		"See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
+	configFilePath = flag.String("configFilePath", "", "Path to file with S3 configs. Configs are loaded from default location if not set.\n"+
+		"See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
+)
+
+func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
+	var err error
+	runWithProgress(progress, func() {
+		err = runParallelInternal(concurrency, parts, f)
+	})
+	return err
+}
+
+func runParallelPerPath(concurrency int, perPath map[string][]common.Part, f func(parts []common.Part) error, progress func(elapsed time.Duration)) error {
+	var err error
+	runWithProgress(progress, func() {
+		err = runParallelPerPathInternal(concurrency, perPath, f)
+	})
+	return err
+}
+
+func runParallelPerPathInternal(concurrency int, perPath map[string][]common.Part, f func(parts []common.Part) error) error {
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	if len(perPath) == 0 {
+		return nil
+	}
+
+	// len(perPath) capacity guarantees non-blocking behavior below.
+	resultCh := make(chan error, len(perPath))
+	workCh := make(chan []common.Part, len(perPath))
+	stopCh := make(chan struct{})
+
+	// Start workers
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for parts := range workCh {
+				select {
+				case <-stopCh:
+					return
+				default:
+				}
+				resultCh <- f(parts)
+			}
+		}()
+	}
+
+	// Feed workers with work.
+	for _, parts := range perPath {
+		workCh <- parts
+	}
+	close(workCh)
+
+	// Read results.
+	var err error
+	for i := 0; i < len(perPath); i++ {
+		err = <-resultCh
+		if err != nil {
+			// Stop the work.
+			close(stopCh)
+			break
+		}
+	}
+
+	// Wait for all the workers to stop.
+	wg.Wait()
+
+	return err
+}
+
+func runParallelInternal(concurrency int, parts []common.Part, f func(p common.Part) error) error {
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	if len(parts) == 0 {
+		return nil
+	}
+
+	// len(parts) capacity guarantees non-blocking behavior below.
+	resultCh := make(chan error, len(parts))
+	workCh := make(chan common.Part, len(parts))
+	stopCh := make(chan struct{})
+
+	// Start workers
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for p := range workCh {
+				select {
+				case <-stopCh:
+					return
+				default:
+				}
+				resultCh <- f(p)
+			}
+		}()
+	}
+
+	// Feed workers with work.
+	for _, p := range parts {
+		workCh <- p
+	}
+	close(workCh)
+
+	// Read results.
+	var err error
+	for i := 0; i < len(parts); i++ {
+		err = <-resultCh
+		if err != nil {
+			// Stop the work.
+			close(stopCh)
+			break
+		}
+	}
+
+	// Wait for all the workers to stop.
+	wg.Wait()
+
+	return err
+}
+
+func runWithProgress(progress func(elapsed time.Duration), f func()) {
+	startTime := time.Now()
+	doneCh := make(chan struct{})
+	go func() {
+		f()
+		close(doneCh)
+	}()
+
+	tc := time.NewTicker(10 * time.Second)
+	for {
+		select {
+		case <-doneCh:
+			tc.Stop()
+			// The last progress call.
+			progress(time.Since(startTime))
+			return
+		case <-tc.C:
+			progress(time.Since(startTime))
+		}
+	}
+}
+
+func getPartsSize(parts []common.Part) uint64 {
+	n := uint64(0)
+	for _, p := range parts {
+		n += p.Size
+	}
+	return n
+}
+
+// NewRemoteFS returns new remote fs from the given path.
+func NewRemoteFS(path string) (common.RemoteFS, error) {
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	n := strings.Index(path, "://")
+	if n < 0 {
+		return nil, fmt.Errorf("Missing scheme in path %q. Supported schemes: `gcs://`, `s3://`, `fs://`", path)
+	}
+	scheme := path[:n]
+	dir := path[n+len("://"):]
+	switch scheme {
+	case "fs":
+		if !strings.HasPrefix(dir, "/") {
+			return nil, fmt.Errorf("dir must be absolute; got %q", dir)
+		}
+		fs := &fsremote.FS{
+			Dir: dir,
+		}
+		return fs, nil
+	case "gcs":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fs := &gcsremote.FS{
+			CredsFilePath: *credsFilePath,
+			Bucket:        bucket,
+			Dir:           dir,
+		}
+		if err := fs.Init(); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to gcs: %s", err)
+		}
+		return fs, nil
+	case "s3":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fs := &s3remote.FS{
+			CredsFilePath:  *credsFilePath,
+			ConfigFilePath: *configFilePath,
+			Bucket:         bucket,
+			Dir:            dir,
+		}
+		if err := fs.Init(); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to s3: %s", err)
+		}
+		return fs, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q in `-dst`", scheme)
+	}
+}

lib/backup/common/fs.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"io"
+)
+
+// OriginFS is an interface for remote origin filesystem.
+//
+// This filesystem is used for performing server-side file copies
+// instead of uploading data from local filesystem.
+type OriginFS interface {
+	// String must return human-readable representation of OriginFS.
+	String() string
+
+	// ListParts must return all the parts for the OriginFS.
+	ListParts() ([]Part, error)
+}
+
+// RemoteFS is a filesystem where backups are stored.
+type RemoteFS interface {
+	// String must return human-readable representation of RemoteFS.
+	String() string
+
+	// ListParts must return all the parts for the RemoteFS.
+	ListParts() ([]Part, error)
+
+	// DeletePart must delete part p from RemoteFS.
+	DeletePart(p Part) error
+
+	// RemoveEmptyDirs must recursively remove empty directories in RemoteFS.
+	RemoveEmptyDirs() error
+
+	// CopyPart must copy part p from dstFS to RemoteFS.
+	CopyPart(dstFS OriginFS, p Part) error
+
+	// DownloadPart must download part p from RemoteFS to w.
+	DownloadPart(p Part, w io.Writer) error
+
+	// UploadPart must upload part p from r to RemoteFS.
+	UploadPart(p Part, r io.Reader) error
+}

lib/backup/common/part.go

@@ -0,0 +1,136 @@
+package common
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Part is an atomic unit for transfer during backup / restore.
+//
+// Each source file can be split into parts with up to MaxPartSize sizes.
+type Part struct {
+	// Path is the path to file for backup.
+	Path string
+
+	// FileSize is the size of the whole file for the given part.
+	FileSize uint64
+
+	// Offset is offset in the file to backup.
+	Offset uint64
+
+	// Size is the size of the part to backup starting from Offset.
+	Size uint64
+
+	// ActualSize is the actual size of the part.
+	//
+	// The part is considered broken if it isn't equal to Size.
+	// Such a part must be removed from remote storage.
+	ActualSize uint64
+}
+
+func (p *Part) key() string {
+	return fmt.Sprintf("%s%016X%016X%016X%016X", p.Path, p.FileSize, p.Offset, p.Size, p.ActualSize)
+}
+
+// String returns human-readable representation of the part.
+func (p *Part) String() string {
+	return fmt.Sprintf("part{path: %q, file_size: %d, offset: %d, size: %d}", p.Path, p.FileSize, p.Offset, p.Size)
+}
+
+// RemotePath returns remote path for the part p and the given prefix.
+func (p *Part) RemotePath(prefix string) string {
+	for strings.HasSuffix(prefix, "/") {
+		prefix = prefix[:len(prefix)-1]
+	}
+	return fmt.Sprintf("%s/%s/%016X_%016X_%016X", prefix, p.Path, p.FileSize, p.Offset, p.Size)
+}
+
+var partNameRegexp = regexp.MustCompile(`^(.+)/([0-9A-F]{16})_([0-9A-F]{16})_([0-9A-F]{16})$`)
+
+// ParseFromRemotePath parses p from remotePath.
+//
+// Returns true on success.
+func (p *Part) ParseFromRemotePath(remotePath string) bool {
+	tmp := partNameRegexp.FindStringSubmatch(remotePath)
+	if len(tmp) != 5 {
+		return false
+	}
+	path := tmp[1]
+	for strings.HasPrefix(path, "/") {
+		path = path[1:]
+	}
+	fileSize, err := strconv.ParseUint(tmp[2], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse fileSize from %q: %s", tmp[2], err)
+	}
+	offset, err := strconv.ParseUint(tmp[3], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse offset from %q: %s", tmp[3], err)
+	}
+	size, err := strconv.ParseUint(tmp[4], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse size from %q: %s", tmp[4], err)
+	}
+	p.Path = path
+	p.FileSize = fileSize
+	p.Offset = offset
+	p.Size = size
+	return true
+}
+
+// MaxPartSize is the maximum size for each part.
+//
+// The MaxPartSize reduces bandwidth usage during retires on network errors
+// when transferring multi-TB files.
+const MaxPartSize = 128 * 1024 * 1024
+
+// SortParts sorts parts by (Path, Offset)
+func SortParts(parts []Part) {
+	sort.Slice(parts, func(i, j int) bool {
+		a := parts[i]
+		b := parts[j]
+		if a.Path != b.Path {
+			return a.Path < b.Path
+		}
+		return a.Offset < b.Offset
+	})
+}
+
+// PartsDifference returns a - b
+func PartsDifference(a, b []Part) []Part {
+	m := make(map[string]bool, len(b))
+	for _, p := range b {
+		k := p.key()
+		m[k] = true
+	}
+	var d []Part
+	for _, p := range a {
+		k := p.key()
+		if !m[k] {
+			d = append(d, p)
+		}
+	}
+	return d
+}
+
+// PartsIntersect returns the intersection of a and b
+func PartsIntersect(a, b []Part) []Part {
+	m := make(map[string]bool, len(a))
+	for _, p := range a {
+		k := p.key()
+		m[k] = true
+	}
+	var d []Part
+	for _, p := range b {
+		k := p.key()
+		if m[k] {
+			d = append(d, p)
+		}
+	}
+	return d
+}

lib/backup/fscommon/fscommon.go

@@ -0,0 +1,246 @@
+package fscommon
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FsyncFile fsyncs path contents and the parent directory contents.
+func FsyncFile(path string) error {
+	if err := fsync(path); err != nil {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("cannot fsync file %q: %s", path, err)
+	}
+	dir := filepath.Dir(path)
+	if err := fsync(dir); err != nil {
+		return fmt.Errorf("cannot fsync dir %q: %s", dir, err)
+	}
+	return nil
+}
+
+// FsyncDir fsyncs dir contents.
+func FsyncDir(dir string) error {
+	return fsync(dir)
+}
+
+func fsync(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+// AppendFiles appends all the files from dir to dst.
+//
+// All the appended files will have dir prefix.
+func AppendFiles(dst []string, dir string) ([]string, error) {
+	d, err := os.Open(dir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open %q: %s", dir, err)
+	}
+	dst, err = appendFilesInternal(dst, d)
+	if err1 := d.Close(); err1 != nil {
+		err = err1
+	}
+	return dst, err
+}
+
+func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
+	dir := d.Name()
+	dfi, err := d.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", dir, err)
+	}
+	if !dfi.IsDir() {
+		return nil, fmt.Errorf("%q isn't a directory", dir)
+	}
+	fis, err := d.Readdir(-1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read directory contents in %q: %s", dir, err)
+	}
+	for _, fi := range fis {
+		name := fi.Name()
+		if name == "." || name == ".." {
+			continue
+		}
+		path := dir + "/" + name
+		if fi.IsDir() {
+			// Process directory
+			dst, err = AppendFiles(dst, path)
+			if err != nil {
+				return nil, fmt.Errorf("cannot list %q: %s", path, err)
+			}
+			continue
+		}
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Process file
+			dst = append(dst, path)
+			continue
+		}
+		pathOrig := path
+	again:
+		// Process symlink
+		pathReal, err := filepath.EvalSymlinks(pathOrig)
+		if err != nil {
+			if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
+				// Skip symlink that points to nowhere.
+				continue
+			}
+			return nil, fmt.Errorf("cannot resolve symlink %q: %s", pathOrig, err)
+		}
+		sfi, err := os.Stat(pathReal)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat %q from symlink %q: %s", pathReal, path, err)
+		}
+		if sfi.IsDir() {
+			// Symlink points to directory
+			dstNew, err := AppendFiles(dst, pathReal)
+			if err != nil {
+				return nil, fmt.Errorf("cannot list files at %q from symlink %q: %s", pathReal, path, err)
+			}
+			pathReal += "/"
+			for i := len(dst); i < len(dstNew); i++ {
+				x := dstNew[i]
+				if !strings.HasPrefix(x, pathReal) {
+					return nil, fmt.Errorf("unexpected prefix for path %q; want %q", x, pathReal)
+				}
+				dstNew[i] = path + "/" + x[len(pathReal):]
+			}
+			dst = dstNew
+			continue
+		}
+		if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Symlink points to file
+			dst = append(dst, path)
+			continue
+		}
+		// Symlink points to symlink. Process it again.
+		pathOrig = pathReal
+		goto again
+	}
+	return dst, nil
+}
+
+// RemoveEmptyDirs recursively removes empty directories under the given dir.
+func RemoveEmptyDirs(dir string) error {
+	_, err := removeEmptyDirs(dir)
+	return err
+}
+
+func removeEmptyDirs(dir string) (bool, error) {
+	d, err := os.Open(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return true, nil
+		}
+		return false, err
+	}
+	ok, err := removeEmptyDirsInternal(d)
+	if err1 := d.Close(); err1 != nil {
+		err = err1
+	}
+	if err != nil {
+		return false, err
+	}
+	return ok, nil
+}
+
+func removeEmptyDirsInternal(d *os.File) (bool, error) {
+	dir := d.Name()
+	dfi, err := d.Stat()
+	if err != nil {
+		return false, fmt.Errorf("cannot stat %q: %s", dir, err)
+	}
+	if !dfi.IsDir() {
+		return false, fmt.Errorf("%q isn't a directory", dir)
+	}
+	fis, err := d.Readdir(-1)
+	if err != nil {
+		return false, fmt.Errorf("cannot read directory contents in %q: %s", dir, err)
+	}
+	dirEntries := 0
+	for _, fi := range fis {
+		name := fi.Name()
+		if name == "." || name == ".." {
+			continue
+		}
+		path := dir + "/" + name
+		if fi.IsDir() {
+			// Process directory
+			ok, err := removeEmptyDirs(path)
+			if err != nil {
+				return false, fmt.Errorf("cannot list %q: %s", path, err)
+			}
+			if !ok {
+				dirEntries++
+			}
+			continue
+		}
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Skip plain files.
+			dirEntries++
+			continue
+		}
+		pathOrig := path
+	again:
+		// Process symlink
+		pathReal, err := filepath.EvalSymlinks(pathOrig)
+		if err != nil {
+			if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
+				// Remove symlink that points to nowere.
+				logger.Infof("removing broken symlink %q", pathOrig)
+				if err := os.Remove(pathOrig); err != nil {
+					return false, fmt.Errorf("cannot remove %q: %s", pathOrig, err)
+				}
+				continue
+			}
+			return false, fmt.Errorf("cannot resolve symlink %q: %s", pathOrig, err)
+		}
+		sfi, err := os.Stat(pathReal)
+		if err != nil {
+			return false, fmt.Errorf("cannot stat %q from symlink %q: %s", pathReal, path, err)
+		}
+		if sfi.IsDir() {
+			// Symlink points to directory
+			ok, err := removeEmptyDirs(pathReal)
+			if err != nil {
+				return false, fmt.Errorf("cannot list files at %q from symlink %q: %s", pathReal, path, err)
+			}
+			if !ok {
+				dirEntries++
+			} else {
+				// Remove the symlink
+				logger.Infof("removing symlink that points to empty dir %q", pathOrig)
+				if err := os.Remove(pathOrig); err != nil {
+					return false, fmt.Errorf("cannot remove %q: %s", pathOrig, err)
+				}
+			}
+			continue
+		}
+		if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Symlink points to file. Skip it.
+			dirEntries++
+			continue
+		}
+		// Symlink points to symlink. Process it again.
+		pathOrig = pathReal
+		goto again
+	}
+	if dirEntries > 0 {
+		return false, nil
+	}
+	logger.Infof("removing empty dir %q", dir)
+	if err := os.Remove(dir); err != nil {
+		return false, fmt.Errorf("cannot remove %q: %s", dir, err)
+	}
+	return true, nil
+}

lib/backup/fslocal/fslocal.go

@@ -0,0 +1,201 @@
+package fslocal
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represents local filesystem.
+//
+// Backups are made from local fs.
+// Data is restored from backups to local fs.
+type FS struct {
+	// Dir is a path to local directory to work with.
+	Dir string
+}
+
+// String returns user-readable representation for the fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fslocal %q", fs.Dir)
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			// Return empty part list for non-existing directory.
+			// The directory will be created later.
+			return nil, nil
+		}
+		return nil, err
+	}
+	files, err := fscommon.AppendFiles(nil, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var parts []common.Part
+	dir += "/"
+	for _, file := range files {
+		if !strings.HasPrefix(file, dir) {
+			logger.Panicf("BUG: unexpected prefix for file %q; want %q", file, dir)
+		}
+		fi, err := os.Stat(file)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat %q: %s", file, err)
+		}
+		path := file[len(dir):]
+		size := uint64(fi.Size())
+		if size == 0 {
+			parts = append(parts, common.Part{
+				Path:   path,
+				Offset: 0,
+				Size:   0,
+			})
+			continue
+		}
+		offset := uint64(0)
+		for offset < size {
+			n := size - offset
+			if n > common.MaxPartSize {
+				n = common.MaxPartSize
+			}
+			parts = append(parts, common.Part{
+				Path:       path,
+				FileSize:   size,
+				Offset:     offset,
+				Size:       n,
+				ActualSize: n,
+			})
+			offset += n
+		}
+	}
+	return parts, nil
+}
+
+// NewReadCloser returns io.ReadCloser for the given part p located in fs.
+func (fs *FS) NewReadCloser(p common.Part) (io.ReadCloser, error) {
+	path := fs.path(p)
+	r, err := filestream.OpenReaderAt(path, int64(p.Offset), true)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open %q at %q: %s", p.Path, fs.Dir, err)
+	}
+	lrc := &limitedReadCloser{
+		r: r,
+		n: p.Size,
+	}
+	return lrc, nil
+}
+
+// NewWriteCloser returns io.WriteCloser for the given part p located in fs.
+func (fs *FS) NewWriteCloser(p common.Part) (io.WriteCloser, error) {
+	path := fs.path(p)
+	if err := fs.mkdirAll(path); err != nil {
+		return nil, err
+	}
+	w, err := filestream.OpenWriterAt(path, int64(p.Offset), true)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open writer for %q at offset %d: %s", path, p.Offset, err)
+	}
+	wc := &writeCloser{
+		w:    w,
+		path: path,
+	}
+	return wc, nil
+}
+
+// DeletePath deletes the given path from fs and returns the size
+// for the deleted file.
+func (fs *FS) DeletePath(path string) (uint64, error) {
+	p := common.Part{
+		Path: path,
+	}
+	fullPath := fs.path(p)
+	f, err := os.Open(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// The file could be deleted earlier via symlink.
+			return 0, nil
+		}
+		return 0, fmt.Errorf("cannot open %q at %q: %s", path, fullPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return 0, fmt.Errorf("cannot stat %q at %q: %s", path, fullPath, err)
+	}
+	size := uint64(fi.Size())
+	if err := os.Remove(fullPath); err != nil {
+		return 0, fmt.Errorf("cannot remove %q: %s", fullPath, err)
+	}
+	return size, nil
+}
+
+// RemoveEmptyDirs recursively removes all the empty directories in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	return fscommon.RemoveEmptyDirs(fs.Dir)
+}
+
+func (fs *FS) mkdirAll(filePath string) error {
+	dir := filepath.Dir(filePath)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return fmt.Errorf("cannot create directory %q: %s", dir, err)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	dir := fs.Dir
+	for strings.HasSuffix(dir, "/") {
+		dir = dir[:len(dir)-1]
+	}
+	return fs.Dir + "/" + p.Path
+}
+
+type limitedReadCloser struct {
+	r *filestream.Reader
+	n uint64
+}
+
+func (lrc *limitedReadCloser) Read(p []byte) (int, error) {
+	if lrc.n == 0 {
+		return 0, io.EOF
+	}
+	if uint64(len(p)) > lrc.n {
+		p = p[:lrc.n]
+	}
+	n, err := lrc.r.Read(p)
+	if n > len(p) {
+		return n, fmt.Errorf("too much data read; got %d bytes; want %d bytes", n, len(p))
+	}
+	lrc.n -= uint64(n)
+	return n, err
+}
+
+func (lrc *limitedReadCloser) Close() error {
+	lrc.r.MustClose()
+	return nil
+}
+
+type writeCloser struct {
+	w    *filestream.Writer
+	path string
+}
+
+func (wc *writeCloser) Write(p []byte) (int, error) {
+	return wc.w.Write(p)
+}
+
+func (wc *writeCloser) Close() error {
+	wc.w.MustClose()
+	return fscommon.FsyncFile(wc.path)
+}

lib/backup/fsnil/fsnil.go

@@ -0,0 +1,20 @@
+package fsnil
+
+import (
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+// FS represents nil remote filesystem.
+type FS struct{}
+
+// String returns human-readable string representation for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fsnil")
+}
+
+// ListParts returns all the parts from fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	return nil, nil
+}

lib/backup/fsremote/fsremote.go

@@ -0,0 +1,190 @@
+package fsremote
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represents remote filesystem.
+//
+// Backups are uploaded there.
+// Data is downloaded from there during restore.
+type FS struct {
+	// Dir is a path to remote directory with backup data.
+	Dir string
+}
+
+// String returns human-readable string representation for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fsremote %q", fs.Dir)
+}
+
+// ListParts returns all the parts from fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			// Return empty part list for non-existing directory.
+			// The directory will be created later.
+			return nil, nil
+		}
+		return nil, err
+	}
+	files, err := fscommon.AppendFiles(nil, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var parts []common.Part
+	dir += "/"
+	for _, file := range files {
+		if !strings.HasPrefix(file, dir) {
+			logger.Panicf("BUG: unexpected prefix for file %q; want %q", file, dir)
+		}
+		var p common.Part
+		if !p.ParseFromRemotePath(file[len(dir):]) {
+			logger.Infof("skipping unknown file %s", file)
+			continue
+		}
+		// Check for correct part size.
+		fi, err := os.Stat(file)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat file %q for part %q: %s", file, p.Path, err)
+		}
+		p.ActualSize = uint64(fi.Size())
+		parts = append(parts, p)
+	}
+	return parts, nil
+}
+
+// DeletePart deletes the given part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	path := fs.path(p)
+	if err := os.Remove(path); err != nil {
+		return fmt.Errorf("cannot remove %q: %s", path, err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes all the empty directories in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	return fscommon.RemoveEmptyDirs(fs.Dir)
+}
+
+// CopyPart copies the part p from srcFS to fs.
+//
+// srcFS must have *FS type.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be fsremote", srcFS, fs)
+	}
+	srcPath := src.path(p)
+	dstPath := fs.path(p)
+	if err := fs.mkdirAll(dstPath); err != nil {
+		return err
+	}
+	// Attempt to create hardlink from srcPath to dstPath.
+	if err := os.Link(srcPath, dstPath); err == nil {
+		return fscommon.FsyncFile(dstPath)
+	}
+
+	// Cannot create hardlink. Just copy file contents
+	srcFile, err := os.Open(srcPath)
+	if err != nil {
+		return fmt.Errorf("cannot open file %q: %s", srcPath, err)
+	}
+	dstFile, err := os.Create(dstPath)
+	if err != nil {
+		_ = srcFile.Close()
+		return fmt.Errorf("cannot create file %q: %s", dstPath, err)
+	}
+	n, err := io.Copy(dstFile, srcFile)
+	if err1 := dstFile.Close(); err1 != nil {
+		err = err1
+	}
+	if err1 := srcFile.Close(); err1 != nil {
+		err = err1
+	}
+	if err != nil {
+		_ = os.RemoveAll(dstPath)
+		return err
+	}
+	if uint64(n) != p.Size {
+		_ = os.RemoveAll(dstPath)
+		return fmt.Errorf("unexpected number of bytes copied from %q to %q; got %d bytes; want %d bytes", srcPath, dstPath, n, p.Size)
+	}
+	if err := fscommon.FsyncFile(dstPath); err != nil {
+		_ = os.RemoveAll(dstPath)
+		return err
+	}
+	return nil
+}
+
+// DownloadPart download part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	path := fs.path(p)
+	r, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("cannot open %q: %s", path, err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download data from %q: %s", path, err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q; got %d bytes; want %d bytes", path, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	path := fs.path(p)
+	if err := fs.mkdirAll(path); err != nil {
+		return err
+	}
+	w, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("cannot create file %q: %s", path, err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := w.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("cannot upload data to %q: %s", path, err)
+	}
+	if uint64(n) != p.Size {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("wrong data size uploaded to %q; got %d bytes; want %d bytes", path, n, p.Size)
+	}
+	if err := fscommon.FsyncFile(path); err != nil {
+		_ = os.RemoveAll(path)
+		return err
+	}
+	return nil
+}
+
+func (fs *FS) mkdirAll(filePath string) error {
+	dir := filepath.Dir(filePath)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return fmt.Errorf("cannot create directory %q: %s", dir, err)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	return p.RemotePath(fs.Dir)
+}

lib/backup/gcsremote/gcs.go

@@ -0,0 +1,180 @@
+package gcsremote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+// FS represents filesystem for backups in GCS.
+//
+// Init must be called before calling other FS methods.
+type FS struct {
+	// Path to GCP credentials file.
+	//
+	// Default credentials are used if empty.
+	CredsFilePath string
+
+	// GCS bucket to use.
+	Bucket string
+
+	// Directory in the bucket to write to.
+	Dir string
+
+	bkt *storage.BucketHandle
+}
+
+// Init initializes fs.
+func (fs *FS) Init() error {
+	if fs.bkt != nil {
+		logger.Panicf("BUG: fs.Init has been already called")
+	}
+	for strings.HasPrefix(fs.Dir, "/") {
+		fs.Dir = fs.Dir[1:]
+	}
+	if !strings.HasSuffix(fs.Dir, "/") {
+		fs.Dir += "/"
+	}
+	ctx := context.Background()
+	var client *storage.Client
+	if len(fs.CredsFilePath) > 0 {
+		creds := option.WithCredentialsFile(fs.CredsFilePath)
+		c, err := storage.NewClient(ctx, creds)
+		if err != nil {
+			return fmt.Errorf("cannot create gcs client with credsFile %q: %s", fs.CredsFilePath, err)
+		}
+		client = c
+	} else {
+		c, err := storage.NewClient(ctx)
+		if err != nil {
+			return fmt.Errorf("cannot create default gcs client: %q", err)
+		}
+		client = c
+	}
+	fs.bkt = client.Bucket(fs.Bucket)
+	return nil
+}
+
+// String returns human-readable description for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("GCS{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	ctx := context.Background()
+	q := &storage.Query{
+		Prefix: dir,
+	}
+	it := fs.bkt.Objects(ctx, q)
+	var parts []common.Part
+	for {
+		attr, err := it.Next()
+		if err == iterator.Done {
+			return parts, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error when iterating objects at %q: %s", dir, err)
+		}
+		file := attr.Name
+		if !strings.HasPrefix(file, dir) {
+			return nil, fmt.Errorf("unexpected prefix for gcs key %q; want %q", file, dir)
+		}
+		var p common.Part
+		if !p.ParseFromRemotePath(file[len(dir):]) {
+			logger.Infof("skipping unknown object %q", file)
+			continue
+		}
+		p.ActualSize = uint64(attr.Size)
+		parts = append(parts, p)
+	}
+}
+
+// DeletePart deletes part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	if err := o.Delete(ctx); err != nil {
+		return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes empty dirs in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	// GCS has no directories, so nothing to remove.
+	return nil
+}
+
+// CopyPart copies p from srcFS to fs.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be GCS", srcFS, fs)
+	}
+	srcObj := src.object(p)
+	dstObj := fs.object(p)
+
+	copier := dstObj.CopierFrom(srcObj)
+	ctx := context.Background()
+	attr, err := copier.Run(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot copy %q from %s to %s: %s", p.Path, src, fs, err)
+	}
+	if uint64(attr.Size) != p.Size {
+		return fmt.Errorf("unexpected %q size after copying from %s to %s; got %d bytes; want %d bytes", p.Path, src, fs, attr.Size, p.Size)
+	}
+	return nil
+}
+
+// DownloadPart downloads part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	r, err := o.NewReader(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot open reader for %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download %q from at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads part p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	w := o.NewWriter(ctx)
+	n, err := io.Copy(w, r)
+	if err1 := w.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot upload data to %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size uploaded to %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+func (fs *FS) object(p common.Part) *storage.ObjectHandle {
+	path := p.RemotePath(fs.Dir)
+	return fs.bkt.Object(path)
+}

lib/backup/s3remote/s3.go

@@ -0,0 +1,220 @@
+package s3remote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// FS represents filesystem for backups in S3.
+//
+// Init must be called before calling other FS methods.
+type FS struct {
+	// Path to S3 credentials file.
+	CredsFilePath string
+
+	// Pat to S3 configs file.
+	ConfigFilePath string
+
+	// GCS bucket to use.
+	Bucket string
+
+	// Directory in the bucket to write to.
+	Dir string
+
+	s3       *s3.S3
+	uploader *s3manager.Uploader
+}
+
+// Init initializes fs.
+func (fs *FS) Init() error {
+	if fs.s3 != nil {
+		logger.Panicf("BUG: Init is already called")
+	}
+	for strings.HasPrefix(fs.Dir, "/") {
+		fs.Dir = fs.Dir[1:]
+	}
+	if !strings.HasSuffix(fs.Dir, "/") {
+		fs.Dir += "/"
+	}
+	opts := session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}
+	if len(fs.CredsFilePath) > 0 {
+		opts.SharedConfigFiles = []string{
+			fs.ConfigFilePath,
+			fs.CredsFilePath,
+		}
+	}
+	sess, err := session.NewSessionWithOptions(opts)
+	if err != nil {
+		return fmt.Errorf("cannot create S3 session: %s", err)
+	}
+
+	// Determine bucket region.
+	ctx := context.Background()
+	region, err := s3manager.GetBucketRegion(ctx, sess, fs.Bucket, "")
+	if err != nil {
+		return fmt.Errorf("cannot determine region for bucket %q: %s", fs.Bucket, err)
+	}
+	sess.Config.WithRegion(region)
+	logger.Infof("bucket %q is stored at region %q; switching to this region", fs.Bucket, region)
+
+	fs.s3 = s3.New(sess)
+	fs.uploader = s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
+		// We manage upload concurrency by ourselves.
+		u.Concurrency = 1
+	})
+	return nil
+}
+
+// String returns human-readable description for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("S3{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	input := &s3.ListObjectsV2Input{
+		Bucket: aws.String(fs.Bucket),
+		Prefix: aws.String(dir),
+	}
+	var errOuter error
+	var parts []common.Part
+	err := fs.s3.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, o := range page.Contents {
+			file := *o.Key
+			if !strings.HasPrefix(file, dir) {
+				errOuter = fmt.Errorf("unexpected prefix for s3 key %q; want %q", file, dir)
+				return false
+			}
+			var p common.Part
+			if !p.ParseFromRemotePath(file[len(dir):]) {
+				logger.Infof("skipping unknown object %q", file)
+				continue
+			}
+			p.ActualSize = uint64(*o.Size)
+			parts = append(parts, p)
+		}
+		return !lastPage
+	})
+	if errOuter != nil && err == nil {
+		err = errOuter
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error when listing s3 objects inside dir %q: %s", dir, err)
+	}
+	return parts, nil
+}
+
+// DeletePart deletes part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	path := fs.path(p)
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	_, err := fs.s3.DeleteObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes empty dirs in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	// S3 has no directories, so nothing to remove.
+	return nil
+}
+
+// CopyPart copies p from srcFS to fs.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be S3", srcFS, fs)
+	}
+	srcPath := src.path(p)
+	dstPath := fs.path(p)
+	copySource := fmt.Sprintf("/%s/%s", src.Bucket, srcPath)
+
+	input := &s3.CopyObjectInput{
+		Bucket:     aws.String(fs.Bucket),
+		CopySource: aws.String(copySource),
+		Key:        aws.String(dstPath),
+	}
+	_, err := fs.s3.CopyObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot copy %q from %s to %s (copySource %q): %s", p.Path, src, fs, copySource, err)
+	}
+	return nil
+}
+
+// DownloadPart downloads part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	path := fs.path(p)
+	input := &s3.GetObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	o, err := fs.s3.GetObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot open %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	r := o.Body
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download %q from at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads part p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	path := fs.path(p)
+	sr := &statReader{
+		r: r,
+	}
+	input := &s3manager.UploadInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+		Body:   sr,
+	}
+	_, err := fs.uploader.Upload(input)
+	if err != nil {
+		return fmt.Errorf("cannot upoad data to %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	if uint64(sr.size) != p.Size {
+		return fmt.Errorf("wrong data size uploaded to %q at %s; got %d bytes; want %d bytes", p.Path, fs, sr.size, p.Size)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	return p.RemotePath(fs.Dir)
+}
+
+type statReader struct {
+	r    io.Reader
+	size int64
+}
+
+func (sr *statReader) Read(p []byte) (int, error) {
+	n, err := sr.r.Read(p)
+	sr.size += int64(n)
+	return n, err
+}