lib/logger: prevent from blocking when log output isn't consumed in timely manner

Drop log messages instead of blocking and increment `vm_log_messages_dropped_total` metric.

.github/workflows/github-pages.yml

@@ -0,0 +1,30 @@
+name: github-pages
+on:
+  push:
+    paths:
+      - 'docs/*.md'
+      - 'README.md'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git gpages
+          cp docs/*.md gpages
+          cp README.md gpages
+          cd gpages
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add "*.md"
+          git commit -m "update github pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf gpages

.github/workflows/main.yml

@@ -1,7 +1,13 @@
 name: main
 on:
-  - push
-  - pull_request
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
 jobs:
   build:
     name: Build
@@ -24,20 +30,21 @@ jobs:
         env:
           GO111MODULE: on
         run: |
-            export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
-            make check-all
-            git diff --exit-code
-            make test-full
-            make test-pure
-            make test-full-386
-            make victoria-metrics
-            make victoria-metrics-pure
-            make victoria-metrics-arm
-            make victoria-metrics-arm64
-            GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
-            GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+          make check-all
+          git diff --exit-code
+          make test-full
+          make test-pure
+          make test-full-386
+          make victoria-metrics
+          make victoria-metrics-pure
+          make victoria-metrics-arm
+          make victoria-metrics-arm64
+          make vmutils
+          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
       - name: Publish coverage
-        uses: codecov/codecov-action@v1.0.0
+        uses: codecov/codecov-action@v1.0.4
         with:
           token: ${{secrets.CODECOV_TOKEN}}
           file: ./coverage.txt

.github/workflows/wiki.yml

@@ -0,0 +1,29 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*.md'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          cd docs
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          find ./ -name '*.md' -exec cp -prv '{}' 'wiki' ';'
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add "*.md"
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -1,3 +1,4 @@
+/tmp
 /tags
 /pkg
 *.pprof

Makefile

@@ -19,12 +19,36 @@ include deployment/*/Makefile
 clean:
 	rm -rf bin/*
 
-publish: publish-victoria-metrics
+publish: \
+	publish-victoria-metrics \
+	publish-vmbackup \
+	publish-vmrestore
 
-package: package-victoria-metrics
+package: \
+	package-victoria-metrics \
+	package-vmbackup \
+	package-vmrestore
 
-release: victoria-metrics-prod
-	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod
+vmutils: \
+	vmbackup \
+	vmrestore
+
+release: \
+	release-victoria-metrics \
+	release-vmutils
+
+release-victoria-metrics: victoria-metrics-prod
+	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod && \
+		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
+
+release-vmutils: \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmbackup-prod vmrestore-prod && \
+		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
 	GO111MODULE=on gofmt -l -w -s ./lib
@@ -39,13 +63,15 @@ lint: install-golint
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get -u github.com/golang/lint/golint
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
 
 install-errcheck:
 	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck

README.md

@@ -1,31 +1,40 @@
 [![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+[![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics.svg?maxAge=604800)](https://hub.docker.com/r/victoriametrics/victoria-metrics)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](http://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 
-<img alt="Victoria Metrics" src="logo.png">
+<img alt="Victoria Metrics" src="logo.png" height="200px">
 
-## Single-node VictoriaMetrics
+## VictoriaMetrics
 
 VictoriaMetrics is fast, cost-effective and scalable time-series database. It can be used as long-term remote storage for Prometheus.
 It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
 [docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and
-in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
+in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
 
 Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 
+## Case studies
+
+* [Wix.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wixcom)
+* [Wedos.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wedoscom)
+* [Dreamteam](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#dreamteam)
+
+
 ## Prominent features
 
 * Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
-  Additionally, VictoriaMetrics extends PromQL with opt-in [useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
+  VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL) query language, which is inspired by PromQL.
 * Supports global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
 * High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
   and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
   [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 * [Uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) when working with millions of unique time series (aka high cardinality).
+* Optimized for time series with high churn rate. Think about [prometheus-operator](https://github.com/coreos/prometheus-operator) metrics from frequent deployments in Kubernetes.
 * High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
   may be crammed into limited storage comparing to TimescaleDB.
 * Optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
@@ -33,19 +42,22 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
   and [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
 * Easy operation:
-  * VictoriaMetrics consists of a single executable without external dependencies.
+  * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
   * All the data is stored in a single directory pointed by `-storageDataPath` flag.
-  * Easy backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-* Storage is protected from corruption on unclean shutdown (i.e. hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+  to S3 or GCS with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) / [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+  See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+* Storage is protected from corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 * Supports metrics' ingestion and [backfilling](#backfilling) via the following protocols:
   * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-  * [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
-  * [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
+  * [InfluxDB line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+  * [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
     if `-graphiteListenAddr` is set.
-  * [OpenTSDB put message](http://opentsdb.net/docs/build/html/api_telnet/put.html) if `-opentsdbListenAddr` is set.
-  * [HTTP OpenTSDB /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) if `-opentsdbHTTPListenAddr` is set.
-* Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry and various Enterprise workloads.
+  * [OpenTSDB put message](#sending-data-via-telnet-put-protocol) if `-opentsdbListenAddr` is set.
+  * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests) if `-opentsdbHTTPListenAddr` is set.
+  * [/api/v1/import](#how-to-import-time-series-data)
+* Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various Enterprise workloads.
 * Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 
@@ -63,6 +75,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   - [How to send data from Graphite-compatible agents such as StatsD?](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
   - [Querying Graphite data](#querying-graphite-data)
   - [How to send data from OpenTSDB-compatible agents?](#how-to-send-data-from-opentsdb-compatible-agents)
+  - [Prometheus querying API usage](#prometheus-querying-api-usage)
   - [How to build from sources](#how-to-build-from-sources)
     - [Development build](#development-build)
     - [Production build](#production-build)
@@ -75,6 +88,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   - [How to work with snapshots?](#how-to-work-with-snapshots)
   - [How to delete time series?](#how-to-delete-time-series)
   - [How to export time series?](#how-to-export-time-series)
+  - [How to import time series data?](#how-to-import-time-series-data)
   - [Federation](#federation)
   - [Capacity planning](#capacity-planning)
   - [High availability](#high-availability)
@@ -220,7 +234,7 @@ VictoriaMetrics maps Influx data using the following rules:
   unless `db` tag exists in the Influx line.
 * Field names are mapped to time series names prefixed with `{measurement}{separator}` value,
   where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag.
-  See also `-influxSkipSingleField` command-line flag.
+  See also `-influxSkipSingleField` command-line flag. If `{measurement}` is empty, then time series names correspond to field names.
 * Field values are mapped to time series values.
 * Tags are mapped to Prometheus labels as-is.
 
@@ -248,7 +262,7 @@ An arbitrary number of lines delimited by '\n' may be sent in a single request.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"measurement_.*"}'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
@@ -286,7 +300,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
@@ -299,7 +313,7 @@ The `/api/v1/export` endpoint should return the following response:
 ### Querying Graphite data
 
 Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read either via
-[Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/)
+[Prometheus querying API](#prometheus-querying-api-usage)
 or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
 
 
@@ -331,7 +345,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
 
 ```
-curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}'
+curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
 ```
 
 The `/api/v1/export` endpoint should return the following response:
@@ -379,6 +393,31 @@ The `/api/v1/export` endpoint should return the following response:
 ```
 
 
+### Prometheus querying API usage
+
+VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
+
+* [/api/v1/query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries)
+* [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)
+* [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
+* [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
+* [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
+
+These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
+
+VictoriaMetrics accepts additional args for `/api/v1/labels` and `/api/v1/label/.../values` handlers.
+See [this feature request](https://github.com/prometheus/prometheus/issues/6178) for details:
+
+* Any number [time series selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) via `match[]` query arg.
+* Optional `start` and `end` query args for limiting the time range for the selected labels or label values.
+
+Additionally VictoriaMetrics provides the following handlers:
+
+* `/api/v1/series/count` - it returns the total number of time series in the database. Note that this handler scans all the inverted index,
+  so it can be slow if the database contains tens of millions of time series.
+* `/api/v1/labels/count` - it returns a list of `label: values_count` entries. It can be used for determining labels with the maximum number of values.
+
+
 ### How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -461,8 +500,8 @@ The page will return the following JSON response:
 ```
 
 Snapshots are created under `<-storageDataPath>/snapshots` directory, where `<-storageDataPath>`
-is the command-line flag value. Snapshots can be archived to backup storage via `cp -L`, `rsync -L`, `scp -r`
-or any similar tool that follows symlinks during copying.
+is the command-line flag value. Snapshots can be archived to backup storage at any time
+with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
 
 The `http://<victoriametrics-addr>:8428/snapshot/list` page contains the list of available snapshots.
 
@@ -473,9 +512,9 @@ Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete_all` in order to
 
 Steps for restoring from a snapshot:
 1. Stop VictoriaMetrics with `kill -INT`.
-2. Remove the entire contents of the directory pointed by `-storageDataPath` command-line flag.
-3. Copy snapshot contents to the directory pointed by `-storageDataPath`.
-4. Start VictoriaMetrics.
+2. Restore snapshot contents from backup with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md)
+   to the directory pointed by `-storageDataPath`.
+3. Start VictoriaMetrics.
 
 
 ### How to delete time series?
@@ -493,7 +532,8 @@ before actually deleting the metrics.
 
 Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
 where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
-for metrics to export. The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
+for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
+The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
 Each JSON line would contain data for a single time series. An example output:
 
 ```
@@ -504,6 +544,33 @@ Each JSON line would contain data for a single time series. An example output:
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
 
+Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-time-series-data).
+
+
+### How to import time series data?
+
+Time series data can be imported via any supported ingestion protocol:
+
+* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+* [Influx line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+* [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
+* [OpenTSDB telnet put protocol](#sending-data-via-telnet-put-protocol)
+* [OpenTSDB http /api/put](#sending-opentsdb-data-via-http-apiput-requests)
+* `/api/v1/import` http POST handler, which accepts data from [/api/v1/export](#how-to-export-time-series).
+
+The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import`. Example for importing data obtained via `/api/v1/export`:
+
+```
+# Export the data from <source-victoriametrics>:
+curl -s 'http://source-victoriametrics:8428/api/v1/export' -d 'match={__name__!=""}' > exported_data.jsonl
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST 'http://destination-victoriametrics:8428/api/v1/import' -T exported_data.jsonl
+```
+
+Each request to `/api/v1/import` can load up to a single vCPU core on VictoriaMetrics. Import speed can be improved by splitting the original file into smaller parts
+and importing them concurrently. Note that the original file must be split on newlines.
+
 
 ### Federation
 
@@ -667,12 +734,13 @@ mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
 
 VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page.
 Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics.
-There is [an official Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229).
+There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
 
 The most interesting metrics are:
 
 * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
   aka active time series.
+* `rate(vm_new_timeseries_created_total[5m])` - time series churn rate.
 * `vm_rows{type="indexdb"}` - the number of rows in inverted index. High value for this number usually mean high churn rate for time series.
 * Sum of `vm_rows{type="storage/big"}` and `vm_rows{type="storage/small"}` - total number of `(timestamp, value)` data points
   in the database.
@@ -738,6 +806,7 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
   See [these docs](https://github.com/netdata/netdata#integrations).
 * [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
   See [this example](/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+* [Ansible role for installing VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
 
 
 ## Roadmap

app/victoria-metrics/Makefile

@@ -6,9 +6,38 @@ victoria-metrics:
 victoria-metrics-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker
 
+victoria-metrics-pure-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure
+
+victoria-metrics-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+
+victoria-metrics-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+
+victoria-metrics-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+
+victoria-metrics-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+
 package-victoria-metrics:
-	APP_NAME=victoria-metrics \
-	$(MAKE) package-via-docker
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker
+
+package-victoria-metrics-pure:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-pure
+
+package-victoria-metrics-arm:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm
+
+package-victoria-metrics-arm64:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm64
+
+package-victoria-metrics-ppc64le:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-ppc64le
+
+package-victoria-metrics-386:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-386
 
 publish-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) publish-via-docker
@@ -23,27 +52,18 @@ run-victoria-metrics:
 victoria-metrics-arm:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
 
-victoria-metrics-arm-prod:
-	APP_NAME=victoria-metrics APP_SUFFIX='-arm' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm' $(MAKE) app-via-docker
-
 victoria-metrics-arm64:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
 
-victoria-metrics-arm64-prod:
-	APP_NAME=victoria-metrics APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker
+victoria-metrics-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
 
 victoria-metrics-386:
 	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
 
-victoria-metrics-386-prod:
-	APP_NAME=victoria-metrics APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker
-
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 
-victoria-metrics-pure-prod:
-	APP_NAME=victoria-metrics APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
-
 ### Packaging as DEB - amd64
 victoria-metrics-package-deb: victoria-metrics-prod
 	./package/package_deb.sh amd64

app/victoria-metrics/deployment/Dockerfile

@@ -1,5 +1,8 @@
+ARG certs_image
+FROM $certs_image AS certs
 FROM scratch
-COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY bin/victoria-metrics-prod .
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./victoria-metrics-prod
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]

app/victoria-metrics/main.go

@@ -21,7 +21,7 @@ func main() {
 	flag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	logger.Infof("starting VictoraMetrics at %q...", *httpListenAddr)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	vmstorage.Init()
 	vmselect.Init()

app/victoria-metrics/main_test.go

@@ -186,7 +186,6 @@ func tearDown() {
 	vmstorage.Stop()
 	vmselect.Stop()
 	fs.MustRemoveAll(storagePath)
-	fs.MustStopDirRemover()
 }
 
 func TestWriteRead(t *testing.T) {

app/victoria-metrics/test/parser.go

@@ -41,7 +41,7 @@ func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
 		case `TIME_NS`:
 			return []byte(fmt.Sprintf("%d", t.UnixNano()))
 		default:
-			log.Fatalf("unkown time pattern %s in %s", parts[0], repl)
+			log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
 		}
 		return repl
 	})

app/vmbackup/Makefile

@@ -0,0 +1,58 @@
+# All these commands must run from repository root.
+
+vmbackup:
+	APP_NAME=vmbackup $(MAKE) app-local
+
+vmbackup-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker
+
+vmbackup-pure-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-pure
+
+vmbackup--arm-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm
+
+vmbackup-arm64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm64
+
+vmbackup-ppc64le-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-ppc64le
+
+vmbackup-386-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-386
+
+package-vmbackup:
+	APP_NAME=vmbackup $(MAKE) package-via-docker
+
+package-vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-pure
+
+package-vmbackup-arm:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm
+
+package-vmbackup-arm64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm64
+
+package-vmbackup-ppc64le:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le
+
+package-vmbackup-386:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-386
+
+publish-vmbackup:
+	APP_NAME=vmbackup $(MAKE) publish-via-docker
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure
+
+vmbackup-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+
+vmbackup-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+
+vmbackup-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup
+
+vmbackup-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup

app/vmbackup/README.md

@@ -0,0 +1,181 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.

app/vmbackup/deployment/Dockerfile

@@ -0,0 +1,7 @@
+ARG certs_image
+FROM $certs_image AS certs
+FROM scratch
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./vmbackup-prod
+ENTRYPOINT ["/vmbackup-prod"]

app/vmbackup/main.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName    = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	dst             = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
+	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
+	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+)
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	originFS, err := newOriginFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Backup{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+		Origin:      originFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+or local filesystem. Backed up data can be restored with vmrestore.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newSrcFS() (*fslocal.FS, error) {
+	if len(*snapshotName) == 0 {
+		return nil, fmt.Errorf("`-snapshotName` cannot be empty")
+	}
+	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
+
+	// Verify the snapshot exists.
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
+	}
+
+	fs := &fslocal.FS{
+		Dir:               snapshotPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newDstFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*dst)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+	}
+	return fs, nil
+}
+
+func newOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, nil
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+	}
+	return fs, nil
+}

app/vminsert/common/insert_ctx.go

@@ -47,7 +47,7 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
 	return metricNameRaw[:len(metricNameRaw):len(metricNameRaw)]
 }
 
-// WriteDataPoint writes (timestamp, value) with the given prefix and lables into ctx buffer.
+// WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
 func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) {
 	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
 	ctx.addRow(metricNameRaw, timestamp, value)
@@ -78,6 +78,26 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.Value = value
 }
 
+// AddLabelBytes adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = name
+	label.Value = value
+
+	ctx.Labels = labels
+}
+
 // AddLabel adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.

app/vminsert/common/lines_reader.go

@@ -20,6 +20,17 @@ const defaultBlockSize = 64 * 1024
 //
 // Returns (dstBuf, tailBuf).
 func ReadLinesBlock(r io.Reader, dstBuf, tailBuf []byte) ([]byte, []byte, error) {
+	return ReadLinesBlockExt(r, dstBuf, tailBuf, maxLineSize)
+}
+
+// ReadLinesBlockExt reads a block of lines delimited by '\n' from tailBuf and r into dstBuf.
+//
+// Trailing chars after the last newline are put into tailBuf.
+//
+// Returns (dstBuf, tailBuf).
+//
+// maxLineLen limits the maximum length of a single line.
+func ReadLinesBlockExt(r io.Reader, dstBuf, tailBuf []byte, maxLineLen int) ([]byte, []byte, error) {
 	if cap(dstBuf) < defaultBlockSize {
 		dstBuf = bytesutil.Resize(dstBuf, defaultBlockSize)
 	}
@@ -48,8 +59,8 @@ again:
 	nn := bytes.LastIndexByte(dstBuf[len(dstBuf)-n:], '\n')
 	if nn < 0 {
 		// Didn't found at least a single line.
-		if len(dstBuf) > maxLineSize {
-			return dstBuf, tailBuf, fmt.Errorf("too long line: more than %d bytes", maxLineSize)
+		if len(dstBuf) > maxLineLen {
+			return dstBuf, tailBuf, fmt.Errorf("too long line: more than %d bytes", maxLineLen)
 		}
 		if cap(dstBuf) < 2*len(dstBuf) {
 			// Increase dsbBuf capacity, so more data could be read into it.

app/vminsert/graphite/request_handler.go

@@ -61,13 +61,13 @@ func (ctx *pushCtx) InsertRows() error {
 const flushTimeout = 3 * time.Second
 
 func (ctx *pushCtx) Read(r io.Reader) bool {
-	graphiteReadCalls.Inc()
+	readCalls.Inc()
 	if ctx.err != nil {
 		return false
 	}
 	if c, ok := r.(net.Conn); ok {
 		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			graphiteReadErrors.Inc()
+			readErrors.Inc()
 			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
 			return false
 		}
@@ -79,7 +79,7 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 			ctx.err = nil
 		} else {
 			if ctx.err != io.EOF {
-				graphiteReadErrors.Inc()
+				readErrors.Inc()
 				ctx.err = fmt.Errorf("cannot read graphite plaintext protocol data: %s", ctx.err)
 			}
 			return false
@@ -132,8 +132,8 @@ func (ctx *pushCtx) reset() {
 }
 
 var (
-	graphiteReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
-	graphiteReadErrors = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
+	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
+	readErrors = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
 )
 
 func getPushCtx() *pushCtx {

app/vminsert/graphite/server.go

@@ -21,36 +21,62 @@ var (
 	writeErrorsUDP   = metrics.NewCounter(`vm_graphite_request_errors_total{name="write", net="udp"}`)
 )
 
-// Serve starts graphite server on the given addr.
-func Serve(addr string) {
+// Server accepts Graphite plaintext lines over TCP and UDP.
+type Server struct {
+	addr  string
+	lnTCP net.Listener
+	lnUDP net.PacketConn
+	wg    sync.WaitGroup
+}
+
+// MustStart starts graphite server on the given addr.
+//
+// MustStop must be called on the returned server when it is no longer needed.
+func MustStart(addr string) *Server {
 	logger.Infof("starting TCP Graphite server at %q", addr)
 	lnTCP, err := netutil.NewTCPListener("graphite", addr)
 	if err != nil {
 		logger.Fatalf("cannot start TCP Graphite server at %q: %s", addr, err)
 	}
-	listenerTCP = lnTCP
 
 	logger.Infof("starting UDP Graphite server at %q", addr)
 	lnUDP, err := net.ListenPacket("udp4", addr)
 	if err != nil {
 		logger.Fatalf("cannot start UDP Graphite server at %q: %s", addr, err)
 	}
-	listenerUDP = lnUDP
 
-	var wg sync.WaitGroup
-	wg.Add(1)
+	s := &Server{
+		addr:  addr,
+		lnTCP: lnTCP,
+		lnUDP: lnUDP,
+	}
+	s.wg.Add(1)
 	go func() {
-		defer wg.Done()
-		serveTCP(listenerTCP)
+		defer s.wg.Done()
+		serveTCP(lnTCP)
 		logger.Infof("stopped TCP Graphite server at %q", addr)
 	}()
-	wg.Add(1)
+	s.wg.Add(1)
 	go func() {
-		defer wg.Done()
-		serveUDP(listenerUDP)
+		defer s.wg.Done()
+		serveUDP(lnUDP)
 		logger.Infof("stopped UDP Graphite server at %q", addr)
 	}()
-	wg.Wait()
+	return s
+}
+
+// MustStop stops the server.
+func (s *Server) MustStop() {
+	logger.Infof("stopping TCP Graphite server at %q...", s.addr)
+	if err := s.lnTCP.Close(); err != nil {
+		logger.Errorf("cannot close TCP Graphite server: %s", err)
+	}
+	logger.Infof("stopping UDP Graphite server at %q...", s.addr)
+	if err := s.lnUDP.Close(); err != nil {
+		logger.Errorf("cannot close UDP Graphite server: %s", err)
+	}
+	s.wg.Wait()
+	logger.Infof("TCP and UDP Graphite servers at %q have been stopped", s.addr)
 }
 
 func serveTCP(ln net.Listener) {
@@ -59,6 +85,7 @@ func serveTCP(ln net.Listener) {
 		if err != nil {
 			if ne, ok := err.(net.Error); ok {
 				if ne.Temporary() {
+					logger.Errorf("graphite: temporary error when listening for TCP addr %q: %s", ln.Addr(), err)
 					time.Sleep(time.Second)
 					continue
 				}
@@ -97,6 +124,7 @@ func serveUDP(ln net.PacketConn) {
 					writeErrorsUDP.Inc()
 					if ne, ok := err.(net.Error); ok {
 						if ne.Temporary() {
+							logger.Errorf("graphite: temporary error when listening for UDP addr %q: %s", ln.LocalAddr(), err)
 							time.Sleep(time.Second)
 							continue
 						}
@@ -119,20 +147,3 @@ func serveUDP(ln net.PacketConn) {
 	}
 	wg.Wait()
 }
-
-var (
-	listenerTCP net.Listener
-	listenerUDP net.PacketConn
-)
-
-// Stop stops the server.
-func Stop() {
-	logger.Infof("stopping TCP Graphite server at %q...", listenerTCP.Addr())
-	if err := listenerTCP.Close(); err != nil {
-		logger.Errorf("cannot close TCP Graphite server: %s", err)
-	}
-	logger.Infof("stopping UDP Graphite server at %q...", listenerUDP.LocalAddr())
-	if err := listenerUDP.Close(); err != nil {
-		logger.Errorf("cannot close UDP Graphite server: %s", err)
-	}
-}

app/vminsert/influx/parser.go

@@ -85,9 +85,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag, fieldsPool []Field, noEscapeCh
 		measurementTags = measurementTags[:n]
 	}
 	r.Measurement = unescapeTagValue(measurementTags, noEscapeChars)
-	if len(r.Measurement) == 0 {
-		return tagsPool, fieldsPool, fmt.Errorf("measurement cannot be empty. measurementTags=%q", s)
-	}
+	// Allow empty r.Measurement. In this case metric name is constructed directly from field keys.
 
 	// Parse fields
 	fieldsStart := len(fieldsPool)

app/vminsert/influx/parser_test.go

@@ -86,9 +86,6 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 		}
 	}
 
-	// Missing measurement
-	f(",foo=bar baz=123")
-
 	// No fields
 	f("foo")
 	f("foo,bar=baz 1234")
@@ -147,6 +144,30 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 	f("#foobar baz", &Rows{})
 	f("#foobar baz\n#sss", &Rows{})
 
+	// Missing measurement
+	f(" baz=123", &Rows{
+		Rows: []Row{{
+			Measurement: "",
+			Fields: []Field{{
+				Key:   "baz",
+				Value: 123,
+			}},
+		}},
+	})
+	f(",foo=bar baz=123", &Rows{
+		Rows: []Row{{
+			Measurement: "",
+			Tags: []Tag{{
+				Key:   "foo",
+				Value: "bar",
+			}},
+			Fields: []Field{{
+				Key:   "baz",
+				Value: 123,
+			}},
+		}},
+	})
+
 	// Minimal line without tags and timestamp
 	f("foo bar=123", &Rows{
 		Rows: []Row{{
@@ -327,6 +348,36 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			}},
 		}},
 	})
+	// Test case from https://community.librenms.org/t/integration-with-victoriametrics/9689
+	f("ports,foo=a,bar=et\\ +\\ V,baz=ype INDISCARDS=245333676,OUTDISCARDS=1798680", &Rows{
+		Rows: []Row{{
+			Measurement: "ports",
+			Tags: []Tag{
+				{
+					Key:   "foo",
+					Value: "a",
+				},
+				{
+					Key:   "bar",
+					Value: "et + V",
+				},
+				{
+					Key:   "baz",
+					Value: "ype",
+				},
+			},
+			Fields: []Field{
+				{
+					Key:   "INDISCARDS",
+					Value: 245333676,
+				},
+				{
+					Key:   "OUTDISCARDS",
+					Value: 1798680,
+				},
+			},
+		}},
+	})
 
 	// Multiple lines
 	f("foo,tag=xyz field=1.23 48934\n"+

app/vminsert/influx/request_handler.go

@@ -36,7 +36,7 @@ func InsertHandler(req *http.Request) error {
 }
 
 func insertHandlerInternal(req *http.Request) error {
-	influxReadCalls.Inc()
+	readCalls.Inc()
 
 	r := req.Body
 	if req.Header.Get("Content-Encoding") == "gzip" {
@@ -82,7 +82,7 @@ func (ctx *pushCtx) InsertRows(db string) error {
 	rows := ctx.Rows.Rows
 	rowsLen := 0
 	for i := range rows {
-		rowsLen += len(rows[i].Tags)
+		rowsLen += len(rows[i].Fields)
 	}
 	ic := &ctx.Common
 	ic.Reset(rowsLen)
@@ -104,7 +104,7 @@ func (ctx *pushCtx) InsertRows(db string) error {
 		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
 		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
 		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
-		if !skipFieldKey {
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
 			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
 		}
 		metricGroupPrefixLen := len(ctx.metricGroupBuf)
@@ -132,7 +132,7 @@ func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
 	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlock(r, ctx.reqBuf, ctx.tailBuf)
 	if ctx.err != nil {
 		if ctx.err != io.EOF {
-			influxReadErrors.Inc()
+			readErrors.Inc()
 			ctx.err = fmt.Errorf("cannot read influx line protocol data: %s", ctx.err)
 		}
 		return false
@@ -166,8 +166,8 @@ func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
 }
 
 var (
-	influxReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
-	influxReadErrors = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
+	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
+	readErrors = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
 )
 
 type pushCtx struct {

app/vminsert/main.go

@@ -12,45 +12,54 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	graphiteListenAddr     = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	opentsdbListenAddr     = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
 	maxInsertRequestSize   = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
 )
 
+var (
+	graphiteServer     *graphite.Server
+	opentsdbServer     *opentsdb.Server
+	opentsdbhttpServer *opentsdbhttp.Server
+)
+
 // Init initializes vminsert.
 func Init() {
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
 
 	concurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
-		go graphite.Serve(*graphiteListenAddr)
+		graphiteServer = graphite.MustStart(*graphiteListenAddr)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		go opentsdb.Serve(*opentsdbListenAddr)
+		opentsdbServer = opentsdb.MustStart(*opentsdbListenAddr, int64(*maxInsertRequestSize))
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		go opentsdbhttp.Serve(*opentsdbHTTPListenAddr, int64(*maxInsertRequestSize))
+		opentsdbhttpServer = opentsdbhttp.MustStart(*opentsdbHTTPListenAddr, int64(*maxInsertRequestSize))
 	}
 }
 
 // Stop stops vminsert.
 func Stop() {
 	if len(*graphiteListenAddr) > 0 {
-		graphite.Stop()
+		graphiteServer.MustStop()
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdb.Stop()
+		opentsdbServer.MustStop()
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttp.Stop()
+		opentsdbhttpServer.MustStop()
 	}
 }
 
@@ -67,6 +76,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandler(r); err != nil {
@@ -92,6 +110,9 @@ var (
 	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/write", protocol="prometheus"}`)
 	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/write", protocol="prometheus"}`)
 
+	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import", protocol="vm"}`)
+	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import", protocol="vm"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 

app/vminsert/opentsdb/listener_switch.go

@@ -0,0 +1,159 @@
+package opentsdb
+
+import (
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// listenerSwitch listens for incoming connections and multiplexes them to OpenTSDB http or telnet listeners
+// depending on the first byte in the accepted connection.
+//
+// It is expected that both listeners - http and telnet consume incoming connections as soon as possible.
+type listenerSwitch struct {
+	ln net.Listener
+	wg sync.WaitGroup
+
+	telnetConnsCh chan net.Conn
+	httpConnsCh   chan net.Conn
+
+	closeLock sync.Mutex
+	closed    bool
+	acceptErr error
+	closeErr  error
+}
+
+func newListenerSwitch(ln net.Listener) *listenerSwitch {
+	ls := &listenerSwitch{
+		ln: ln,
+	}
+	ls.telnetConnsCh = make(chan net.Conn)
+	ls.httpConnsCh = make(chan net.Conn)
+	ls.wg.Add(1)
+	go func() {
+		ls.worker()
+		close(ls.telnetConnsCh)
+		close(ls.httpConnsCh)
+		ls.wg.Done()
+	}()
+	return ls
+}
+
+func (ls *listenerSwitch) stop() error {
+	var err error
+	ls.closeLock.Lock()
+	if !ls.closed {
+		err = ls.ln.Close()
+		ls.closeErr = err
+		ls.closed = true
+	}
+	ls.closeLock.Unlock()
+
+	if err == nil {
+		// Wait until worker detects the closed ls.ln and exits.
+		ls.wg.Wait()
+	}
+	return err
+}
+
+func (ls *listenerSwitch) worker() {
+	var buf [1]byte
+	for {
+		c, err := ls.ln.Accept()
+		if err != nil {
+			if ne, ok := err.(net.Error); ok && ne.Temporary() {
+				logger.Infof("listenerSwitch: temporary error at %q: %s; sleeping for a second...", ls.ln.Addr(), err)
+				time.Sleep(time.Second)
+				continue
+			}
+			ls.closeLock.Lock()
+			ls.acceptErr = err
+			ls.closeLock.Unlock()
+			return
+		}
+		if _, err := io.ReadFull(c, buf[:]); err != nil {
+			logger.Errorf("listenerSwitch: cannot read one byte from the underlying connection for %q: %s", ls.ln.Addr(), err)
+			_ = c.Close()
+			continue
+		}
+
+		// It is expected that both listeners - http and telnet consume incoming connections as soon as possible,
+		// so the below code shouldn't block for extended periods of time.
+		pc := &peekedConn{
+			Conn:      c,
+			firstChar: buf[0],
+		}
+		if buf[0] == 'p' {
+			// Assume the request starts with `put`.
+			ls.telnetConnsCh <- pc
+		} else {
+			// Assume the request starts with `POST`.
+			ls.httpConnsCh <- pc
+		}
+	}
+}
+
+type peekedConn struct {
+	net.Conn
+	firstChar     byte
+	firstCharRead bool
+}
+
+func (pc *peekedConn) Read(p []byte) (int, error) {
+	// It is assumed that the pc cannot be read from concurrent goroutines.
+	if pc.firstCharRead {
+		// Fast path - first char already read.
+		return pc.Conn.Read(p)
+	}
+
+	// Slow path - read the first char.
+	if len(p) == 0 {
+		return 0, nil
+	}
+	p[0] = pc.firstChar
+	pc.firstCharRead = true
+	n, err := pc.Conn.Read(p[1:])
+	return n + 1, err
+}
+
+func (ls *listenerSwitch) newTelnetListener() *chanListener {
+	return &chanListener{
+		ls: ls,
+		ch: ls.telnetConnsCh,
+	}
+}
+
+func (ls *listenerSwitch) newHTTPListener() *chanListener {
+	return &chanListener{
+		ls: ls,
+		ch: ls.httpConnsCh,
+	}
+}
+
+type chanListener struct {
+	ls *listenerSwitch
+	ch chan net.Conn
+}
+
+func (cl *chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl.ch
+	if ok {
+		return c, nil
+	}
+
+	cl.ls.closeLock.Lock()
+	err := cl.ls.acceptErr
+	cl.ls.closeLock.Unlock()
+	return nil, err
+}
+
+func (cl *chanListener) Close() error {
+	return cl.ls.stop()
+}
+
+func (cl *chanListener) Addr() net.Addr {
+	return cl.ls.ln.Addr()
+}

app/vminsert/opentsdb/request_handler.go

@@ -61,13 +61,13 @@ func (ctx *pushCtx) InsertRows() error {
 const flushTimeout = 3 * time.Second
 
 func (ctx *pushCtx) Read(r io.Reader) bool {
-	opentsdbReadCalls.Inc()
+	readCalls.Inc()
 	if ctx.err != nil {
 		return false
 	}
 	if c, ok := r.(net.Conn); ok {
 		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			opentsdbReadErrors.Inc()
+			readErrors.Inc()
 			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
 			return false
 		}
@@ -79,7 +79,7 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 			ctx.err = nil
 		} else {
 			if ctx.err != io.EOF {
-				opentsdbReadErrors.Inc()
+				readErrors.Inc()
 				ctx.err = fmt.Errorf("cannot read OpenTSDB put protocol data: %s", ctx.err)
 			}
 			return false
@@ -131,8 +131,8 @@ func (ctx *pushCtx) reset() {
 }
 
 var (
-	opentsdbReadCalls  = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
-	opentsdbReadErrors = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
+	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
+	readErrors = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
 )
 
 func getPushCtx() *pushCtx {

app/vminsert/opentsdb/server.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -21,44 +22,91 @@ var (
 	writeErrorsUDP   = metrics.NewCounter(`vm_opentsdb_request_errors_total{name="write", net="udp"}`)
 )
 
-// Serve starts OpenTSDB collector on the given addr.
-func Serve(addr string) {
+// Server is a server for collecting OpenTSDB TCP and UDP metrics.
+//
+// It accepts simultaneously Telnet put requests and HTTP put requests over TCP.
+type Server struct {
+	addr       string
+	ls         *listenerSwitch
+	httpServer *opentsdbhttp.Server
+	lnUDP      net.PacketConn
+	wg         sync.WaitGroup
+}
+
+// MustStart starts OpenTSDB collector on the given addr.
+//
+// MustStop must be called on the returned server when it is no longer needed.
+func MustStart(addr string, maxRequestSize int64) *Server {
 	logger.Infof("starting TCP OpenTSDB collector at %q", addr)
 	lnTCP, err := netutil.NewTCPListener("opentsdb", addr)
 	if err != nil {
 		logger.Fatalf("cannot start TCP OpenTSDB collector at %q: %s", addr, err)
 	}
-	listenerTCP = lnTCP
+	ls := newListenerSwitch(lnTCP)
+	lnHTTP := ls.newHTTPListener()
+	lnTelnet := ls.newTelnetListener()
+	httpServer := opentsdbhttp.MustServe(lnHTTP, maxRequestSize)
 
 	logger.Infof("starting UDP OpenTSDB collector at %q", addr)
 	lnUDP, err := net.ListenPacket("udp4", addr)
 	if err != nil {
 		logger.Fatalf("cannot start UDP OpenTSDB collector at %q: %s", addr, err)
 	}
-	listenerUDP = lnUDP
 
-	var wg sync.WaitGroup
-	wg.Add(1)
+	s := &Server{
+		addr:       addr,
+		ls:         ls,
+		httpServer: httpServer,
+		lnUDP:      lnUDP,
+	}
+	s.wg.Add(1)
 	go func() {
-		defer wg.Done()
-		serveTCP(listenerTCP)
-		logger.Infof("stopped TCP OpenTSDB collector at %q", addr)
+		defer s.wg.Done()
+		serveTelnet(lnTelnet)
+		logger.Infof("stopped TCP telnet OpenTSDB server at %q", addr)
 	}()
-	wg.Add(1)
+	s.wg.Add(1)
 	go func() {
-		defer wg.Done()
-		serveUDP(listenerUDP)
-		logger.Infof("stopped UDP OpenTSDB collector at %q", addr)
+		defer s.wg.Done()
+		httpServer.Wait()
+		// Do not log when httpServer is stopped, since this is logged by the server itself.
 	}()
-	wg.Wait()
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		serveUDP(lnUDP)
+		logger.Infof("stopped UDP OpenTSDB server at %q", addr)
+	}()
+	return s
 }
 
-func serveTCP(ln net.Listener) {
+// MustStop stops the server.
+func (s *Server) MustStop() {
+	// Stop HTTP server. Do not emit log message, since it is emitted by the httpServer.
+	s.httpServer.MustStop()
+
+	logger.Infof("stopping TCP telnet OpenTSDB server at %q...", s.addr)
+	if err := s.ls.stop(); err != nil {
+		logger.Errorf("cannot stop TCP telnet OpenTSDB server: %s", err)
+	}
+
+	logger.Infof("stopping UDP OpenTSDB server at %q...", s.addr)
+	if err := s.lnUDP.Close(); err != nil {
+		logger.Errorf("cannot stop UDP OpenTSDB server: %s", err)
+	}
+
+	// Wait until all the servers are stopped.
+	s.wg.Wait()
+	logger.Infof("TCP and UDP OpenTSDB servers at %q have been stopped", s.addr)
+}
+
+func serveTelnet(ln net.Listener) {
 	for {
 		c, err := ln.Accept()
 		if err != nil {
 			if ne, ok := err.(net.Error); ok {
 				if ne.Temporary() {
+					logger.Errorf("opentsdb: temporary error when listening for TCP addr %q: %s", ln.Addr(), err)
 					time.Sleep(time.Second)
 					continue
 				}
@@ -97,6 +145,7 @@ func serveUDP(ln net.PacketConn) {
 					writeErrorsUDP.Inc()
 					if ne, ok := err.(net.Error); ok {
 						if ne.Temporary() {
+							logger.Errorf("opentsdb: temporary error when listening for UDP addr %q: %s", ln.LocalAddr(), err)
 							time.Sleep(time.Second)
 							continue
 						}
@@ -119,20 +168,3 @@ func serveUDP(ln net.PacketConn) {
 	}
 	wg.Wait()
 }
-
-var (
-	listenerTCP net.Listener
-	listenerUDP net.PacketConn
-)
-
-// Stop stops the server.
-func Stop() {
-	logger.Infof("stopping TCP OpenTSDB server at %q...", listenerTCP.Addr())
-	if err := listenerTCP.Close(); err != nil {
-		logger.Errorf("cannot close TCP OpenTSDB server: %s", err)
-	}
-	logger.Infof("stopping UDP OpenTSDB server at %q...", listenerUDP.LocalAddr())
-	if err := listenerUDP.Close(); err != nil {
-		logger.Errorf("cannot close UDP OpenTSDB server: %s", err)
-	}
-}

app/vminsert/opentsdbhttp/request_handler.go

@@ -19,9 +19,9 @@ var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb-http"}`)
 	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb-http"}`)
 
-	opentsdbReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb-http"}`)
-	opentsdbReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb-http"}`)
-	opentsdbUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb-http"}`)
+	readCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb-http"}`)
+	readErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb-http"}`)
+	unmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb-http"}`)
 )
 
 // insertHandler processes HTTP OpenTSDB put requests.
@@ -33,13 +33,13 @@ func insertHandler(req *http.Request, maxSize int64) error {
 }
 
 func insertHandlerInternal(req *http.Request, maxSize int64) error {
-	opentsdbReadCalls.Inc()
+	readCalls.Inc()
 
 	r := req.Body
 	if req.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(r)
 		if err != nil {
-			opentsdbReadErrors.Inc()
+			readErrors.Inc()
 			return fmt.Errorf("cannot read gzipped http protocol data: %s", err)
 		}
 		defer common.PutGzipReader(zr)
@@ -53,11 +53,11 @@ func insertHandlerInternal(req *http.Request, maxSize int64) error {
 	lr := io.LimitReader(r, maxSize+1)
 	reqLen, err := ctx.reqBuf.ReadFrom(lr)
 	if err != nil {
-		opentsdbReadErrors.Inc()
+		readErrors.Inc()
 		return fmt.Errorf("cannot read HTTP OpenTSDB request: %s", err)
 	}
 	if reqLen > maxSize {
-		opentsdbReadErrors.Inc()
+		readErrors.Inc()
 		return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed %d bytes", maxSize)
 	}
 
@@ -66,7 +66,7 @@ func insertHandlerInternal(req *http.Request, maxSize int64) error {
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(ctx.reqBuf.B)
 	if err != nil {
-		opentsdbUnmarshalErrors.Inc()
+		unmarshalErrors.Inc()
 		return fmt.Errorf("cannot parse HTTP OpenTSDB json: %s", err)
 	}
 	ctx.Rows.Unmarshal(v)

app/vminsert/opentsdbhttp/server.go

@@ -2,11 +2,14 @@ package opentsdbhttp
 
 import (
 	"context"
+	"net"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -15,56 +18,84 @@ var (
 	writeErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/put", protocol="opentsdb-http"}`)
 )
 
-var (
-	httpServer     *http.Server
-	httpAddr       string
-	maxRequestSize int64
-)
+// Server represents HTTP OpenTSDB server.
+type Server struct {
+	s  *http.Server
+	ln net.Listener
+	wg sync.WaitGroup
+}
 
-// Serve starts HTTP OpenTSDB server on the given addr.
-func Serve(addr string, maxReqSize int64) {
+// MustStart starts HTTP OpenTSDB server on the given addr.
+//
+// MustStop must be called on the returned server when it is no longer needed.
+func MustStart(addr string, maxRequestSize int64) *Server {
 	logger.Infof("starting HTTP OpenTSDB server at %q", addr)
-	httpAddr = addr
-	maxRequestSize = maxReqSize
-	httpServer = &http.Server{
-		Addr:         addr,
-		Handler:      http.HandlerFunc(requestHandler),
+	lnTCP, err := netutil.NewTCPListener("opentsdbhttp", addr)
+	if err != nil {
+		logger.Fatalf("cannot start HTTP OpenTSDB collector at %q: %s", addr, err)
+	}
+	return MustServe(lnTCP, maxRequestSize)
+}
+
+// MustServe serves OpenTSDB HTTP put requests from ln with up to maxRequestSize size.
+//
+// MustStop must be called on the returned server when it is no longer needed.
+func MustServe(ln net.Listener, maxRequestSize int64) *Server {
+	h := newRequestHandler(maxRequestSize)
+	hs := &http.Server{
+		Handler:      h,
 		ReadTimeout:  30 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
+	s := &Server{
+		s:  hs,
+		ln: ln,
+	}
+	s.wg.Add(1)
 	go func() {
-		err := httpServer.ListenAndServe()
+		defer s.wg.Done()
+		err := s.s.Serve(s.ln)
 		if err == http.ErrServerClosed {
 			return
 		}
 		if err != nil {
-			logger.Fatalf("error serving HTTP OpenTSDB: %s", err)
+			logger.Fatalf("error serving HTTP OpenTSDB at %q: %s", s.ln.Addr(), err)
 		}
 	}()
+	return s
 }
 
-// requestHandler handles HTTP OpenTSDB insert request.
-func requestHandler(w http.ResponseWriter, r *http.Request) {
-	switch r.URL.Path {
-	case "/api/put":
-		writeRequests.Inc()
-		if err := insertHandler(r, maxRequestSize); err != nil {
-			writeErrors.Inc()
-			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
-			return
-		}
-		w.WriteHeader(http.StatusNoContent)
-	default:
-		httpserver.Errorf(w, "unexpected path requested on HTTP OpenTSDB server: %q", r.URL.Path)
-	}
+// Wait waits until the server is stopped with MustStop.
+func (s *Server) Wait() {
+	s.wg.Wait()
 }
 
-// Stop stops HTTP OpenTSDB server.
-func Stop() {
-	logger.Infof("stopping HTTP OpenTSDB server at %q...", httpAddr)
+// MustStop stops HTTP OpenTSDB server.
+func (s *Server) MustStop() {
+	logger.Infof("stopping HTTP OpenTSDB server at %q...", s.ln.Addr())
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
-	if err := httpServer.Shutdown(ctx); err != nil {
-		logger.Fatalf("cannot close HTTP OpenTSDB server: %s", err)
+	if err := s.s.Shutdown(ctx); err != nil {
+		logger.Fatalf("cannot close HTTP OpenTSDB server at %q: %s", s.ln.Addr(), err)
 	}
+	s.wg.Wait()
+	logger.Infof("OpenTSDB HTTP server at %q has been stopped", s.ln.Addr())
+}
+
+func newRequestHandler(maxRequestSize int64) http.Handler {
+	rh := func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/put":
+			writeRequests.Inc()
+			if err := insertHandler(r, maxRequestSize); err != nil {
+				writeErrors.Inc()
+				httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+				return
+			}
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			httpserver.Errorf(w, "unexpected path requested on HTTP OpenTSDB server: %q", r.URL.Path)
+		}
+	}
+	return http.HandlerFunc(rh)
 }

app/vminsert/vmimport/parser.go

@@ -0,0 +1,202 @@
+package vmimport
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+)
+
+// Rows contains parsed rows from `/api/v1/import` request.
+type Rows struct {
+	Rows []Row
+
+	tu tagsUnmarshaler
+}
+
+// Reset resets rs.
+func (rs *Rows) Reset() {
+	for i := range rs.Rows {
+		rs.Rows[i].reset()
+	}
+	rs.Rows = rs.Rows[:0]
+
+	rs.tu.reset()
+}
+
+// Unmarshal unmarshals influx line protocol rows from s.
+//
+// See https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/
+//
+// s must be unchanged until rs is in use.
+func (rs *Rows) Unmarshal(s string) {
+	rs.tu.reset()
+	rs.Rows = unmarshalRows(rs.Rows[:0], s, &rs.tu)
+}
+
+// Row is a single row from `/api/v1/import` request.
+type Row struct {
+	Tags       []Tag
+	Values     []float64
+	Timestamps []int64
+}
+
+func (r *Row) reset() {
+	r.Tags = nil
+	r.Values = r.Values[:0]
+	r.Timestamps = r.Timestamps[:0]
+}
+
+func (r *Row) unmarshal(s string, tu *tagsUnmarshaler) error {
+	r.reset()
+	v, err := tu.p.Parse(s)
+	if err != nil {
+		return fmt.Errorf("cannot parse json line: %s", err)
+	}
+
+	// Unmarshal tags
+	metric := v.GetObject("metric")
+	if metric == nil {
+		return fmt.Errorf("missing `metric` object")
+	}
+	tagsStart := len(tu.tagsPool)
+	if err := tu.unmarshalTags(metric); err != nil {
+		return fmt.Errorf("cannot unmarshal `metric`: %s", err)
+	}
+	tags := tu.tagsPool[tagsStart:]
+	r.Tags = tags[:len(tags):len(tags)]
+	if len(r.Tags) == 0 {
+		return fmt.Errorf("missing tags")
+	}
+
+	// Unmarshal values
+	values := v.GetArray("values")
+	if len(values) == 0 {
+		return fmt.Errorf("missing `values` array")
+	}
+	for i, v := range values {
+		f, err := v.Float64()
+		if err != nil {
+			return fmt.Errorf("cannot unmarshal value at position %d: %s", i, err)
+		}
+		r.Values = append(r.Values, f)
+	}
+
+	// Unmarshal timestamps
+	timestamps := v.GetArray("timestamps")
+	if len(timestamps) == 0 {
+		return fmt.Errorf("missing `timestamps` array")
+	}
+	for i, v := range timestamps {
+		ts, err := v.Int64()
+		if err != nil {
+			return fmt.Errorf("cannot unmarshal timestamp at position %d: %s", i, err)
+		}
+		r.Timestamps = append(r.Timestamps, ts)
+	}
+
+	if len(r.Timestamps) != len(r.Values) {
+		return fmt.Errorf("`timestamps` array size must match `values` array size; got %d; want %d", len(r.Timestamps), len(r.Values))
+	}
+	return nil
+}
+
+// Tag represents `/api/v1/import` tag.
+type Tag struct {
+	Key   []byte
+	Value []byte
+}
+
+func (tag *Tag) reset() {
+	// tag.Key and tag.Value point to tu.bytesPool, so there is no need in keeping these byte slices here.
+	tag.Key = nil
+	tag.Value = nil
+}
+
+type tagsUnmarshaler struct {
+	p         fastjson.Parser
+	tagsPool  []Tag
+	bytesPool []byte
+	err       error
+}
+
+func (tu *tagsUnmarshaler) reset() {
+	for i := range tu.tagsPool {
+		tu.tagsPool[i].reset()
+	}
+	tu.tagsPool = tu.tagsPool[:0]
+
+	tu.bytesPool = tu.bytesPool[:0]
+	tu.err = nil
+}
+
+func (tu *tagsUnmarshaler) addTag() *Tag {
+	dst := tu.tagsPool
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Tag{})
+	}
+	tag := &dst[len(dst)-1]
+	tu.tagsPool = dst
+	return tag
+}
+
+func (tu *tagsUnmarshaler) addBytes(b []byte) []byte {
+	bytesPoolLen := len(tu.bytesPool)
+	tu.bytesPool = append(tu.bytesPool, b...)
+	bCopy := tu.bytesPool[bytesPoolLen:]
+	return bCopy[:len(bCopy):len(bCopy)]
+}
+
+func (tu *tagsUnmarshaler) unmarshalTags(o *fastjson.Object) error {
+	tu.err = nil
+	o.Visit(func(key []byte, v *fastjson.Value) {
+		tag := tu.addTag()
+		tag.Key = tu.addBytes(key)
+		sb, err := v.StringBytes()
+		if err != nil && tu.err != nil {
+			tu.err = fmt.Errorf("cannot parse value for tag %q: %s", tag.Key, err)
+		}
+		tag.Value = tu.addBytes(sb)
+	})
+	return tu.err
+}
+
+func unmarshalRows(dst []Row, s string, tu *tagsUnmarshaler) []Row {
+	for len(s) > 0 {
+		n := strings.IndexByte(s, '\n')
+		if n < 0 {
+			// The last line.
+			return unmarshalRow(dst, s, tu)
+		}
+		dst = unmarshalRow(dst, s[:n], tu)
+		s = s[n+1:]
+	}
+	return dst
+}
+
+func unmarshalRow(dst []Row, s string, tu *tagsUnmarshaler) []Row {
+	if len(s) > 0 && s[len(s)-1] == '\r' {
+		s = s[:len(s)-1]
+	}
+	if len(s) == 0 {
+		return dst
+	}
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	if err := r.unmarshal(s, tu); err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal json line %q: %s; skipping it", s, err)
+		invalidLines.Inc()
+	}
+	return dst
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="vmimport"}`)

app/vminsert/vmimport/parser_test.go

@@ -0,0 +1,216 @@
+package vmimport
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRowsUnmarshalFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		var rows Rows
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
+		}
+
+		// Try again
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("expecting zero rows; got %d rows", len(rows.Rows))
+		}
+	}
+
+	// Invalid json line
+	f("")
+	f("\n")
+	f("foo\n")
+	f("123")
+	f("[1,3]")
+	f("{}")
+	f("[]")
+	f(`{"foo":"bar"}`)
+
+	// Invalid metric
+	f(`{"metric":123,"values":[1,2],"timestamps":[3,4]}`)
+	f(`{"metric":[123],"values":[1,2],"timestamps":[3,4]}`)
+	f(`{"metric":[],"values":[1,2],"timestamps":[3,4]}`)
+	f(`{"metric":{},"values":[1,2],"timestamps":[3,4]}`)
+	f(`{"metric":null,"values":[1,2],"timestamps":[3,4]}`)
+	f(`{"values":[1,2],"timestamps":[3,4]}`)
+
+	// Invalid values
+	f(`{"metric":{"foo":"bar"},"values":1,"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":{"x":1},"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":{"x":1},"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":null,"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"timestamps":[3,4]}`)
+
+	// Invalid timestamps
+	f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":3}`)
+	f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":false}`)
+	f(`{"metric":{"foo":"bar"},"values":[1,2],"timestamps":{}}`)
+	f(`{"metric":{"foo":"bar"},"values":[1,2]}`)
+
+	// values and timestamps count mismatch
+	f(`{"metric":{"foo":"bar"},"values":[],"timestamps":[]}`)
+	f(`{"metric":{"foo":"bar"},"values":[],"timestamps":[1]}`)
+	f(`{"metric":{"foo":"bar"},"values":[2],"timestamps":[]}`)
+	f(`{"metric":{"foo":"bar"},"values":[2],"timestamps":[3,4]}`)
+	f(`{"metric":{"foo":"bar"},"values":[2,3],"timestamps":[4]}`)
+
+	// Garbage after the line
+	f(`{"metric":{"foo":"bar"},"values":[2],"timestamps":[4]}{}`)
+}
+
+func TestRowsUnmarshalSuccess(t *testing.T) {
+	f := func(s string, rowsExpected *Rows) {
+		t.Helper()
+		var rows Rows
+		rows.Unmarshal(s)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		// Try unmarshaling again
+		rows.Unmarshal(s)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		rows.Reset()
+		if len(rows.Rows) != 0 {
+			t.Fatalf("non-empty rows after reset: %+v", rows.Rows)
+		}
+	}
+
+	// Empty line
+	f("", &Rows{})
+	f("\n\n", &Rows{})
+	f("\n\r\n", &Rows{})
+
+	// Single line with a single tag
+	f(`{"metric":{"foo":"bar"},"values":[1.23],"timestamps":[456]}`, &Rows{
+		Rows: []Row{{
+			Tags: []Tag{{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			}},
+			Values:     []float64{1.23},
+			Timestamps: []int64{456},
+		}},
+	})
+
+	// Line with multiple tags
+	f(`{"metric":{"foo":"bar","baz":"xx"},"values":[1.23, -3.21],"timestamps" : [456,789]}`, &Rows{
+		Rows: []Row{{
+			Tags: []Tag{
+				{
+					Key:   []byte("foo"),
+					Value: []byte("bar"),
+				},
+				{
+					Key:   []byte("baz"),
+					Value: []byte("xx"),
+				},
+			},
+			Values:     []float64{1.23, -3.21},
+			Timestamps: []int64{456, 789},
+		}},
+	})
+
+	// Multiple lines
+	f(`{"metric":{"foo":"bar","baz":"xx"},"values":[1.23, -3.21],"timestamps" : [456,789]}
+{"metric":{"__name__":"xx"},"values":[34],"timestamps" : [11]}
+`, &Rows{
+		Rows: []Row{
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("foo"),
+						Value: []byte("bar"),
+					},
+					{
+						Key:   []byte("baz"),
+						Value: []byte("xx"),
+					},
+				},
+				Values:     []float64{1.23, -3.21},
+				Timestamps: []int64{456, 789},
+			},
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("__name__"),
+						Value: []byte("xx"),
+					},
+				},
+				Values:     []float64{34},
+				Timestamps: []int64{11},
+			},
+		},
+	})
+
+	// Multiple lines with invalid line in the middle.
+	f(`{"metric":{"xfoo":"bar","baz":"xx"},"values":[1.232, -3.21],"timestamps" : [456,7890]}
+garbage here
+{"metric":{"__name__":"xxy"},"values":[34],"timestamps" : [111]}`, &Rows{
+		Rows: []Row{
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("xfoo"),
+						Value: []byte("bar"),
+					},
+					{
+						Key:   []byte("baz"),
+						Value: []byte("xx"),
+					},
+				},
+				Values:     []float64{1.232, -3.21},
+				Timestamps: []int64{456, 7890},
+			},
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("__name__"),
+						Value: []byte("xxy"),
+					},
+				},
+				Values:     []float64{34},
+				Timestamps: []int64{111},
+			},
+		},
+	})
+
+	// No newline after the second line.
+	f(`{"metric":{"foo":"bar","baz":"xx"},"values":[1.23, -3.21],"timestamps" : [456,789]}
+{"metric":{"__name__":"xx"},"values":[34],"timestamps" : [11]}`, &Rows{
+		Rows: []Row{
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("foo"),
+						Value: []byte("bar"),
+					},
+					{
+						Key:   []byte("baz"),
+						Value: []byte("xx"),
+					},
+				},
+				Values:     []float64{1.23, -3.21},
+				Timestamps: []int64{456, 789},
+			},
+			{
+				Tags: []Tag{
+					{
+						Key:   []byte("__name__"),
+						Value: []byte("xx"),
+					},
+				},
+				Values:     []float64{34},
+				Timestamps: []int64{11},
+			},
+		},
+	})
+}

app/vminsert/vmimport/parser_timing_test.go

@@ -0,0 +1,25 @@
+package vmimport
+
+import (
+	"fmt"
+	"testing"
+)
+
+func BenchmarkRowsUnmarshal(b *testing.B) {
+	s := `{"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
+{"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
+{"metric":{"__name__":"up","job":"node_exporter","instance":"foobar.com:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
+{"metric":{"__name__":"up","job":"prometheus","instance":"xxx.yyy.zzz:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
+`
+	b.SetBytes(int64(len(s)))
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		var rows Rows
+		for pb.Next() {
+			rows.Unmarshal(s)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of rows parsed; got %d; want 4", len(rows.Rows)))
+			}
+		}
+	})
+}

app/vminsert/vmimport/request_handler.go

@@ -0,0 +1,160 @@
+package vmimport
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var maxLineLen = flag.Int("import.maxLineLen", 100*1024*1024, "The maximum length in bytes of a single line accepted by `/api/v1/import`")
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return concurrencylimiter.Do(func() error {
+		return insertHandlerInternal(req)
+	})
+}
+
+func insertHandlerInternal(req *http.Request) error {
+	readCalls.Inc()
+
+	r := req.Body
+	if req.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped vmimport data: %s", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
+	}
+
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+	for ctx.Read(r) {
+		if err := ctx.InsertRows(); err != nil {
+			return err
+		}
+	}
+	return ctx.Error()
+}
+
+func (ctx *pushCtx) InsertRows() error {
+	rows := ctx.Rows.Rows
+	rowsLen := 0
+	for i := range rows {
+		rowsLen += len(rows[i].Values)
+	}
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	rowsTotal := 0
+	for i := range rows {
+		r := &rows[i]
+		ic.Labels = ic.Labels[:0]
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ic.AddLabelBytes(tag.Key, tag.Value)
+		}
+		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		for j, value := range values {
+			timestamp := timestamps[j]
+			ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value)
+		}
+		rowsTotal += len(values)
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ic.FlushBufs()
+}
+
+func (ctx *pushCtx) Read(r io.Reader) bool {
+	if ctx.err != nil {
+		return false
+	}
+	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(r, ctx.reqBuf, ctx.tailBuf, *maxLineLen)
+	if ctx.err != nil {
+		if ctx.err != io.EOF {
+			readErrors.Inc()
+			ctx.err = fmt.Errorf("cannot read vmimport data: %s", ctx.err)
+		}
+		return false
+	}
+	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
+	return true
+}
+
+var (
+	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="vmimport"}`)
+	readErrors = metrics.NewCounter(`vm_read_errors_total{name="vmimport"}`)
+)
+
+type pushCtx struct {
+	Rows   Rows
+	Common common.InsertCtx
+
+	reqBuf        []byte
+	tailBuf       []byte
+	metricNameBuf []byte
+
+	err error
+}
+
+func (ctx *pushCtx) Error() error {
+	if ctx.err == io.EOF {
+		return nil
+	}
+	return ctx.err
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Rows.Reset()
+	ctx.Common.Reset(0)
+
+	ctx.reqBuf = ctx.reqBuf[:0]
+	ctx.tailBuf = ctx.tailBuf[:0]
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+
+	ctx.err = nil
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmrestore/Makefile

@@ -0,0 +1,58 @@
+# All these commands must run from repository root.
+
+vmrestore:
+	APP_NAME=vmrestore $(MAKE) app-local
+
+vmrestore-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker
+
+vmrestore-pure-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-pure
+
+vmrestore--arm-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm
+
+vmrestore-arm64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm64
+
+vmrestore-ppc64le-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-ppc64le
+
+vmrestore-386-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-386
+
+package-vmrestore:
+	APP_NAME=vmrestore $(MAKE) package-via-docker
+
+package-vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-pure
+
+package-vmrestore-arm:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm
+
+package-vmrestore-arm64:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm64
+
+package-vmrestore-ppc64le:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-ppc64le
+
+package-vmrestore-386:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-386
+
+publish-vmrestore:
+	APP_NAME=vmrestore $(MAKE) publish-via-docker
+
+vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) app-local-pure
+
+vmrestore-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm ./app/vmrestore
+
+vmrestore-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm64 ./app/vmrestore
+
+vmrestore-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-ppc64le ./app/vmrestore
+
+vmrestore-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-386 ./app/vmrestore

app/vmrestore/README.md

@@ -0,0 +1,86 @@
+## vmrestore
+
+`vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
+
+Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+when restarting `vmrestore` with the same args.
+
+
+### Usage
+
+VictoriaMetrics must be stopped during the restore process.
+
+```
+vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+
+```
+
+* `<bucket>` is [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets) name.
+* `<path/to/backup>` is the path to backup made with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) on GCS bucket.
+* `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
+  to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
+
+The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup.
+
+
+### Troubleshooting
+
+* If `vmrestore` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
+
+
+### Advanced usage
+
+Run `vmrestore -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum download speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -src string
+    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -storageDataPath string
+    	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmrestore` from the root folder of the repository.
+   It builds `vmrestore` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmrestore-prod` from the root folder of the repository.
+   It builds `vmrestore-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.

app/vmrestore/deployment/Dockerfile

@@ -0,0 +1,7 @@
+ARG certs_image
+FROM $certs_image AS certs
+FROM scratch
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./vmrestore-prod
+ENTRYPOINT ["/vmrestore-prod"]

app/vmrestore/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	src = flag.String("src", "", "Source path with backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
+		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
+	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
+	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+)
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Restore{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot restore from backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmrestore restores VictoriaMetrics data from backups made by vmbackup.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newDstFS() (*fslocal.FS, error) {
+	if len(*storageDataPath) == 0 {
+		return nil, fmt.Errorf("`-storageDataPath` cannot be empty")
+	}
+	fs := &fslocal.FS{
+		Dir:               *storageDataPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize local fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newSrcFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*src)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %s", *src, err)
+	}
+	return fs, nil
+}

app/vmselect/main.go

@@ -167,6 +167,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/api/v1/rules":
+		// Return dumb placeholder
+		rulesRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
+		return true
+	case "/api/v1/alerts":
+		// Return dumb placehloder
+		alertsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
+		return true
 	case "/api/v1/admin/tsdb/delete_series":
 		deleteRequests.Inc()
 		authKey := r.FormValue("authKey")
@@ -228,4 +240,7 @@ var (
 
 	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/federate"}`)
 	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/federate"}`)
+
+	rulesRequests  = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
+	alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 )

app/vmselect/netstorage/netstorage.go

@@ -92,6 +92,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 	doneCh := make(chan error)
 
 	// Start workers.
+	rowsProcessedTotal := uint64(0)
 	for i := 0; i < workersCount; i++ {
 		go func(workerID uint) {
 			rs := getResult()
@@ -99,6 +100,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			maxWorkersCount := gomaxprocs / workersCount
 
 			var err error
+			rowsProcessed := 0
 			for pts := range workCh {
 				if time.Until(rss.deadline.Deadline) < 0 {
 					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.Timeout)
@@ -111,8 +113,10 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 					// Skip empty blocks.
 					continue
 				}
+				rowsProcessed += len(rs.Values)
 				f(rs, workerID)
 			}
+			atomic.AddUint64(&rowsProcessedTotal, uint64(rowsProcessed))
 			// Drain the remaining work
 			for range workCh {
 			}
@@ -124,6 +128,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 	for i := range rss.packedTimeseries {
 		workCh <- &rss.packedTimeseries[i]
 	}
+	seriesProcessedTotal := len(rss.packedTimeseries)
 	rss.packedTimeseries = rss.packedTimeseries[:0]
 	close(workCh)
 
@@ -134,6 +139,8 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			errors = append(errors, err)
 		}
 	}
+	perQueryRowsProcessed.Update(float64(rowsProcessedTotal))
+	perQuerySeriesProcessed.Update(float64(seriesProcessedTotal))
 	if len(errors) > 0 {
 		// Return just the first error, since other errors
 		// is likely duplicate the first error.
@@ -142,6 +149,9 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 	return nil
 }
 
+var perQueryRowsProcessed = metrics.NewHistogram(`vm_per_query_rows_processed_count`)
+var perQuerySeriesProcessed = metrics.NewHistogram(`vm_per_query_series_processed_count`)
+
 var gomaxprocs = runtime.GOMAXPROCS(-1)
 
 type packedTimeseries struct {
@@ -422,13 +432,10 @@ func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) {
 	// Sort labelEntries by the number of label values in each entry.
 	sort.Slice(labelEntries, func(i, j int) bool {
 		a, b := labelEntries[i].Values, labelEntries[j].Values
-		if len(a) < len(b) {
-			return true
+		if len(a) != len(b) {
+			return len(a) > len(b)
 		}
-		if len(a) > len(b) {
-			return false
-		}
-		return labelEntries[i].Key < labelEntries[j].Key
+		return labelEntries[i].Key > labelEntries[j].Key
 	})
 
 	return labelEntries, nil
@@ -452,16 +459,12 @@ func getStorageSearch() *storage.Search {
 }
 
 func putStorageSearch(sr *storage.Search) {
-	n := atomic.LoadUint64(&sr.MissingMetricNamesForMetricID)
-	missingMetricNamesForMetricID.Add(int(n))
 	sr.MustClose()
 	ssPool.Put(sr)
 }
 
 var ssPool sync.Pool
 
-var missingMetricNamesForMetricID = metrics.NewCounter(`vm_missing_metric_names_for_metric_id_total`)
-
 // ProcessSearchQuery performs sq on storage nodes until the given deadline.
 func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, error) {
 	// Setup search.

app/vmselect/prometheus/prometheus.go

@@ -21,12 +21,13 @@ import (
 )
 
 var (
-	latencyOffset = flag.Duration("search.latencyOffset", time.Second*60, "The time when data points become visible in query results after the colection. "+
+	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the colection. "+
 		"Too small value can result in incomplete last points for query results")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum time for search query execution")
-	maxQueryLen      = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
-	maxLookback      = flag.Duration("search.maxLookback", 0, "Synonim to `-search.lookback-delta` from Prometheus. "+
-		"The value is dynamically detected from interval between time series datapoints if not set. It can be overriden on per-query basis via `max_lookback` arg")
+	maxExportDuration = flag.Duration("search.maxExportDuration", 10*time.Minute, "The maximum duration for `/api/v1/export` call")
+	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
+	maxQueryLen       = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
+	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to `-search.lookback-delta` from Prometheus. "+
+		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via `max_lookback` arg")
 )
 
 // Default step used if not set.
@@ -58,7 +59,7 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 	if start >= end {
 		start = end - defaultStep
 	}
@@ -129,9 +130,9 @@ func ExportHandler(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 	format := r.FormValue("format")
-	deadline := getDeadline(r)
+	deadline := getDeadlineForExport(r)
 	if start >= end {
-		start = end - defaultStep
+		end = start + defaultStep
 	}
 	if err := exportHandler(w, matches, start, end, format, deadline); err != nil {
 		return err
@@ -145,7 +146,7 @@ var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, deadline netstorage.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
 	writeLineFunc := WriteExportJSONLine
-	contentType := "application/json"
+	contentType := "application/stream+json"
 	if format == "prometheus" {
 		contentType = "text/plain"
 		writeLineFunc = WriteExportPrometheusLine
@@ -235,7 +236,7 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
 func LabelValuesHandler(labelName string, w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %s", err)
@@ -285,8 +286,15 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
 	if err != nil {
 		return nil, err
 	}
+	for i, tfs := range tagFilterss {
+		// Add `labelName!=''` tag filter in order to filter out series without the labelName.
+		tagFilterss[i] = append(tfs, storage.TagFilter{
+			Key:        []byte(labelName),
+			IsNegative: true,
+		})
+	}
 	if start >= end {
-		start = end - defaultStep
+		end = start + defaultStep
 	}
 	sq := &storage.SearchQuery{
 		MinTimestamp: start,
@@ -326,7 +334,7 @@ var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 // LabelsCountHandler processes /api/v1/labels/count request.
 func LabelsCountHandler(w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 	labelEntries, err := netstorage.GetLabelEntries(deadline)
 	if err != nil {
 		return fmt.Errorf(`cannot obtain label entries: %s`, err)
@@ -345,10 +353,38 @@ var labelsCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
 func LabelsHandler(w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
-	deadline := getDeadline(r)
-	labels, err := netstorage.GetLabels(deadline)
-	if err != nil {
-		return fmt.Errorf("cannot obtain labels: %s", err)
+	deadline := getDeadlineForQuery(r)
+
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %s", err)
+	}
+	var labels []string
+	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+		var err error
+		labels, err = netstorage.GetLabels(deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels: %s", err)
+		}
+	} else {
+		// Extended functionality that allows filtering by label filters and time range
+		// i.e. /api/v1/labels?match[]=foobar{baz="abc"}&start=...&end=...
+		matches := r.Form["match[]"]
+		if len(matches) == 0 {
+			matches = []string{"{__name__!=''}"}
+		}
+		ct := currentTime()
+		end, err := getTime(r, "end", ct)
+		if err != nil {
+			return err
+		}
+		start, err := getTime(r, "start", end-defaultStep)
+		if err != nil {
+			return err
+		}
+		labels, err = labelsWithMatches(matches, start, end, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %s", matches, start, end, err)
+		}
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -357,12 +393,57 @@ func LabelsHandler(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
+func labelsWithMatches(matches []string, start, end int64, deadline netstorage.Deadline) ([]string, error) {
+	if len(matches) == 0 {
+		logger.Panicf("BUG: matches must be non-empty")
+	}
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return nil, err
+	}
+	if start >= end {
+		end = start + defaultStep
+	}
+	sq := &storage.SearchQuery{
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch data for %q: %s", sq, err)
+	}
+
+	m := make(map[string]struct{})
+	var mLock sync.Mutex
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		mLock.Lock()
+		tags := rs.MetricName.Tags
+		for i := range tags {
+			t := &tags[i]
+			m[string(t.Key)] = struct{}{}
+		}
+		m["__name__"] = struct{}{}
+		mLock.Unlock()
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error when data fetching: %s", err)
+	}
+
+	labels := make([]string, 0, len(m))
+	for label := range m {
+		labels = append(labels, label)
+	}
+	sort.Strings(labels)
+	return labels, nil
+}
+
 var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels"}`)
 
 // SeriesCountHandler processes /api/v1/series/count request.
 func SeriesCountHandler(w http.ResponseWriter, r *http.Request) error {
 	startTime := time.Now()
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 	n, err := netstorage.GetSeriesCount(deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %s", err)
@@ -402,14 +483,14 @@ func SeriesHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return err
 	}
 	if start >= end {
-		start = end - defaultStep
+		end = start + defaultStep
 	}
 	sq := &storage.SearchQuery{
 		MinTimestamp: start,
@@ -471,7 +552,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	deadline := getDeadline(r)
+	deadline := getDeadlineForQuery(r)
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
@@ -480,25 +561,19 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	if len(query) > *maxQueryLen {
 		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
 	}
-	if ct-start < queryOffset {
-		start -= queryOffset
+	if !getBool(r, "nocache") && ct-start < queryOffset {
+		// Adjust start time only if `nocache` arg isn't set.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
+		start = ct - queryOffset
 	}
 	if childQuery, windowStr, offsetStr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
-		var window int64
-		if len(windowStr) > 0 {
-			var err error
-			window, err = promql.DurationValue(windowStr, step)
-			if err != nil {
-				return err
-			}
+		window, err := parsePositiveDuration(windowStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse window: %s", err)
 		}
-		var offset int64
-		if len(offsetStr) > 0 {
-			var err error
-			offset, err = promql.DurationValue(offsetStr, step)
-			if err != nil {
-				return err
-			}
+		offset, err := parseDuration(offsetStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse offset: %s", err)
 		}
 		start -= offset
 		end := start
@@ -509,6 +584,31 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 		queryDuration.UpdateDuration(startTime)
 		return nil
 	}
+	if childQuery, windowStr, stepStr, offsetStr := promql.IsRollup(query); childQuery != "" {
+		newStep, err := parsePositiveDuration(stepStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse step: %s", err)
+		}
+		if newStep > 0 {
+			step = newStep
+		}
+		window, err := parsePositiveDuration(windowStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse window: %s", err)
+		}
+		offset, err := parseDuration(offsetStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse offset: %s", err)
+		}
+		start -= offset
+		end := start
+		start = end - window
+		if err := queryRangeHandler(w, childQuery, start, end, step, r, ct); err != nil {
+			return err
+		}
+		queryDuration.UpdateDuration(startTime)
+		return nil
+	}
 
 	ec := promql.EvalConfig{
 		Start:         start,
@@ -530,6 +630,20 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 
 var queryDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/query"}`)
 
+func parseDuration(s string, step int64) (int64, error) {
+	if len(s) == 0 {
+		return 0, nil
+	}
+	return promql.DurationValue(s, step)
+}
+
+func parsePositiveDuration(s string, step int64) (int64, error) {
+	if len(s) == 0 {
+		return 0, nil
+	}
+	return promql.PositiveDurationValue(s, step)
+}
+
 // QueryRangeHandler processes /api/v1/query_range request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
@@ -553,7 +667,15 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	deadline := getDeadline(r)
+	if err := queryRangeHandler(w, query, start, end, step, r, ct); err != nil {
+		return err
+	}
+	queryRangeDuration.UpdateDuration(startTime)
+	return nil
+}
+
+func queryRangeHandler(w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64) error {
+	deadline := getDeadlineForQuery(r)
 	mayCache := !getBool(r, "nocache")
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
@@ -565,7 +687,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
 	}
 	if start > end {
-		start = end
+		end = start + defaultStep
 	}
 	if err := promql.ValidateMaxPointsPerTimeseries(start, end, step); err != nil {
 		return err
@@ -597,7 +719,6 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 
 	w.Header().Set("Content-Type", "application/json")
 	WriteQueryRangeResponse(w, result)
-	queryRangeDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -746,12 +867,21 @@ func getMaxLookback(r *http.Request) (int64, error) {
 	return getDuration(r, "max_lookback", d)
 }
 
-func getDeadline(r *http.Request) netstorage.Deadline {
+func getDeadlineForQuery(r *http.Request) netstorage.Deadline {
+	dMax := int64(maxQueryDuration.Seconds() * 1e3)
+	return getDeadlineWithMaxDuration(r, dMax)
+}
+
+func getDeadlineForExport(r *http.Request) netstorage.Deadline {
+	dMax := int64(maxExportDuration.Seconds() * 1e3)
+	return getDeadlineWithMaxDuration(r, dMax)
+}
+
+func getDeadlineWithMaxDuration(r *http.Request, dMax int64) netstorage.Deadline {
 	d, err := getDuration(r, "timeout", 0)
 	if err != nil {
 		d = 0
 	}
-	dMax := int64(maxQueryDuration.Seconds() * 1e3)
 	if d <= 0 || d > dMax {
 		d = dMax
 	}

app/vmselect/promql/aggr.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/histogram"
 )
 
 var aggrFuncs = map[string]aggrFunc{
@@ -25,12 +27,21 @@ var aggrFuncs = map[string]aggrFunc{
 	"topk":         newAggrFuncTopK(false),
 	"quantile":     aggrFuncQuantile,
 
-	// Extended PromQL funcs
-	"median":   aggrFuncMedian,
-	"limitk":   aggrFuncLimitK,
-	"distinct": newAggrFunc(aggrFuncDistinct),
-	"sum2":     newAggrFunc(aggrFuncSum2),
-	"geomean":  newAggrFunc(aggrFuncGeomean),
+	// PromQL extension funcs
+	"median":         aggrFuncMedian,
+	"limitk":         aggrFuncLimitK,
+	"distinct":       newAggrFunc(aggrFuncDistinct),
+	"sum2":           newAggrFunc(aggrFuncSum2),
+	"geomean":        newAggrFunc(aggrFuncGeomean),
+	"histogram":      newAggrFunc(aggrFuncHistogram),
+	"topk_min":       newAggrFuncRangeTopK(minValue, false),
+	"topk_max":       newAggrFuncRangeTopK(maxValue, false),
+	"topk_avg":       newAggrFuncRangeTopK(avgValue, false),
+	"topk_median":    newAggrFuncRangeTopK(medianValue, false),
+	"bottomk_min":    newAggrFuncRangeTopK(minValue, true),
+	"bottomk_max":    newAggrFuncRangeTopK(maxValue, true),
+	"bottomk_avg":    newAggrFuncRangeTopK(avgValue, true),
+	"bottomk_median": newAggrFuncRangeTopK(medianValue, true),
 }
 
 type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error)
@@ -184,6 +195,38 @@ func aggrFuncGeomean(tss []*timeseries) []*timeseries {
 	return tss[:1]
 }
 
+func aggrFuncHistogram(tss []*timeseries) []*timeseries {
+	var h metrics.Histogram
+	m := make(map[string]*timeseries)
+	for i := range tss[0].Values {
+		h.Reset()
+		for _, ts := range tss {
+			v := ts.Values[i]
+			h.Update(v)
+		}
+		h.VisitNonZeroBuckets(func(vmrange string, count uint64) {
+			ts := m[vmrange]
+			if ts == nil {
+				ts = &timeseries{}
+				ts.CopyFromShallowTimestamps(tss[0])
+				ts.MetricName.RemoveTag("vmrange")
+				ts.MetricName.AddTag("vmrange", vmrange)
+				values := ts.Values
+				for k := range values {
+					values[k] = 0
+				}
+				m[vmrange] = ts
+			}
+			ts.Values[i] = float64(count)
+		})
+	}
+	rvs := make([]*timeseries, 0, len(m))
+	for _, ts := range m {
+		rvs = append(rvs, ts)
+	}
+	return vmrangeBucketsToLE(rvs)
+}
+
 func aggrFuncMin(tss []*timeseries) []*timeseries {
 	if len(tss) == 1 {
 		// Fast path - nothing to min.
@@ -425,37 +468,138 @@ func newAggrFuncTopK(isReverse bool) aggrFunc {
 			return nil, err
 		}
 		afe := func(tss []*timeseries) []*timeseries {
-			rvs := tss
-			for n := range rvs[0].Values {
-				sort.Slice(rvs, func(i, j int) bool {
-					a := rvs[i].Values[n]
-					b := rvs[j].Values[n]
-					cmp := lessWithNaNs(a, b)
+			for n := range tss[0].Values {
+				sort.Slice(tss, func(i, j int) bool {
+					a := tss[i].Values[n]
+					b := tss[j].Values[n]
 					if isReverse {
-						cmp = !cmp
+						a, b = b, a
 					}
-					return cmp
+					return lessWithNaNs(a, b)
 				})
-				if math.IsNaN(ks[n]) {
-					ks[n] = 0
-				}
-				k := int(ks[n])
-				if k < 0 {
-					k = 0
-				}
-				if k > len(rvs) {
-					k = len(rvs)
-				}
-				for _, ts := range rvs[:len(rvs)-k] {
-					ts.Values[n] = nan
-				}
+				fillNaNsAtIdx(n, ks[n], tss)
 			}
-			return removeNaNs(rvs)
+			return removeNaNs(tss)
 		}
 		return aggrFuncExt(afe, args[1], &afa.ae.Modifier, true)
 	}
 }
 
+type tsWithValue struct {
+	ts    *timeseries
+	value float64
+}
+
+func newAggrFuncRangeTopK(f func(values []float64) float64, isReverse bool) aggrFunc {
+	return func(afa *aggrFuncArg) ([]*timeseries, error) {
+		args := afa.args
+		if err := expectTransformArgsNum(args, 2); err != nil {
+			return nil, err
+		}
+		ks, err := getScalar(args[0], 0)
+		if err != nil {
+			return nil, err
+		}
+		afe := func(tss []*timeseries) []*timeseries {
+			maxs := make([]tsWithValue, len(tss))
+			for i, ts := range tss {
+				value := f(ts.Values)
+				maxs[i] = tsWithValue{
+					ts:    ts,
+					value: value,
+				}
+			}
+			sort.Slice(maxs, func(i, j int) bool {
+				a := maxs[i].value
+				b := maxs[j].value
+				if isReverse {
+					a, b = b, a
+				}
+				return lessWithNaNs(a, b)
+			})
+			for i := range maxs {
+				tss[i] = maxs[i].ts
+			}
+			for i, k := range ks {
+				fillNaNsAtIdx(i, k, tss)
+			}
+			return removeNaNs(tss)
+		}
+		return aggrFuncExt(afe, args[1], &afa.ae.Modifier, true)
+	}
+}
+
+func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
+	if math.IsNaN(k) {
+		k = 0
+	}
+	kn := int(k)
+	if kn < 0 {
+		kn = 0
+	}
+	if kn > len(tss) {
+		kn = len(tss)
+	}
+	for _, ts := range tss[:len(tss)-kn] {
+		ts.Values[idx] = nan
+	}
+}
+
+func minValue(values []float64) float64 {
+	if len(values) == 0 {
+		return nan
+	}
+	min := values[0]
+	for _, v := range values[1:] {
+		if v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func maxValue(values []float64) float64 {
+	if len(values) == 0 {
+		return nan
+	}
+	max := values[0]
+	for _, v := range values[1:] {
+		if v > max {
+			max = v
+		}
+	}
+	return max
+}
+
+func avgValue(values []float64) float64 {
+	sum := float64(0)
+	count := 0
+	for _, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		count++
+		sum += v
+	}
+	if count == 0 {
+		return nan
+	}
+	return sum / float64(count)
+}
+
+func medianValue(values []float64) float64 {
+	h := histogram.GetFast()
+	for _, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		h.Update(v)
+	}
+	value := h.Quantile(0.5)
+	histogram.PutFast(h)
+	return value
+}
+
 func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {

app/vmselect/promql/arch.go

@@ -0,0 +1,5 @@
+package promql
+
+import "unsafe"
+
+const maxByteSliceLen = 1<<(31+9*(unsafe.Sizeof(int(0))/8)) - 1

app/vmselect/promql/arch_386.go

@@ -1,3 +0,0 @@
-package promql
-
-const maxByteSliceLen = 1<<31 - 1

app/vmselect/promql/arch_amd64.go

@@ -1,3 +0,0 @@
-package promql
-
-const maxByteSliceLen = 1 << 40

app/vmselect/promql/arch_arm.go

@@ -1,3 +0,0 @@
-package promql
-
-const maxByteSliceLen = 1<<31 - 1

app/vmselect/promql/arch_arm64.go

@@ -1,3 +0,0 @@
-package promql
-
-const maxByteSliceLen = 1 << 40

app/vmselect/promql/eval.go

@@ -440,7 +440,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	var step int64
 	if len(re.Step) > 0 {
 		var err error
-		step, err = DurationValue(re.Step, ec.Step)
+		step, err = PositiveDurationValue(re.Step, ec.Step)
 		if err != nil {
 			return nil, err
 		}
@@ -450,7 +450,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	var window int64
 	if len(re.Window) > 0 {
 		var err error
-		window, err = DurationValue(re.Window, ec.Step)
+		window, err = PositiveDurationValue(re.Window, ec.Step)
 		if err != nil {
 			return nil, err
 		}
@@ -551,7 +551,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 	var window int64
 	if len(windowStr) > 0 {
 		var err error
-		window, err = DurationValue(windowStr, ec.Step)
+		window, err = PositiveDurationValue(windowStr, ec.Step)
 		if err != nil {
 			return nil, err
 		}
@@ -595,7 +595,18 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.
 	pointsPerTimeseries := 1 + (ec.End-ec.Start)/ec.Step
-	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(rssLen*len(rcs)))
+	timeseriesLen := rssLen
+	if iafc != nil {
+		// Incremental aggregates require hold only GOMAXPROCS timeseries in memory.
+		timeseriesLen = runtime.GOMAXPROCS(-1)
+		if iafc.ae.Modifier.Op != "" {
+			// Increase the number of timeseries for non-empty group list: `aggr() by (something)`,
+			// since each group can have own set of time series in memory.
+			// Estimate the number of such groups is lower than 100 :)
+			timeseriesLen *= 100
+		}
+	}
+	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs)))
 	rollupMemorySize := mulNoOverflow(rollupPoints, 16)
 	rml := getRollupMemoryLimiter()
 	if !rml.Get(uint64(rollupMemorySize)) {

app/vmselect/promql/exec.go

@@ -110,7 +110,7 @@ func timeseriesToResult(tss []*timeseries, maySort bool) ([]netstorage.Result, e
 	for i, ts := range tss {
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
 		if _, ok := m[string(bb.B)]; ok {
-			return nil, fmt.Errorf(`duplicate output timeseries: %s%s`, ts.MetricName.MetricGroup, stringMetricName(&ts.MetricName))
+			return nil, fmt.Errorf(`duplicate output timeseries: %s`, stringMetricName(&ts.MetricName))
 		}
 		m[string(bb.B)] = struct{}{}
 

app/vmselect/promql/exec_test.go

@@ -198,6 +198,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("time() offset -100s", func(t *testing.T) {
+		t.Parallel()
+		q := `time() offset -100s`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("(a, b) offset 100s", func(t *testing.T) {
 		t.Parallel()
 		q := `sort((label_set(time(), "foo", "bar"), label_set(time()+10, "foo", "baz")) offset 100s)`
@@ -270,6 +281,30 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run("(a offset -100s, b offset -50s) offset -400s", func(t *testing.T) {
+		t.Parallel()
+		q := `sort((label_set(time() offset -100s, "foo", "bar"), label_set(time()+10, "foo", "baz") offset -50s) offset -400s)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1260, 1460, 1660, 1860, 2060, 2260},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("baz"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1300, 1500, 1700, 1900, 2100, 2300},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run("time()[:100s] offset 100s", func(t *testing.T) {
 		t.Parallel()
 		q := `time()[:100s] offset 100s`
@@ -2289,6 +2324,74 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_quantile(0.6, label_set(100, "le", "200"), "foobar"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foobar"),
+			Value: []byte("lower"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{120, 120, 120, 120, 120, 120},
+			Timestamps: timestampsExpected,
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foobar"),
+			Value: []byte("upper"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(single-value-valid-le-max-phi)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(1, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(single-value-valid-le-min-phi)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{55, 55, 55, 55, 55, 55},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(single-value-valid-le-min-phi-no-zero-bucket)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_quantile(0, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar-phi)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(time() / 2 / 1e3, label_set(100, "le", "200"))`
@@ -2349,7 +2452,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+	t.Run(`histogram_quantile(nan-bucket-count-some)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
 			label_set(90, "foo", "bar", "le", "10")
@@ -2358,7 +2461,7 @@ func TestExecSuccess(t *testing.T) {
 		)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{30, 30, 30, 30, 30, 30},
+			Values:     []float64{10, 10, 10, 10, 10, 10},
 			Timestamps: timestampsExpected,
 		}
 		r.MetricName.Tags = []storage.Tag{{
@@ -2368,7 +2471,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+	t.Run(`histogram_quantile(normal-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.2,
 			label_set(0, "foo", "bar", "le", "10")
@@ -2387,6 +2490,56 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantile(normal-bucket-count, boundsLabel)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_quantile(0.2,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf"),
+			"xxx"
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("lower"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{30, 30, 30, 30, 30, 30},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("upper"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(zero-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
@@ -2397,7 +2550,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+	t.Run(`histogram_quantile(nan-bucket-count-all)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
 			label_set(nan, "foo", "bar", "le", "10")
@@ -2407,6 +2560,190 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`prometheus_buckets(missing-vmrange)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(prometheus_buckets((
+			alias(label_set(time()/20, "foo", "bar", "le", "0.2"), "xyz"),
+			alias(label_set(time()/100, "foo", "bar", "vmrange", "foobar"), "xxx"),
+			alias(label_set(time()/100, "foo", "bar", "vmrange", "30...foobar"), "xxx"),
+			alias(label_set(time()/100, "foo", "bar", "vmrange", "30...40"), "xxx"),
+			alias(label_set(time()/80, "foo", "bar", "vmrange", "0...900", "le", "54"), "yyy"),
+			alias(label_set(time()/40, "foo", "bar", "vmrange", "900...+Inf", "le", "2343"), "yyy"),
+		)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("xxx")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("30"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 12, 14, 16, 18, 20},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("xxx")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("40"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 12, 14, 16, 18, 20},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("xxx")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12.5, 15, 17.5, 20, 22.5, 25},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("yyy")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("900"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{37.5, 45, 52.5, 60, 67.5, 75},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.MetricGroup = []byte("yyy")
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		r6 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{50, 60, 70, 80, 90, 100},
+			Timestamps: timestampsExpected,
+		}
+		r6.MetricName.MetricGroup = []byte("xyz")
+		r6.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("0.2"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6}
+		f(q, resultExpected)
+	})
+	t.Run(`prometheus_buckets(valid)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(prometheus_buckets((
+			alias(label_set(90, "foo", "bar", "vmrange", "0...0"), "xxx"),
+			alias(label_set(time()/20, "foo", "bar", "vmrange", "0...0.2"), "xxx"),
+			alias(label_set(time()/100, "foo", "bar", "vmrange", "0.2...40"), "xxx"),
+			alias(label_set(time()/10, "foo", "bar", "vmrange", "40...Inf"), "xxx"),
+		)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{90, 90, 90, 90, 90, 90},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("xxx")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("0"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{140, 150, 160, 170, 180, 190},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("xxx")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("0.2"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{150, 162, 174, 186, 198, 210},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("xxx")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("40"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{250, 282, 314, 346, 378, 410},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("xxx")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("le"),
+				Value: []byte("Inf"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
 	t.Run(`median_over_time()`, func(t *testing.T) {
 		t.Parallel()
 		q := `median_over_time({})`
@@ -2457,6 +2794,108 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram(scalar)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram(123)+(
+			label_set(0, "le", "1.0e2"),
+			label_set(0, "le", "1.5e2"),
+			label_set(1, "le", "+Inf"),
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1.0e2"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1.5e2"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram(vector)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram((
+			label_set(1, "foo", "bar"),
+			label_set(1.1, "xx", "yy"),
+			alias(1.15, "foobar"),
+		))+(
+			label_set(0, "le", "9.5e-1"),
+			label_set(0, "le", "1.0e0"),
+			label_set(0, "le", "1.5e0"),
+			label_set(1, "le", "+Inf"),
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9.5e-1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1.0e0"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1.5e0"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
 	t.Run(`avg(scalar) wiTHout (xx, yy)`, func(t *testing.T) {
 		t.Parallel()
 		q := `avg wiTHout (xx, yy) (123)`
@@ -2779,6 +3218,126 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`topk_min(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(topk_min(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`bottomk_min(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(bottomk_min(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_max(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`bottomk_max(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(bottomk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_avg(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(topk_avg(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`bottomk_avg(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(bottomk_avg(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_median(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(topk_median(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`bottomk_median(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(bottomk_median(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
 	t.Run(`topk(1, nan_timeseries)`, func(t *testing.T) {
 		t.Parallel()
 		q := `topk(1, label_set(NaN, "foo", "bar") or label_set(time()/150, "baz", "sss")) default 0`
@@ -3710,6 +4269,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`lag()`, func(t *testing.T) {
+		t.Parallel()
+		q := `lag(time()[60s:17s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 10, 6, 2, 15, 11},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`()`, func(t *testing.T) {
 		t.Parallel()
 		q := `()`
@@ -3858,6 +4428,35 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`((1),(2,3))`, func(t *testing.T) {
+		t.Parallel()
+		q := `((
+			alias(1, "x1"),
+		),(
+			alias(2, "x2"),
+			alias(3, "x3"),
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("x1")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("x2")
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("x3")
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`union(more-than-two)`, func(t *testing.T) {
 		t.Parallel()
 		q := `union(
@@ -4143,8 +4742,16 @@ func TestExecError(t *testing.T) {
 	f(`count_values()`)
 	f(`quantile()`)
 	f(`topk()`)
+	f(`topk_min()`)
+	f(`topk_max()`)
+	f(`topk_avg()`)
+	f(`topk_median()`)
 	f(`limitk()`)
 	f(`bottomk()`)
+	f(`bottomk_min()`)
+	f(`bottomk_max()`)
+	f(`bottomk_avg()`)
+	f(`bottomk_median()`)
 	f(`time(123)`)
 	f(`start(1)`)
 	f(`end(1)`)
@@ -4175,6 +4782,8 @@ func TestExecError(t *testing.T) {
 	f(`alias()`)
 	f(`alias(1)`)
 	f(`alias(1, "foo", "bar")`)
+	f(`lifetime()`)
+	f(`lag()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)
@@ -4185,6 +4794,7 @@ func TestExecError(t *testing.T) {
 	f(`clamp_max(1, 1 or label_set(2, "xx", "foo"))`)
 	f(`clamp_min(1, 1 or label_set(2, "xx", "foo"))`)
 	f(`topk(label_set(2, "xx", "foo") or 1, 12)`)
+	f(`topk_avg(label_set(2, "xx", "foo") or 1, 12)`)
 	f(`limitk(label_set(2, "xx", "foo") or 1, 12)`)
 	f(`round(1, 1 or label_set(2, "xx", "foo"))`)
 	f(`histogram_quantile(1 or label_set(2, "xx", "foo"), 1)`)
@@ -4260,27 +4870,27 @@ func testResultsEqual(t *testing.T, result, resultExpected []netstorage.Result)
 	for i := range result {
 		r := &result[i]
 		rExpected := &resultExpected[i]
-		testMetricNamesEqual(t, &r.MetricName, &rExpected.MetricName)
+		testMetricNamesEqual(t, &r.MetricName, &rExpected.MetricName, i)
 		testRowsEqual(t, r.Values, r.Timestamps, rExpected.Values, rExpected.Timestamps)
 	}
 }
 
-func testMetricNamesEqual(t *testing.T, mn, mnExpected *storage.MetricName) {
+func testMetricNamesEqual(t *testing.T, mn, mnExpected *storage.MetricName, pos int) {
 	t.Helper()
 	if string(mn.MetricGroup) != string(mnExpected.MetricGroup) {
-		t.Fatalf(`unexpected MetricGroup; got %q; want %q`, mn.MetricGroup, mnExpected.MetricGroup)
+		t.Fatalf(`unexpected MetricGroup at #%d; got %q; want %q`, pos, mn.MetricGroup, mnExpected.MetricGroup)
 	}
 	if len(mn.Tags) != len(mnExpected.Tags) {
-		t.Fatalf(`unexpected tags count; got %d; want %d`, len(mn.Tags), len(mnExpected.Tags))
+		t.Fatalf(`unexpected tags count at #%d; got %d; want %d`, pos, len(mn.Tags), len(mnExpected.Tags))
 	}
 	for i := range mn.Tags {
 		tag := &mn.Tags[i]
 		tagExpected := &mnExpected.Tags[i]
 		if string(tag.Key) != string(tagExpected.Key) {
-			t.Fatalf(`unexpected tag key; got %q; want %q`, tag.Key, tagExpected.Key)
+			t.Fatalf(`unexpected tag key at #%d,%d; got %q; want %q`, pos, i, tag.Key, tagExpected.Key)
 		}
 		if string(tag.Value) != string(tagExpected.Value) {
-			t.Fatalf(`unexpected tag value; got %q; want %q`, tag.Value, tagExpected.Value)
+			t.Fatalf(`unexpected tag value for key %q at #%d,%d; got %q; want %q`, tag.Key, pos, i, tag.Value, tagExpected.Value)
 		}
 	}
 }

app/vmselect/promql/lexer.go

@@ -105,7 +105,7 @@ again:
 		token = s[:n]
 		goto tokenFoundLabel
 	}
-	if n := scanDuration(s); n > 0 {
+	if n := scanDuration(s, false); n > 0 {
 		token = s[:n]
 		goto tokenFoundLabel
 	}
@@ -368,15 +368,30 @@ func isPositiveNumberPrefix(s string) bool {
 	return isDecimalChar(s[1])
 }
 
-func isDuration(s string) bool {
-	n := scanDuration(s)
+func isPositiveDuration(s string) bool {
+	n := scanDuration(s, false)
 	return n == len(s)
 }
 
+// PositiveDurationValue returns the duration in milliseconds for the given s
+// and the given step.
+func PositiveDurationValue(s string, step int64) (int64, error) {
+	d, err := DurationValue(s, step)
+	if err != nil {
+		return 0, err
+	}
+	if d < 0 {
+		return 0, fmt.Errorf("duration cannot be negative; got %q", s)
+	}
+	return d, nil
+}
+
 // DurationValue returns the duration in milliseconds for the given s
 // and the given step.
+//
+// The returned duration value can be negative.
 func DurationValue(s string, step int64) (int64, error) {
-	n := scanDuration(s)
+	n := scanDuration(s, true)
 	if n != len(s) {
 		return 0, fmt.Errorf("cannot parse duration %q", s)
 	}
@@ -408,8 +423,14 @@ func DurationValue(s string, step int64) (int64, error) {
 	return int64(mp * f * 1e3), nil
 }
 
-func scanDuration(s string) int {
+func scanDuration(s string, canBeNegative bool) int {
+	if len(s) == 0 {
+		return -1
+	}
 	i := 0
+	if s[0] == '-' && canBeNegative {
+		i++
+	}
 	for i < len(s) && isDecimalChar(s[i]) {
 		i++
 	}

app/vmselect/promql/lexer_test.go

@@ -286,61 +286,116 @@ func testLexerError(t *testing.T, s string) {
 	}
 }
 
-func TestDurationSuccess(t *testing.T) {
+func TestPositiveDurationSuccess(t *testing.T) {
+	f := func(s string, step, expectedD int64) {
+		t.Helper()
+		d, err := PositiveDurationValue(s, step)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if d != expectedD {
+			t.Fatalf("unexpected duration; got %d; want %d", d, expectedD)
+		}
+	}
+
 	// Integer durations
-	testDurationSuccess(t, "123s", 42, 123*1000)
-	testDurationSuccess(t, "123m", 42, 123*60*1000)
-	testDurationSuccess(t, "1h", 42, 1*60*60*1000)
-	testDurationSuccess(t, "2d", 42, 2*24*60*60*1000)
-	testDurationSuccess(t, "3w", 42, 3*7*24*60*60*1000)
-	testDurationSuccess(t, "4y", 42, 4*365*24*60*60*1000)
-	testDurationSuccess(t, "1i", 42*1000, 42*1000)
-	testDurationSuccess(t, "3i", 42, 3*42)
+	f("123s", 42, 123*1000)
+	f("123m", 42, 123*60*1000)
+	f("1h", 42, 1*60*60*1000)
+	f("2d", 42, 2*24*60*60*1000)
+	f("3w", 42, 3*7*24*60*60*1000)
+	f("4y", 42, 4*365*24*60*60*1000)
+	f("1i", 42*1000, 42*1000)
+	f("3i", 42, 3*42)
 
 	// Float durations
-	testDurationSuccess(t, "0.234s", 42, 234)
-	testDurationSuccess(t, "1.5s", 42, 1.5*1000)
-	testDurationSuccess(t, "1.5m", 42, 1.5*60*1000)
-	testDurationSuccess(t, "1.2h", 42, 1.2*60*60*1000)
-	testDurationSuccess(t, "1.1d", 42, 1.1*24*60*60*1000)
-	testDurationSuccess(t, "1.1w", 42, 1.1*7*24*60*60*1000)
-	testDurationSuccess(t, "1.3y", 42, 1.3*365*24*60*60*1000)
-	testDurationSuccess(t, "0.1i", 12340, 0.1*12340)
+	f("0.234s", 42, 234)
+	f("1.5s", 42, 1.5*1000)
+	f("1.5m", 42, 1.5*60*1000)
+	f("1.2h", 42, 1.2*60*60*1000)
+	f("1.1d", 42, 1.1*24*60*60*1000)
+	f("1.1w", 42, 1.1*7*24*60*60*1000)
+	f("1.3y", 42, 1.3*365*24*60*60*1000)
+	f("0.1i", 12340, 0.1*12340)
 }
 
-func testDurationSuccess(t *testing.T, s string, step, expectedD int64) {
-	t.Helper()
-	d, err := DurationValue(s, step)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+func TestPositiveDurationError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		if isPositiveDuration(s) {
+			t.Fatalf("unexpected valid duration %q", s)
+		}
+		d, err := PositiveDurationValue(s, 42)
+		if err == nil {
+			t.Fatalf("expecting non-nil error for duration %q", s)
+		}
+		if d != 0 {
+			t.Fatalf("expecting zero duration; got %d", d)
+		}
 	}
-	if d != expectedD {
-		t.Fatalf("unexpected duration; got %d; want %d", d, expectedD)
+	f("")
+	f("foo")
+	f("m")
+	f("12")
+	f("1.23")
+	f("1.23mm")
+	f("123q")
+	f("-123s")
+}
+
+func TestDurationSuccess(t *testing.T) {
+	f := func(s string, step, expectedD int64) {
+		t.Helper()
+		d, err := DurationValue(s, step)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if d != expectedD {
+			t.Fatalf("unexpected duration; got %d; want %d", d, expectedD)
+		}
 	}
+
+	// Integer durations
+	f("123s", 42, 123*1000)
+	f("-123s", 42, -123*1000)
+	f("123m", 42, 123*60*1000)
+	f("1h", 42, 1*60*60*1000)
+	f("2d", 42, 2*24*60*60*1000)
+	f("3w", 42, 3*7*24*60*60*1000)
+	f("4y", 42, 4*365*24*60*60*1000)
+	f("1i", 42*1000, 42*1000)
+	f("3i", 42, 3*42)
+	f("-3i", 42, -3*42)
+
+	// Float durations
+	f("0.234s", 42, 234)
+	f("-0.234s", 42, -234)
+	f("1.5s", 42, 1.5*1000)
+	f("1.5m", 42, 1.5*60*1000)
+	f("1.2h", 42, 1.2*60*60*1000)
+	f("1.1d", 42, 1.1*24*60*60*1000)
+	f("1.1w", 42, 1.1*7*24*60*60*1000)
+	f("1.3y", 42, 1.3*365*24*60*60*1000)
+	f("-1.3y", 42, -1.3*365*24*60*60*1000)
+	f("0.1i", 12340, 0.1*12340)
 }
 
 func TestDurationError(t *testing.T) {
-	testDurationError(t, "")
-	testDurationError(t, "foo")
-	testDurationError(t, "m")
-	testDurationError(t, "12")
-	testDurationError(t, "1.23")
-	testDurationError(t, "1.23mm")
-	testDurationError(t, "123q")
-}
-
-func testDurationError(t *testing.T, s string) {
-	t.Helper()
-
-	if isDuration(s) {
-		t.Fatalf("unexpected valud duration %q", s)
-	}
-
-	d, err := DurationValue(s, 42)
-	if err == nil {
-		t.Fatalf("expecting non-nil error for duration %q", s)
-	}
-	if d != 0 {
-		t.Fatalf("expecting zero duration; got %d", d)
-	}
+	f := func(s string) {
+		t.Helper()
+		d, err := DurationValue(s, 42)
+		if err == nil {
+			t.Fatalf("expecting non-nil error for duration %q", s)
+		}
+		if d != 0 {
+			t.Fatalf("expecting zero duration; got %d", d)
+		}
+	}
+	f("")
+	f("foo")
+	f("m")
+	f("12")
+	f("1.23")
+	f("1.23mm")
+	f("123q")
 }

app/vmselect/promql/parser.go

@@ -116,13 +116,17 @@ func removeParensExpr(e expr) expr {
 		return fe
 	}
 	if pe, ok := e.(*parensExpr); ok {
+		args := *pe
+		for i, arg := range args {
+			args[i] = removeParensExpr(arg)
+		}
 		if len(*pe) == 1 {
-			return removeParensExpr((*pe)[0])
+			return args[0]
 		}
 		// Treat parensExpr as a function with empty name, i.e. union()
 		fe := &funcExpr{
 			Name: "",
-			Args: *pe,
+			Args: args,
 		}
 		return fe
 	}
@@ -1169,7 +1173,7 @@ func (p *parser) parseWindowAndStep() (string, string, bool, error) {
 	}
 	var window string
 	if !strings.HasPrefix(p.lex.Token, ":") {
-		window, err = p.parseDuration()
+		window, err = p.parsePositiveDuration()
 		if err != nil {
 			return "", "", false, err
 		}
@@ -1188,7 +1192,7 @@ func (p *parser) parseWindowAndStep() (string, string, bool, error) {
 			}
 		}
 		if p.lex.Token != "]" {
-			step, err = p.parseDuration()
+			step, err = p.parsePositiveDuration()
 			if err != nil {
 				return "", "", false, err
 			}
@@ -1218,13 +1222,34 @@ func (p *parser) parseOffset() (string, error) {
 }
 
 func (p *parser) parseDuration() (string, error) {
-	if !isDuration(p.lex.Token) {
+	isNegative := false
+	if p.lex.Token == "-" {
+		isNegative = true
+		if err := p.lex.Next(); err != nil {
+			return "", err
+		}
+	}
+	if !isPositiveDuration(p.lex.Token) {
 		return "", fmt.Errorf(`duration: unexpected token %q; want "duration"`, p.lex.Token)
 	}
 	d := p.lex.Token
 	if err := p.lex.Next(); err != nil {
 		return "", err
 	}
+	if isNegative {
+		d = "-" + d
+	}
+	return d, nil
+}
+
+func (p *parser) parsePositiveDuration() (string, error) {
+	d, err := p.parseDuration()
+	if err != nil {
+		return "", err
+	}
+	if strings.HasPrefix(d, "-") {
+		return "", fmt.Errorf("positiveDuration: expecting positive duration; got %q", d)
+	}
 	return d, nil
 }
 
@@ -1265,6 +1290,22 @@ func (p *parser) parseIdentExpr() (expr, error) {
 	}
 }
 
+// IsRollup verifies whether s is a rollup with non-empty window.
+//
+// It returns the wrapped query with the corresponding window, step and offset.
+func IsRollup(s string) (childQuery string, window, step, offset string) {
+	expr, err := parsePromQLWithCache(s)
+	if err != nil {
+		return
+	}
+	re, ok := expr.(*rollupExpr)
+	if !ok || len(re.Window) == 0 {
+		return
+	}
+	wrappedQuery := re.Expr.AppendString(nil)
+	return string(wrappedQuery), re.Window, re.Step, re.Offset
+}
+
 // IsMetricSelectorWithRollup verifies whether s contains PromQL metric selector
 // wrapped into rollup.
 //

app/vmselect/promql/parser_test.go

@@ -77,15 +77,18 @@ func TestParsePromQLSuccess(t *testing.T) {
 	same(`{}[5m:3s]`)
 	another(`{}[ 5m : 3s ]`, `{}[5m:3s]`)
 	same(`{} offset 5m`)
+	same(`{} offset -5m`)
 	same(`{}[5m] offset 10y`)
 	same(`{}[5.3m:3.4s] offset 10y`)
 	same(`{}[:3.4s] offset 10y`)
+	same(`{}[:3.4s] offset -10y`)
 	same(`{Foo="bAR"}`)
 	same(`{foo="bar"}`)
 	same(`{foo="bar"}[5m]`)
 	same(`{foo="bar"}[5m:]`)
 	same(`{foo="bar"}[5m:3s]`)
 	same(`{foo="bar"} offset 10y`)
+	same(`{foo="bar"} offset -10y`)
 	same(`{foo="bar"}[5m] offset 10y`)
 	same(`{foo="bar"}[5m:3s] offset 10y`)
 	another(`{foo="bar"}[5m] oFFSEt 10y`, `{foo="bar"}[5m] offset 10y`)
@@ -252,6 +255,8 @@ func TestParsePromQLSuccess(t *testing.T) {
 	another(`(-foo + ((bar) / (baz))) + ((23))`, `((0 - foo) + (bar / baz)) + 23`)
 	another(`(FOO + ((Bar) / (baZ))) + ((23))`, `(FOO + (Bar / baZ)) + 23`)
 	same(`(foo, bar)`)
+	another(`((foo, bar),(baz))`, `((foo, bar), baz)`)
+	same(`(foo, (bar, baz), ((x, y), (z, y), xx))`)
 	another(`1+(foo, bar,)`, `1 + (foo, bar)`)
 	another(`((foo(bar,baz)), (1+(2)+(3,4)+()))`, `(foo(bar, baz), (3 + (3, 4)) + ())`)
 	same(`()`)
@@ -456,7 +461,6 @@ func TestParsePromQLError(t *testing.T) {
 
 	// invalid metricExpr
 	f(`{__name__="ff"} offset 55`)
-	f(`{__name__="ff"} offset -5m`)
 	f(`foo[55]`)
 	f(`m[-5m]`)
 	f(`{`)
@@ -489,6 +493,9 @@ func TestParsePromQLError(t *testing.T) {
 	f(`m[5m:-`)
 	f(`m[5m:-1`)
 	f(`m[5m:-1]`)
+	f(`m[5m:-1s]`)
+	f(`m[-5m:1s]`)
+	f(`m[-5m:-1s]`)
 	f(`m[:`)
 	f(`m[:-`)
 	f(`m[:1]`)

app/vmselect/promql/rollup.go

@@ -3,7 +3,6 @@ package promql
 import (
 	"fmt"
 	"math"
-	"sort"
 	"strings"
 	"sync"
 
@@ -23,8 +22,8 @@ var rollupFuncs = map[string]newRollupFunc{
 	"deriv_fast":         newRollupFuncOneArg(rollupDerivFast),
 	"holt_winters":       newRollupHoltWinters,
 	"idelta":             newRollupFuncOneArg(rollupIdelta),
-	"increase":           newRollupFuncOneArg(rollupDelta),  // + rollupFuncsRemoveCounterResets
-	"irate":              newRollupFuncOneArg(rollupIderiv), // + rollupFuncsRemoveCounterResets
+	"increase":           newRollupFuncOneArg(rollupIncrease), // + rollupFuncsRemoveCounterResets
+	"irate":              newRollupFuncOneArg(rollupIderiv),   // + rollupFuncsRemoveCounterResets
 	"predict_linear":     newRollupPredictLinear,
 	"rate":               newRollupFuncOneArg(rollupDerivFast), // + rollupFuncsRemoveCounterResets
 	"resets":             newRollupFuncOneArg(rollupResets),
@@ -48,6 +47,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"integrate":           newRollupFuncOneArg(rollupIntegrate),
 	"ideriv":              newRollupFuncOneArg(rollupIderiv),
 	"lifetime":            newRollupFuncOneArg(rollupLifetime),
+	"lag":                 newRollupFuncOneArg(rollupLag),
 	"scrape_interval":     newRollupFuncOneArg(rollupScrapeInterval),
 	"rollup":              newRollupFuncOneArg(rollupFake),
 	"rollup_rate":         newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
@@ -113,8 +113,10 @@ type rollupFuncArg struct {
 	values        []float64
 	timestamps    []int64
 
-	idx  int
-	step int64
+	currTimestamp int64
+	idx           int
+	step          int64
+	realPrevValue float64
 }
 
 func (rfa *rollupFuncArg) reset() {
@@ -122,8 +124,10 @@ func (rfa *rollupFuncArg) reset() {
 	rfa.prevTimestamp = 0
 	rfa.values = nil
 	rfa.timestamps = nil
+	rfa.currTimestamp = 0
 	rfa.idx = 0
 	rfa.step = 0
+	rfa.realPrevValue = nan
 }
 
 // rollupFunc must return rollup value for the given rfa.
@@ -202,6 +206,7 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 	rfa := getRollupFuncArg()
 	rfa.idx = 0
 	rfa.step = rc.Step
+	rfa.realPrevValue = nan
 
 	i := 0
 	j := 0
@@ -226,6 +231,10 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 
 		rfa.values = values[i:j]
 		rfa.timestamps = timestamps[i:j]
+		rfa.currTimestamp = tEnd
+		if i > 0 {
+			rfa.realPrevValue = values[i-1]
+		}
 		value := rc.Func(rfa)
 		rfa.idx++
 		dstValues = append(dstValues, value)
@@ -269,17 +278,42 @@ func seekFirstTimestampIdxAfter(timestamps []int64, seekTimestamp int64, nHint i
 		return startIdx + len(timestamps)
 	}
 	// Slow path: too big len(timestamps), so use binary search.
-	i := sort.Search(len(timestamps), func(n int) bool {
-		return n >= 0 && n < len(timestamps) && timestamps[n] > seekTimestamp
-	})
-	return startIdx + i
+	i := binarySearchInt64(timestamps, seekTimestamp+1)
+	return startIdx + int(i)
+}
+
+func binarySearchInt64(a []int64, v int64) uint {
+	// Copy-pasted sort.Search from https://golang.org/src/sort/search.go?s=2246:2286#L49
+	i, j := uint(0), uint(len(a))
+	for i < j {
+		h := (i + j) >> 1
+		if h < uint(len(a)) && a[h] < v {
+			i = h + 1
+		} else {
+			j = h
+		}
+	}
+	return i
 }
 
 func getMaxPrevInterval(timestamps []int64) int64 {
 	if len(timestamps) < 2 {
 		return int64(maxSilenceInterval)
 	}
-	d := (timestamps[len(timestamps)-1] - timestamps[0]) / int64(len(timestamps)-1)
+
+	// Estimate scrape interval as 0.6 quantile for the first 100 intervals.
+	h := histogram.GetFast()
+	tsPrev := timestamps[0]
+	timestamps = timestamps[1:]
+	if len(timestamps) > 100 {
+		timestamps = timestamps[:100]
+	}
+	for _, ts := range timestamps {
+		h.Update(float64(ts - tsPrev))
+		tsPrev = ts
+	}
+	d := int64(h.Quantile(0.6))
+	histogram.PutFast(h)
 	if d <= 0 {
 		return int64(maxSilenceInterval)
 	}
@@ -666,6 +700,14 @@ func rollupStdvar(rfa *rollupFuncArg) float64 {
 }
 
 func rollupDelta(rfa *rollupFuncArg) float64 {
+	return rollupDeltaInternal(rfa, false)
+}
+
+func rollupIncrease(rfa *rollupFuncArg) float64 {
+	return rollupDeltaInternal(rfa, true)
+}
+
+func rollupDeltaInternal(rfa *rollupFuncArg, canUseRealPrevValue bool) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
 	values := rfa.values
@@ -675,6 +717,10 @@ func rollupDelta(rfa *rollupFuncArg) float64 {
 			return nan
 		}
 		if len(values) == 1 {
+			if canUseRealPrevValue && !math.IsNaN(rfa.realPrevValue) {
+				// Fix against removeCounterResets.
+				return values[0] - rfa.realPrevValue
+			}
 			// Assume that the previous non-existing value was 0.
 			return values[0]
 		}
@@ -799,6 +845,18 @@ func rollupLifetime(rfa *rollupFuncArg) float64 {
 	return float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) * 1e-3
 }
 
+func rollupLag(rfa *rollupFuncArg) float64 {
+	// Calculate the duration between the current timestamp and the last data point.
+	timestamps := rfa.timestamps
+	if len(timestamps) == 0 {
+		if math.IsNaN(rfa.prevValue) {
+			return nan
+		}
+		return float64(rfa.currTimestamp-rfa.prevTimestamp) * 1e-3
+	}
+	return float64(rfa.currTimestamp-timestamps[len(timestamps)-1]) * 1e-3
+}
+
 func rollupScrapeInterval(rfa *rollupFuncArg) float64 {
 	// Calculate the average interval between data points.
 	timestamps := rfa.timestamps
@@ -970,6 +1028,8 @@ func rollupIntegrate(rfa *rollupFuncArg) float64 {
 		timestamp := timestamps[i]
 		dt := float64(timestamp-prevTimestamp) * 1e-3
 		sum += 0.5 * (v + prevValue) * dt
+		prevTimestamp = timestamp
+		prevValue = v
 	}
 	return sum
 }

app/vmselect/promql/rollup_result_cache_test.go

@@ -388,7 +388,7 @@ func testTimeseriesEqual(t *testing.T, tss, tssExpected []*timeseries) {
 	}
 	for i, ts := range tss {
 		tsExpected := tssExpected[i]
-		testMetricNamesEqual(t, &ts.MetricName, &tsExpected.MetricName)
+		testMetricNamesEqual(t, &ts.MetricName, &tsExpected.MetricName, i)
 		testRowsEqual(t, ts.Values, ts.Timestamps, tsExpected.Values, tsExpected.Timestamps)
 	}
 }

app/vmselect/promql/rollup_test.go

@@ -291,7 +291,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("stdvar_over_time", 945.7430555555555)
 	f("first_over_time", 123)
 	f("last_over_time", 34)
-	f("integrate", 61.0275)
+	f("integrate", 5.4705)
 	f("distinct_over_time", 8)
 	f("ideriv", 0)
 	f("decreases_over_time", 5)
@@ -632,6 +632,20 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{10, 50, 90, 130}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("lag", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:   rollupLag,
+			Start:  0,
+			End:    160,
+			Step:   40,
+			Window: 0,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
+		timestampsExpected := []int64{0, 40, 80, 120, 160}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
 	t.Run("lifetime_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupLifetime,
@@ -796,7 +810,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4.6035, 4.3934999999999995, 2.166, 0.34}
+		valuesExpected := []float64{nan, 1.526, 2.2795, 1.325, 0.34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -830,6 +844,27 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 	})
 }
 
+func TestRollupBigNumberOfValues(t *testing.T) {
+	const srcValuesCount = 1e4
+	rc := rollupConfig{
+		Func:   rollupDefault,
+		End:    srcValuesCount,
+		Step:   srcValuesCount / 5,
+		Window: srcValuesCount / 4,
+	}
+	rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+	srcValues := make([]float64, srcValuesCount)
+	srcTimestamps := make([]int64, srcValuesCount)
+	for i := 0; i < srcValuesCount; i++ {
+		srcValues[i] = float64(i)
+		srcTimestamps[i] = int64(i / 2)
+	}
+	values := rc.Do(nil, srcValues, srcTimestamps)
+	valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan}
+	timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000}
+	testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+}
+
 func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExpected []float64, timestampsExpected []int64) {
 	t.Helper()
 	if len(values) != len(valuesExpected) {

app/vmselect/promql/timeseries.go

@@ -288,7 +288,6 @@ func marshalMetricTagsFast(dst []byte, tags []storage.Tag) []byte {
 }
 
 func marshalMetricNameSorted(dst []byte, mn *storage.MetricName) []byte {
-	// Do not marshal AccountID and ProjectID, since they are unused.
 	dst = marshalBytesFast(dst, mn.MetricGroup)
 	sortMetricTags(mn.Tags)
 	dst = marshalMetricTagsFast(dst, mn.Tags)

app/vmselect/promql/transform.go

@@ -91,6 +91,7 @@ var transformFuncs = map[string]transformFunc{
 	"cos":                newTransformFuncOneArg(transformCos),
 	"asin":               newTransformFuncOneArg(transformAsin),
 	"acos":               newTransformFuncOneArg(transformAcos),
+	"prometheus_buckets": transformPrometheusBuckets,
 }
 
 func getTransformFunc(s string) transformFunc {
@@ -272,14 +273,152 @@ func transformFloor(v float64) float64 {
 	return math.Floor(v)
 }
 
+func transformPrometheusBuckets(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 1); err != nil {
+		return nil, err
+	}
+	rvs := vmrangeBucketsToLE(args[0])
+	return rvs, nil
+}
+
+func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
+	rvs := make([]*timeseries, 0, len(tss))
+
+	// Group timeseries by MetricGroup+tags excluding `vmrange` tag.
+	type x struct {
+		startStr string
+		endStr   string
+		start    float64
+		end      float64
+		ts       *timeseries
+	}
+	m := make(map[string][]x)
+	bb := bbPool.Get()
+	defer bbPool.Put(bb)
+	for _, ts := range tss {
+		vmrange := ts.MetricName.GetTagValue("vmrange")
+		if len(vmrange) == 0 {
+			if le := ts.MetricName.GetTagValue("le"); len(le) > 0 {
+				// Keep Prometheus-compatible buckets.
+				rvs = append(rvs, ts)
+			}
+			continue
+		}
+		n := strings.Index(bytesutil.ToUnsafeString(vmrange), "...")
+		if n < 0 {
+			continue
+		}
+		startStr := string(vmrange[:n])
+		start, err := strconv.ParseFloat(startStr, 64)
+		if err != nil {
+			continue
+		}
+		endStr := string(vmrange[n+len("..."):])
+		end, err := strconv.ParseFloat(endStr, 64)
+		if err != nil {
+			continue
+		}
+		ts.MetricName.RemoveTag("le")
+		ts.MetricName.RemoveTag("vmrange")
+		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
+		m[string(bb.B)] = append(m[string(bb.B)], x{
+			startStr: startStr,
+			endStr:   endStr,
+			start:    start,
+			end:      end,
+			ts:       ts,
+		})
+	}
+
+	// Convert `vmrange` label in each group of time series to `le` label.
+	copyTS := func(src *timeseries, leStr string) *timeseries {
+		var ts timeseries
+		ts.CopyFromShallowTimestamps(src)
+		values := ts.Values
+		for i := range values {
+			values[i] = 0
+		}
+		ts.MetricName.RemoveTag("le")
+		ts.MetricName.AddTag("le", leStr)
+		return &ts
+	}
+	isZeroTS := func(ts *timeseries) bool {
+		for _, v := range ts.Values {
+			if v > 0 {
+				return false
+			}
+		}
+		return true
+	}
+	for _, xss := range m {
+		sort.Slice(xss, func(i, j int) bool { return xss[i].end < xss[j].end })
+		xssNew := make([]x, 0, len(xss)+2)
+		var xsPrev x
+		for _, xs := range xss {
+			ts := xs.ts
+			if isZeroTS(ts) {
+				// Skip time series with zeros. They are substituted by xssNew below.
+				continue
+			}
+			if xs.start != xsPrev.end {
+				xssNew = append(xssNew, x{
+					endStr: xs.startStr,
+					end:    xs.start,
+					ts:     copyTS(ts, xs.startStr),
+				})
+			}
+			ts.MetricName.AddTag("le", xs.endStr)
+			xssNew = append(xssNew, xs)
+			xsPrev = xs
+		}
+		if !math.IsInf(xsPrev.end, 1) {
+			xssNew = append(xssNew, x{
+				endStr: "+Inf",
+				end:    math.Inf(1),
+				ts:     copyTS(xsPrev.ts, "+Inf"),
+			})
+		}
+		xss = xssNew
+		for i := range xss[0].ts.Values {
+			count := float64(0)
+			for _, xs := range xss {
+				ts := xs.ts
+				v := ts.Values[i]
+				if !math.IsNaN(v) && v > 0 {
+					count += v
+				}
+				ts.Values[i] = count
+			}
+		}
+		for _, xs := range xss {
+			rvs = append(rvs, xs.ts)
+		}
+	}
+	return rvs
+}
+
 func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
-	if err := expectTransformArgsNum(args, 2); err != nil {
-		return nil, err
+	if len(args) < 2 || len(args) > 3 {
+		return nil, fmt.Errorf("unexpected number of args; got %d; want 2...3", len(args))
 	}
 	phis, err := getScalar(args[0], 0)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot parse phi: %s", err)
+	}
+
+	// Convert buckets with `vmrange` labels to buckets with `le` labels.
+	tss := vmrangeBucketsToLE(args[1])
+
+	// Parse boundsLabel. See https://github.com/prometheus/prometheus/issues/5706 for details.
+	var boundsLabel string
+	if len(args) > 2 {
+		s, err := getString(args[2], 2)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse boundsLabel (arg #3): %s", err)
+		}
+		boundsLabel = s
 	}
 
 	// Group metrics by all tags excluding "le"
@@ -289,7 +428,7 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	m := make(map[string][]x)
 	bb := bbPool.Get()
-	for _, ts := range args[1] {
+	for _, ts := range tss {
 		tagValue := ts.MetricName.GetTagValue("le")
 		if len(tagValue) == 0 {
 			continue
@@ -313,23 +452,21 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	lastNonInf := func(i int, xss []x) float64 {
 		for len(xss) > 0 {
 			xsLast := xss[len(xss)-1]
-			if xsLast.ts.Values[i] == 0 {
+			v := xsLast.ts.Values[i]
+			if v == 0 {
 				return nan
 			}
-			if !math.IsInf(xsLast.le, 0) {
-				break
+			if !math.IsNaN(v) && !math.IsInf(xsLast.le, 0) {
+				return xsLast.le
 			}
 			xss = xss[:len(xss)-1]
 		}
-		if len(xss) == 0 {
-			return nan
-		}
-		return xss[len(xss)-1].le
+		return nan
 	}
-	quantile := func(i int, phis []float64, xss []x) float64 {
+	quantile := func(i int, phis []float64, xss []x) (q, lower, upper float64) {
 		phi := phis[i]
 		if math.IsNaN(phi) {
-			return nan
+			return nan, nan, nan
 		}
 		// Fix broken buckets.
 		// They are already sorted by le, so their values must be in ascending order,
@@ -337,45 +474,62 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 		vPrev := float64(0)
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
-			if math.IsNaN(v) || v < vPrev {
+			if v < vPrev {
 				xs.ts.Values[i] = vPrev
-			} else {
+			} else if !math.IsNaN(v) {
 				vPrev = v
 			}
 		}
-		if len(xss) == 0 {
-			return nan
+		vLast := nan
+		for len(xss) > 0 {
+			vLast = xss[len(xss)-1].ts.Values[i]
+			if !math.IsNaN(vLast) {
+				break
+			}
+			xss = xss[:len(xss)-1]
+		}
+		if vLast == 0 || math.IsNaN(vLast) {
+			return nan, nan, nan
 		}
 		if phi < 0 {
-			return -inf
+			return -inf, -inf, xss[0].ts.Values[i]
 		}
 		if phi > 1 {
-			return inf
-		}
-		vLast := xss[len(xss)-1].ts.Values[i]
-		if vLast == 0 {
-			return nan
+			return inf, vLast, inf
 		}
 		vReq := vLast * phi
 		vPrev = 0
 		lePrev := float64(0)
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
+			if math.IsNaN(v) {
+				// Skip NaNs - they may appear if the selected time range
+				// contains multiple different bucket sets.
+				continue
+			}
 			le := xs.le
+			if v <= 0 {
+				// Skip zero buckets.
+				lePrev = le
+				continue
+			}
 			if v < vReq {
 				vPrev = v
 				lePrev = le
 				continue
 			}
 			if math.IsInf(le, 0) {
-				return lastNonInf(i, xss)
+				vv := lastNonInf(i, xss)
+				return vv, vv, inf
 			}
 			if v == vPrev {
-				return lePrev
+				return lePrev, lePrev, v
 			}
-			return lePrev + (le-lePrev)*(vReq-vPrev)/(v-vPrev)
+			vv := lePrev + (le-lePrev)*(vReq-vPrev)/(v-vPrev)
+			return vv, lePrev, le
 		}
-		return lastNonInf(i, xss)
+		vv := lastNonInf(i, xss)
+		return vv, vv, inf
 	}
 	rvs := make([]*timeseries, 0, len(m))
 	for _, xss := range m {
@@ -383,12 +537,31 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 			return xss[i].le < xss[j].le
 		})
 		dst := xss[0].ts
+		var tsLower, tsUpper *timeseries
+		if len(boundsLabel) > 0 {
+			tsLower = &timeseries{}
+			tsLower.CopyFromShallowTimestamps(dst)
+			tsLower.MetricName.RemoveTag(boundsLabel)
+			tsLower.MetricName.AddTag(boundsLabel, "lower")
+			tsUpper = &timeseries{}
+			tsUpper.CopyFromShallowTimestamps(dst)
+			tsUpper.MetricName.RemoveTag(boundsLabel)
+			tsUpper.MetricName.AddTag(boundsLabel, "upper")
+		}
 		for i := range dst.Values {
-			dst.Values[i] = quantile(i, phis, xss)
+			v, lower, upper := quantile(i, phis, xss)
+			dst.Values[i] = v
+			if len(boundsLabel) > 0 {
+				tsLower.Values[i] = lower
+				tsUpper.Values[i] = upper
+			}
 		}
 		rvs = append(rvs, dst)
+		if len(boundsLabel) > 0 {
+			rvs = append(rvs, tsLower)
+			rvs = append(rvs, tsUpper)
+		}
 	}
-
 	return rvs, nil
 }
 

app/vmstorage/main.go

@@ -305,6 +305,9 @@ func registerStorageMetrics() {
 		return float64(idbm().PartsRefCount)
 	})
 
+	metrics.NewGauge(`vm_new_timeseries_created_total`, func() float64 {
+		return float64(idbm().NewTimeseriesCreated)
+	})
 	metrics.NewGauge(`vm_missing_tsids_for_metric_id_total`, func() float64 {
 		return float64(idbm().MissingTSIDsForMetricID)
 	})
@@ -320,6 +323,12 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_date_metric_ids_search_hits_total`, func() float64 {
 		return float64(idbm().DateMetricIDsSearchHits)
 	})
+	metrics.NewGauge(`vm_index_blocks_with_metric_ids_processed_total`, func() float64 {
+		return float64(idbm().IndexBlocksWithMetricIDsProcessed)
+	})
+	metrics.NewGauge(`vm_index_blocks_with_metric_ids_incorrect_order_total`, func() float64 {
+		return float64(idbm().IndexBlocksWithMetricIDsIncorrectOrder)
+	})
 
 	metrics.NewGauge(`vm_assisted_merges_total{type="storage/small"}`, func() float64 {
 		return float64(tm().SmallAssistedMerges)
@@ -398,6 +407,24 @@ func registerStorageMetrics() {
 		return float64(idbm().ItemsCount)
 	})
 
+	metrics.NewGauge(`vm_date_range_search_calls_total`, func() float64 {
+		return float64(idbm().DateRangeSearchCalls)
+	})
+	metrics.NewGauge(`vm_date_range_hits_total`, func() float64 {
+		return float64(idbm().DateRangeSearchHits)
+	})
+
+	metrics.NewGauge(`vm_missing_metric_names_for_metric_id_total`, func() float64 {
+		return float64(idbm().MissingMetricNamesForMetricID)
+	})
+
+	metrics.NewGauge(`vm_date_metric_id_cache_syncs_total`, func() float64 {
+		return float64(m().DateMetricIDCacheSyncsCount)
+	})
+	metrics.NewGauge(`vm_date_metric_id_cache_resets_total`, func() float64 {
+		return float64(m().DateMetricIDCacheResetsCount)
+	})
+
 	metrics.NewGauge(`vm_cache_entries{type="storage/tsid"}`, func() float64 {
 		return float64(m().TSIDCacheSize)
 	})
@@ -447,6 +474,9 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_size_bytes{type="storage/date_metricID"}`, func() float64 {
 		return float64(m().DateMetricIDCacheSizeBytes)
 	})
+	metrics.NewGauge(`vm_cache_size_bytes{type="storage/hour_metric_ids"}`, func() float64 {
+		return float64(m().HourMetricIDCacheSizeBytes)
+	})
 	metrics.NewGauge(`vm_cache_size_bytes{type="indexdb/tagFilters"}`, func() float64 {
 		return float64(idbm().TagCacheSizeBytes)
 	})
@@ -463,9 +493,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_requests_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheRequests)
 	})
-	metrics.NewGauge(`vm_cache_requests_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheRequests)
-	})
 	metrics.NewGauge(`vm_cache_requests_total{type="storage/bigIndexBlocks"}`, func() float64 {
 		return float64(tm().BigIndexBlocksCacheRequests)
 	})
@@ -497,9 +524,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_misses_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheMisses)
 	})
-	metrics.NewGauge(`vm_cache_misses_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheMisses)
-	})
 	metrics.NewGauge(`vm_cache_misses_total{type="storage/bigIndexBlocks"}`, func() float64 {
 		return float64(tm().BigIndexBlocksCacheMisses)
 	})
@@ -532,7 +556,4 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_cache_collisions_total{type="storage/metricName"}`, func() float64 {
 		return float64(m().MetricNameCacheCollisions)
 	})
-	metrics.NewGauge(`vm_cache_collisions_total{type="storage/date_metricID"}`, func() float64 {
-		return float64(m().DateMetricIDCacheCollisions)
-	})
 }

dashboards/victoriametrics.json

@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "6.3.5"
+      "version": "6.5.0"
     },
     {
       "type": "panel",
@@ -60,12 +60,12 @@
       }
     ]
   },
-  "description": "Overview for single node VictoriaMetrics v1.28.0 or higher",
+  "description": "Overview for single node VictoriaMetrics v1.30.3 or higher",
   "editable": true,
   "gnetId": 10229,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1572208904768,
+  "iteration": 1575825261972,
   "links": [
     {
       "icon": "doc",
@@ -96,6 +96,7 @@
   "panels": [
     {
       "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -109,6 +110,7 @@
     },
     {
       "content": "<div style=\"text-align: center; font-size: 2em\">$version</div>",
+      "datasource": "${DS_PROMETHEUS}",
       "description": "",
       "gridPos": {
         "h": 3,
@@ -470,6 +472,7 @@
     },
     {
       "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -496,6 +499,7 @@
         "x": 0,
         "y": 11
       },
+      "hiddenSeries": false,
       "id": 12,
       "legend": {
         "alignAsTable": true,
@@ -589,6 +593,7 @@
         "x": 12,
         "y": 11
       },
+      "hiddenSeries": false,
       "id": 22,
       "legend": {
         "alignAsTable": true,
@@ -682,6 +687,7 @@
         "x": 0,
         "y": 19
       },
+      "hiddenSeries": false,
       "id": 51,
       "legend": {
         "avg": false,
@@ -778,6 +784,7 @@
         "x": 12,
         "y": 19
       },
+      "hiddenSeries": false,
       "id": 33,
       "legend": {
         "avg": false,
@@ -873,7 +880,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "${DS_PROMETHEUS}",
-      "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+      "description": "Shows how many of new time-series are created every second.  High churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -882,6 +889,95 @@
         "x": 0,
         "y": 27
       },
+      "hiddenSeries": false,
+      "id": 66,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(vm_new_timeseries_created_total{job=\"$job\"}[5m]))",
+          "legendFormat": "churn rate",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Churn rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "hiddenSeries": false,
       "id": 35,
       "legend": {
         "alignAsTable": true,
@@ -960,6 +1056,99 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 35
+      },
+      "hiddenSeries": false,
+      "id": 60,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(vm_slow_queries_total{job=\"$job\"}[5m]))",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "slow queries rate",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Slow queries",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -973,8 +1162,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 27
+        "y": 35
       },
+      "hiddenSeries": false,
       "id": 59,
       "legend": {
         "alignAsTable": true,
@@ -1082,8 +1272,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 35
+        "y": 43
       },
+      "hiddenSeries": false,
       "id": 37,
       "legend": {
         "avg": false,
@@ -1174,8 +1365,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 35
+        "y": 43
       },
+      "hiddenSeries": false,
       "id": 49,
       "legend": {
         "avg": false,
@@ -1255,11 +1447,12 @@
     },
     {
       "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 43
+        "y": 51
       },
       "id": 14,
       "panels": [],
@@ -1279,8 +1472,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 44
+        "y": 52
       },
+      "hiddenSeries": false,
       "id": 10,
       "legend": {
         "alignAsTable": true,
@@ -1378,8 +1572,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 44
+        "y": 52
       },
+      "hiddenSeries": false,
       "id": 34,
       "legend": {
         "avg": false,
@@ -1483,8 +1678,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 52
+        "y": 60
       },
+      "hiddenSeries": false,
       "id": 30,
       "legend": {
         "avg": false,
@@ -1573,8 +1769,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 52
+        "y": 60
       },
+      "hiddenSeries": false,
       "id": 36,
       "legend": {
         "avg": false,
@@ -1663,8 +1860,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 60
+        "y": 68
       },
+      "hiddenSeries": false,
       "id": 53,
       "legend": {
         "avg": false,
@@ -1753,8 +1951,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 60
+        "y": 68
       },
+      "hiddenSeries": false,
       "id": 55,
       "legend": {
         "avg": false,
@@ -1829,6 +2028,184 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "description": "The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 76
+      },
+      "hiddenSeries": false,
+      "id": 62,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(vm_active_merges{job=\"$job\"}) by(type)",
+          "legendFormat": "{{type}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Active merges",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "description": "The number of rows merged per second by storage nodes.",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 76
+      },
+      "hiddenSeries": false,
+      "id": 64,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(vm_rows_merged_total{job=\"$job\"}[5m])) by(type)",
+          "legendFormat": "{{type}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Merge speed",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -1842,8 +2219,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 68
+        "y": 84
       },
+      "hiddenSeries": false,
       "id": 58,
       "legend": {
         "avg": false,
@@ -1921,13 +2299,107 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 84
+      },
+      "hiddenSeries": false,
+      "id": 67,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(vm_log_messages_total{job=\"$job\"}[5m])) by (level) ",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{level}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Logging rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 76
+        "y": 92
       },
       "id": 46,
       "panels": [],
@@ -1947,8 +2419,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 77
+        "y": 93
       },
+      "hiddenSeries": false,
       "id": 44,
       "legend": {
         "avg": false,
@@ -2046,14 +2519,16 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 77
+        "y": 93
       },
+      "hiddenSeries": false,
       "id": 57,
       "legend": {
         "avg": false,
@@ -2141,8 +2616,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 85
+        "y": 101
       },
+      "hiddenSeries": false,
       "id": 47,
       "legend": {
         "avg": false,
@@ -2231,8 +2707,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 85
+        "y": 101
       },
+      "hiddenSeries": false,
       "id": 42,
       "legend": {
         "avg": false,
@@ -2320,8 +2797,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 93
+        "y": 109
       },
+      "hiddenSeries": false,
       "id": 48,
       "legend": {
         "avg": false,
@@ -2400,7 +2878,7 @@
     }
   ],
   "refresh": "30s",
-  "schemaVersion": 19,
+  "schemaVersion": 21,
   "style": "dark",
   "tags": [],
   "templating": {
@@ -2483,5 +2961,5 @@
   "timezone": "",
   "title": "VictoriaMetrics",
   "uid": "wNf0q_kZk",
-  "version": 3
-}
+  "version": 1
+}

deployment/docker/Makefile

@@ -1,13 +1,15 @@
-DOCKER_NAMESPACE := victoriametrics
-BUILDER_IMAGE := local/builder:go1.13.3
-CERTS_IMAGE := local/certs:1.0.2
+# All these commands must run from repository root.
+
+DOCKER_NAMESPACE := docker.io/victoriametrics
+BUILDER_IMAGE := local/builder:go1.13.5
+CERTS_IMAGE := local/certs:1.0.3
 
 package-certs:
-	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(CERTS_IMAGE)') \
+	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(CERTS_IMAGE)$$') \
 		|| docker build -t $(CERTS_IMAGE) deployment/docker/certs
 
 package-builder:
-	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)') \
+	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)$$') \
 		|| docker build -t $(BUILDER_IMAGE) deployment/docker/builder
 
 app-via-docker: package-certs package-builder
@@ -25,21 +27,106 @@ app-via-docker: package-certs package-builder
 			-o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME)
 
 package-via-docker:
-	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)') || (\
+	(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
 		$(MAKE) app-via-docker && \
-		docker build -t $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) -f app/$(APP_NAME)/deployment/Dockerfile .)
+		docker build \
+			--build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \
+			--build-arg certs_image=$(CERTS_IMAGE) \
+			-t $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \
+			-f app/$(APP_NAME)/deployment/Dockerfile bin)
 
-publish-via-docker: package-via-docker
-	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)
-	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest
-	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):latest
+package-manifest: \
+		package-via-docker-amd64 \
+		package-via-docker-arm \
+		package-via-docker-arm64 \
+		package-via-docker-ppc64le \
+		package-via-docker-386
+	$(MAKE) package-manifest-internal
+
+package-manifest-internal:
+	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE)
+	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE)
+	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE)
+	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE)
+	docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE)
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create --amend $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE)
+	GOARCH=amd64 $(MAKE) package-manifest-annotate-goarch
+	GOARCH=arm $(MAKE) package-manifest-annotate-goarch
+	GOARCH=arm64 $(MAKE) package-manifest-annotate-goarch
+	GOARCH=ppc64le $(MAKE) package-manifest-annotate-goarch
+	GOARCH=386 $(MAKE) package-manifest-annotate-goarch
+
+package-manifest-annotate-goarch:
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
+				$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-$(GOARCH)$(RACE) --os linux --arch $(GOARCH)
+
+publish-via-docker: package-manifest
+	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-amd64$(RACE)
+	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-arm$(RACE)
+	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-arm64$(RACE)
+	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-ppc64le$(RACE)
+	docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-386$(RACE)
+	PKG_TAG=latest $(MAKE) package-manifest-internal
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push --purge $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)
+	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push --purge $(DOCKER_NAMESPACE)/$(APP_NAME):latest$(RACE)
 
 run-via-docker: package-via-docker
 	docker run -it --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--net host \
 		$(DOCKER_OPTS) \
-		$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) $(ARGS)
+		$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS)
+
+app-via-docker-goarch:
+	APP_SUFFIX='-$(GOARCH)' \
+	DOCKER_OPTS='--env CGO_ENABLED=0 --env GOOS=linux --env GOARCH=$(GOARCH)' \
+	$(MAKE) app-via-docker
+
+app-via-docker-pure:
+	APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
+
+app-via-docker-amd64:
+	GOARCH=amd64 $(MAKE) app-via-docker-goarch
+
+app-via-docker-arm:
+	GOARCH=arm $(MAKE) app-via-docker-goarch
+
+app-via-docker-arm64:
+	GOARCH=arm64 $(MAKE) app-via-docker-goarch
+
+app-via-docker-ppc64le:
+	GOARCH=ppc64le $(MAKE) app-via-docker-goarch
+
+app-via-docker-386:
+	GOARCH=386 $(MAKE) app-via-docker-goarch
+
+package-via-docker-goarch:
+	APP_SUFFIX='-$(GOARCH)' \
+	DOCKER_OPTS='--env CGO_ENABLED=0 --env GOOS=linux --env GOARCH=$(GOARCH)' \
+	$(MAKE) package-via-docker
+
+package-via-docker-pure:
+	APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) package-via-docker
+
+package-via-docker-amd64:
+	GOARCH=amd64 $(MAKE) package-via-docker-goarch
+
+package-via-docker-arm:
+	GOARCH=arm $(MAKE) package-via-docker-goarch
+
+package-via-docker-arm64:
+	GOARCH=arm64 $(MAKE) package-via-docker-goarch
+
+package-via-docker-ppc64le:
+	GOARCH=ppc64le $(MAKE) package-via-docker-goarch
+
+package-via-docker-386:
+	GOARCH=386 $(MAKE) package-via-docker-goarch
 
 remove-docker-images:
 	docker image ls --format '{{.Repository}}\t{{.ID}}' | grep $(DOCKER_NAMESPACE)/ | grep -v /builder | awk '{print $$2}' | xargs docker image rm -f

deployment/docker/builder/Dockerfile

@@ -1,2 +1,2 @@
-FROM golang:1.13.3
+FROM golang:1.13.5
 STOPSIGNAL SIGINT

deployment/docker/certs/Dockerfile

@@ -1,3 +1,3 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-FROM alpine:3.9 as certs
+FROM alpine:3.10 as certs
 RUN apk --update add ca-certificates

deployment/docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.5'
 services:
   prometheus:
     container_name: prometheus
-    image: prom/prometheus:v2.12.0
+    image: prom/prometheus:v2.14.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -35,7 +35,7 @@ services:
     restart: always
   grafana:
     container_name: grafana
-    image: grafana/grafana:6.3.5
+    image: grafana/grafana:6.5.0
     entrypoint: >
       /bin/sh -c "
       cd /var/lib/grafana &&

deployment/docker/provisioning/datasources/datasource.yml

@@ -5,10 +5,10 @@ datasources:
       type: prometheus
       access: proxy
       url: http://prometheus:9090
-      isDefault: false
+      isDefault: true
 
     - name: VictoriaMetrics
       type: prometheus
       access: proxy
       url: http://victoriametrics:8428
-      isDefault: true
+      isDefault: false

docs/Articles.md

@@ -0,0 +1,22 @@
+# Articles
+
+* [Open-sourcing VictoriaMetrics](https://medium.com/@valyala/open-sourcing-victoriametrics-f31e34485c2b)
+* [How we created VictoriaMetrics](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac)
+* [VictoriaMetrics vs TimescaleDB vs InfluxDB benchmarks on 40K unique time series](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+* [VictoriaMetrics vs TimescaleDB vs InfluxDB benchmarks on 400K, 4M and 40M unique time series](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+* [Insert benchmarks for VictoriaMetrics vs InfluxDB on high-cardinality data](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
+* [Measuring vertical scalability for time series databases in Google Cloud](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+* [How VictoriaMetrics creates instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+* [Prometheus Subqueries in VictoriaMetrics](https://medium.com/@valyala/prometheus-subqueries-in-victoriametrics-9b1492b720b3)
+* [Why irate from Prometheus doesn't capture spikes](https://medium.com/@valyala/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
+* [Why mmap'ed files in Go may hurt performance](https://medium.com/@valyala/mmap-in-go-considered-harmful-d92a25cb161d)
+* [WAL Usage Looks Broken in Modern TSDBs](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704)
+* [Analyzing Prometheus data with external tools](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639)
+* [Stripping dependency bloat in VictoriaMetrics Docker image](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
+* [PromQL tutorial for beginners](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085)
+* [Achieving better compression for time series data than Gorilla](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
+* [Comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+* [Speeding up backups for big time series databases](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883)
+* [Evaluation performance and correctness: VictoriaMetrics response](https://medium.com/@valyala/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)
+* [Improving histogram usability for Prometheus and Grafana](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
+* [Prometheus storage: tech terms for humans](https://medium.com/@valyala/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)

docs/CaseStudies.md

@@ -0,0 +1,64 @@
+## Case studies
+
+Below are approved public case studies from VictoriaMetrics users. Join our [community Slack channel](http://slack.victoriametrics.com/)
+and feel free asking for references, reviews and additional case studies from real VictoriaMetrics users there.
+
+
+### Wix.com
+
+[Wix.com](https://en.wikipedia.org/wiki/Wix.com) is the leading web development platform.
+
+> We needed to redesign metric infrastructure from the ground up after the move to Kubernethes. A few approaches/designs have been tried before the one that works great has been chosen: Prometheus instance in every datacenter with 2 hours retention for local storage and remote write into [HA pair of single-node VictoriaMetrics instances](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#high-availability).
+
+Numbers:
+
+* The number of active time series per VictoriaMetrics instance is 20M.
+* The total number of time series per VictoriaMetrics instance is 400M+.
+* Ingestion rate per VictoriaMetrics instance is 800K data points per second.
+* The average time series churn rate is ~3M per day.
+* The average query rate is ~1K per minute (mostly alert queries).
+* Query duration: median is ~70ms, 99th percentile is ~2sec.
+* Retention: 6 months.
+
+> Alternatives that we’ve played with before choosing VictoriaMetrics are: federated Prometheus, Cortex, IronDB and Thanos.
+> Points that were critical to us when we were choosing a central tsdb, in order of importance:
+
+* At least 3 month worth of history.
+* Raw data, no aggregation, no sampling.
+* High query speed.
+* Clean fail state for HA (multi-node clusters may return partial data resulting in false alerts).
+* Enough head room/scaling capacity for future growth, up to 100M active time series.
+* Ability to split DB replicas per workload. Alert queries go to one replica, user queries go to another (speed for users, effective cache).
+
+> Optimizing for those points and our specific workload VictoriaMetrics proved to be the best option. As an icing on a cake we’ve got [PromQL extensions](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL) - `default 0` and `histogram` are my favorite ones, for example. What we specially like is having a lot of tsdb params easily available via config options, that makes tsdb easy to tune for specific use case. Also worth noting is a great community in [Slack channel](http://slack.victoriametrics.com/) and of course maintainer support.
+
+Alex Ulstein, Head of Monitoring, Wix.com
+
+
+### Wedos.com
+
+> [Wedos](https://www.wedos.com/) is the Biggest Czech Hosting. We have our own private data center, that holds only our servers and technologies. The second data center, where the servers will be cooled in an oil bath, is being built. We started using [cluster VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md) to store Prometheus metrics from all our infrastructure after receiving positive references from our friends who successfully use VictoriaMetrics.
+
+Numbers:
+
+* The number of acitve time series: 5M.
+* Ingestion rate: 170K data points per second.
+* Query duration: median is ~2ms, 99th percentile is ~50ms.
+
+> We like configuration simplicity and zero maintenance for VictoriaMetrics - once installed and forgot about it. It works out of the box without any issues.
+
+
+### Dreamteam
+
+[Dreamteam](https://dreamteam.gg/) successfully uses single-node VictoriaMetrics in multiple environments.
+
+Numbers:
+
+* Active time series: from 350K to 725K.
+* Total number of time series: from 100M to 320M.
+* Total number of datapoints: from 120 billions to 155 billions.
+* Retention: 3 months.
+
+VictoriaMetrics in production environment runs on 2 M5 EC2 instances in "HA" mode, managed by Terraform and Ansible TF module.
+2 Prometheus instances are writing to both VMs, with 2 [Promxy](https://github.com/jacksontj/promxy) replicas
+as load balancer for reads.

docs/Cluster-VictoriaMetrics.md

@@ -0,0 +1,330 @@
+# Cluster version
+
+VictoriaMetrics is fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
+
+It is recommended using [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics) instead of cluster version
+for ingestion rates lower than 10 million of data points per second.
+Single-node version [scales perfectly](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+with the number of CPU cores, RAM and available storage space.
+Single-node version is easier to configure and operate comparing to cluster version, so think twice before sticking to cluster version.
+
+Join [our Slack](http://slack.victoriametrics.com/) or [contact us](mailto:info@victoriametrics.com) with consulting and support questions.
+
+
+## Prominent features
+
+- Supports all the features of [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- Performance and capacity scales horizontally.
+- Supports multiple independent namespaces for time series data (aka multi-tenancy).
+
+
+## Architecture overview
+
+VictoriaMetrics cluster consists of the following services:
+
+- `vmstorage` - stores the data
+- `vminsert` - proxies the ingested data to `vmstorage` shards using consistent hashing
+- `vmselect` - performs incoming queries using the data from `vmstorage`
+
+Each service may scale independently and may run on the most suitable hardware.
+`vmstorage` nodes don't know about each other, don't communicate with each other and don't share any data.
+This is [shared nothing architecture](https://en.wikipedia.org/wiki/Shared-nothing_architecture).
+It increases cluster availability, simplifies cluster maintenance and cluster scaling.
+
+<img src="https://docs.google.com/drawings/d/e/2PACX-1vTvk2raU9kFgZ84oF-OKolrGwHaePhHRsZEcfQ1I_EC5AB_XPWwB392XshxPramLJ8E4bqptTnFn5LL/pub?w=1104&amp;h=746">
+
+
+## Binaries
+
+Compiled binaries for cluster version are available in the `assets` section of [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+See archives containing `cluster` word.
+
+Docker images for cluster version are available here:
+
+- `vminsert` - https://hub.docker.com/r/victoriametrics/vminsert/tags
+- `vmselect` - https://hub.docker.com/r/victoriametrics/vmselect/tags
+- `vmstorage` - https://hub.docker.com/r/victoriametrics/vmstorage/tags
+
+
+## Building from sources
+
+Source code for cluster version is available at [cluster branch](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+### Development Builds
+
+1. [Install go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make` from the repository root. It should build `vmstorage`, `vmselect`
+   and `vminsert` binaries and put them into the `bin` folder.
+
+
+### Production builds
+
+There is no need in installing Go on a host system since binaries are built
+inside [the official docker container for Go](https://hub.docker.com/_/golang).
+This makes reproducible builds.
+So [install docker](https://docs.docker.com/install/) and run the following command:
+
+```
+make vminsert-prod vmselect-prod vmstorage-prod
+```
+
+Production binaries are built into statically linked binaries for `GOARCH=amd64`, `GOOS=linux`.
+They are put into `bin` folder with `-prod` suffixes:
+```
+$ make vminsert-prod vmselect-prod vmstorage-prod
+$ ls -1 bin
+vminsert-prod
+vmselect-prod
+vmstorage-prod
+```
+
+### Building docker images
+
+Run `make package`. It will build the following docker images locally:
+
+* `victoriametrics/vminsert:<PKG_TAG>`
+* `victoriametrics/vmselect:<PKG_TAG>`
+* `victoriametrics/vmstorage:<PKG_TAG>`
+
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package`.
+
+
+
+## Operation
+
+### Cluster setup
+
+A minimal cluster must contain the following nodes:
+
+* a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags
+* a single `vminsert` node with `-storageNode=<vmstorage_host>:8400`
+* a single `vmselect` node with `-storageNode=<vmstorage_host>:8401`
+
+It is recommended to run at least two nodes for each service
+for high availability purposes.
+
+An http load balancer must be put in front of `vminsert` and `vmselect` nodes:
+- requests starting with `/insert` must be routed to port `8480` on `vminsert` nodes.
+- requests starting with `/select` must be routed to port `8481` on `vmselect` nodes.
+
+Ports may be altered by setting `-httpListenAddr` on the corresponding nodes.
+
+It is recommended setting up [monitoring](#monitoring) for the cluster.
+
+
+### Monitoring
+
+All the cluster components expose various metrics in Prometheus-compatible format at `/metrics` page on the TCP port set in `-httpListenAddr` command-line flag.
+By default the following TCP ports are used:
+- `vminsert` - 8480
+- `vmselect` - 8481
+- `vmstorage` - 8482
+
+It is recommended setting up Prometheus to scrape `/metrics` pages from all the cluster components, so they can be monitored and analyzed
+with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176).
+
+
+### URL format
+
+* URLs for data ingestion: `http://<vminsert>:8480/insert/<accountID>/<suffix>`, where:
+  - `<accountID>` is an arbitrary number identifying namespace for data ingestion (aka tenant)
+  - `<suffix>` may have the following values:
+     - `prometheus` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+     - `influx/write` or `influx/api/v2/write` - for inserting data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+     - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
+
+* URLs for querying: `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
+  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
+  - `<suffix>` may have the following values:
+    - `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries)
+    - `api/v1/query_range` - performs [PromQL range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)
+    - `api/v1/series` - performs [series query](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
+    - `api/v1/labels` - returns a [list of label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
+    - `api/v1/label/<label_name>/values` - returns values for the given `<label_name>` according [to API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
+    - `federate` - returns [federated metrics](https://prometheus.io/docs/prometheus/latest/federation/)
+    - `api/v1/export` - exports raw data. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details
+
+* URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
+  Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
+  be used on a regular basis, since it carries non-zero overhead.
+
+* `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
+  - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
+    which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
+    command-line flag value.
+  - `/snapshot/list` - list available snasphots.
+  - `/snapshot/delete?snapshot=<id>` - delete the given snapshot.
+  - `/snapshot/delete_all` - delete all the snapshots.
+
+  Snapshots may be created independently on each `vmstorage` node. There is no need in synchronizing snapshots' creation
+  across `vmstorage` nodes.
+
+
+### Cluster resizing and scalability.
+
+Cluster performance and capacity scales with adding new nodes.
+
+* `vminsert` and `vmselect` nodes are stateless and may be added / removed at any time.
+  Do not forget updating the list of these nodes on http load balancer.
+  Adding more `vminsert` nodes scales data ingestion rate. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/175#issuecomment-536925841)
+  about ingestion rate scalability.
+  Adding more `vmselect` nodes scales select queries rate.
+* `vmstorage` nodes own the ingested data, so they cannot be removed without data loss.
+  Adding more `vmstorage` nodes scales cluster capacity.
+
+Steps to add `vmstorage` node:
+
+1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster.
+2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8401`.
+3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8400`.
+
+
+### Cluster availability
+
+* HTTP load balancer must stop routing requests to unavailable `vminsert` and `vmselect` nodes.
+* The cluster remains available if at least a single `vmstorage` node exists:
+
+  - `vminsert` re-routes incoming data from unavailable `vmstorage` nodes to healthy `vmstorage` nodes
+  - `vmselect` continues serving partial responses if at least a single `vmstorage` node is available.
+
+
+### Updating / reconfiguring cluster nodes
+
+All the node types - `vminsert`, `vmselect` and `vmstorage` - may be updated via graceful shutdown.
+Send `SIGINT` signal to the corresponding process, wait until it finishes and then start new version
+with new configs.
+
+Cluster should remain in working state if at least a single node of each type remains available during
+the update process. See [cluster availability](#cluster-availability) section for details.
+
+
+### Capacity planning
+
+Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the most suitable hardware.
+
+#### vminsert
+
+* The recommended total number of vCPU cores for all the `vminsert` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended number of vCPU cores per each `vminsert` instance should equal to the number of `vmstorage` instances in the cluster.
+* The amount of RAM per each `vminsert` instance should be 1GB or more. RAM is used as a buffer for spikes in ingestion rate.
+* Sometimes `-rpc.disableCompression` command-line flag on `vminsert` instances could increase ingestion capacity at the cost
+  of higher network bandwidth usage between `vminsert` and `vmstorage`.
+
+#### vmstorage
+
+* The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
+* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
+  Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
+* The recommended total amount of storage space for all the `vmstorage` instances can be calculated
+  from the ingestion rate and retention: `storage_space = ingestion_rate * retention_seconds`.
+
+#### vmselect
+
+The recommended hardware for `vmselect` instances highly depends on the type of queries. Lightweight queries over small number of time series usually require
+small number of vCPU cores and small amount of RAM on `vmselect`, while heavy queries over big number of time series (>10K) usually require
+bigger number of vCPU cores and bigger amounts of RAM.
+
+
+### Helm
+
+Helm chart simplifies managing cluster version of VictoriaMetrics in Kubernetes.
+It is available in the [helm-charts](https://github.com/VictoriaMetrics/helm-charts) repository.
+
+Upgrade follows `Cluster resizing procedure` under the hood.
+
+
+### Replication and data safety
+
+VictoriaMetrics offloads replication to the underlying storage pointed by `-storageDataPath`.
+It is recommended storing data on [Google Compute Engine persistent disks](https://cloud.google.com/compute/docs/disks/#pdspecs),
+since they are protected from data loss and data corruption. They also provide consistently high performance
+and [may be resized](https://cloud.google.com/compute/docs/disks/add-persistent-disk) without downtime.
+HDD-based persistent disks should be enough for the majority of use cases.
+
+It is recommended using durable replicated persistent volumes in Kubernetes.
+
+Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883).
+
+
+### Backups
+
+It is recommended performing periodical backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+for protecting from user errors such as accidental data deletion.
+
+The following steps must be performed for each `vmstorage` node for creating a backup:
+
+1. Create an instant snapshot by navigating to `/snapshot/create` HTTP handler. It will create snapshot and return its name.
+2. Archive the created snapshot from `<-storageDataPath>/snapshots/<snapshot_name>` folder using [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmbackup/README.md).
+   The archival process doesn't interfere with `vmstorage` work, so it may be performed at any suitable time.
+3. Delete unused snapshots via `/snapshot/delete?snapshot=<snapshot_name>` or `/snapshot/delete_all` in order to free up occupied storage space.
+
+There is no need in synchronizing backups among all the `vmstorage` nodes.
+
+Restoring from backup:
+
+1. Stop `vmstorage` node with `kill -INT`.
+2. Restore data from backup using [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/app/vmrestore/README.md) into `-storageDataPath` directory.
+3. Start `vmstorage` node.
+
+
+## Community and contributions
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
+
+Due to `KISS` cluster version of VictoriaMetrics has no the following "features" popular in distributed computing world:
+
+- Fragile gossip protocols. See [failed attempt in Thanos](https://github.com/improbable-eng/thanos/blob/030bc345c12c446962225221795f4973848caab5/docs/proposals/completed/201809_gossip-removal.md).
+- Hard-to-understand-and-implement-properly [Paxos protocols](https://www.quora.com/In-distributed-systems-what-is-a-simple-explanation-of-the-Paxos-algorithm).
+- Complex replication schemes, which may go nuts in unforesseen edge cases. The replication is offloaded to the underlying durable replicated storage
+  such as [persistent disks in Google Compute Engine](https://cloud.google.com/compute/docs/disks/#pdspecs).
+- Automatic data reshuffling between storage nodes, which may hurt cluster performance and availability.
+- Automatic cluster resizing, which may cost you a lot of money if improperly configured.
+- Automatic discovering and addition of new nodes in the cluster, which may mix data between dev and prod clusters :)
+- Automatic leader election, which may result in split brain disaster on network errors.
+
+
+## Reporting bugs
+
+Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+
+
+## Victoria Metrics Logo
+
+[Zip](VM_logo.zip) contains three folders with different image orientation (main color and inverted version).
+
+Files included in each folder:
+
+* 2 JPEG Preview files
+* 2 PNG Preview files with transparent background
+* 2 EPS Adobe Illustrator EPS10 files
+
+
+### Logo Usage Guidelines
+
+#### Font used:
+
+* Lato Black
+* Lato Regular
+
+#### Color Palette:
+
+* HEX [#110f0f](https://www.color-hex.com/color/110f0f)
+* HEX [#ffffff](https://www.color-hex.com/color/ffffff)
+
+### We kindly ask:
+
+- Please don't use any other font instead of suggested.
+- There should be sufficient clear space around the logo.
+- Do not change spacing, alignment, or relative locations of the design elements.
+- Do not change the proportions of any of the design elements or the design itself. You    may resize as needed but must retain all proportions.

docs/ExtendedPromQL.md

@@ -0,0 +1,90 @@
+# MetricsQL
+
+VictoriaMetrics implements MetricsQL - query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
+It is backwards compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work the same after switching from Prometheus to VictoriaMetrics.
+
+The following functionality is implemented differently in MetricsQL comparing to PromQL in order to improve user experience:
+* MetricsQL takes into account the previous point before the window in square brackets for range functions such as `rate` and `increase`.
+  It also doesn't extrapolate range function results. This addresses [this issue from Prometheus](https://github.com/prometheus/prometheus/issues/3746).
+* MetricsQL returns the expected non-empty responses for requests with `step` values smaller than scrape interval. This addresses [this issue from Grafana](https://github.com/grafana/grafana/issues/11451).
+* MetricsQL treats `scalar` type the same as `instant vector` without labels, since subtle difference between these types usually confuses users.
+  See [the corresponding Prometheus docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#expression-language-data-types) for details.
+
+Other PromQL functionality should work the same in MetricsQL. [File an issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues)
+if you notice discrepancies between PromQL and MetricsQL results other than mentioned above.
+
+MetricsQL provides additional functionality mentioned below, which is aimed towards solving practical cases.
+Feel free [filing a feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you think MetricsQL misses certain useful functionality.
+
+*Note that the functionality mentioned below doesn't work in PromQL, so it is impossible switching back to Prometheus after you start using it.*
+
+This functionality can be tried at [an editable Grafana dashboard](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo).
+
+- [`WITH` templates](https://play.victoriametrics.com/promql/expand-with-exprs). This feature simplifies writing and managing complex queries. Go to [`WITH` templates playground](https://victoriametrics.com/promql/expand-with-exprs) and try it.
+- Metric names and metric labels may contain escaped chars. For instance, `foo\-bar{baz\=aa="b"}` is valid expression. It returns time series with name `foo-bar` containing label `baz=aa` with value `b`. Additionally, `\xXX` escape sequence is supported, where `XX` is hexadecimal representation of escaped char.
+- `offset`, range duration and step value for range vector may refer to the current step aka `$__interval` value from Grafana.
+  For instance, `rate(metric[10i] offset 5i)` would return per-second rate over a range covering 10 previous steps with the offset of 5 steps.
+- `offset` may be put anywere in the query. For instance, `sum(foo) offset 24h`.
+- `offset` may be negative. For example, `q offset -1h`.
+- `default` binary operator. `q1 default q2` substitutes `NaN` values from `q1` with the corresponding values from `q2`.
+- `histogram_quantile` accepts optional third arg - `boundsLabel`. In this case it returns `lower` and `upper` bounds for the estimated percentile. See [this issue for details](https://github.com/prometheus/prometheus/issues/5706).
+- `if` binary operator. `q1 if q2` removes values from `q1` for `NaN` values from `q2`.
+- `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for non-`NaN` values from `q2`.
+- Trailing commas on all the lists are allowed - label filters, function args and with expressions. For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`. This simplifies maintenance of multi-line queries.
+- String literals may be concatenated. This is useful with `WITH` templates: `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`.
+- Range duration in functions such as [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate()) may be omitted. VictoriaMetrics automatically selects range duration depending on the current step used for building the graph. For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
+- [Range duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors) and [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be fractional. For instance, `rate(node_network_receive_bytes_total[1.5m] offset 0.5d)`.
+- Comments starting with `#` and ending with newline. For instance, `up # this is a comment for 'up' metric`.
+- Rollup functions - `rollup(m[d])`, `rollup_rate(m[d])`, `rollup_deriv(m[d])`, `rollup_increase(m[d])`, `rollup_delta(m[d])` - return `min`, `max` and `avg`
+  values for all the `m` data points over `d` duration.
+- `rollup_candlestick(m[d])` - returns `open`, `close`, `low` and `high` values (OHLC) for all the `m` data points over `d` duration. This function is useful for financial applications.
+- `union(q1, ... qN)` function for building multiple graphs for `q1`, ... `qN` subqueries with a single query. The `union` function name may be skipped -
+  the following queries are equivalent: `union(q1, q2)` and `(q1, q2)`.
+- `ru(freeResources, maxResources)` function for returning resource utilization percentage in the range `0% - 100%`. For instance, `ru(node_memory_MemFree_bytes, node_memory_MemTotal_bytes)` returns memory utilization over [node_exporter](https://github.com/prometheus/node_exporter) metrics.
+- `ttf(slowlyChangingFreeResources)` function for returning the time in seconds when the given `slowlyChangingFreeResources` expression reaches zero. For instance, `ttf(node_filesystem_avail_byte)` returns the time to storage space exhaustion. This function may be useful for capacity planning.
+- Functions for label manipulation:
+  - `alias(q, name)` for setting metric name across all the time series `q`.
+  - `label_set(q, label1, value1, ... labelN, valueN)` for setting the given values for the given labels on `q`.
+  - `label_del(q, label1, ... labelN)` for deleting the given labels from `q`.
+  - `label_keep(q, label1, ... labelN)` for deleting all the labels except the given labels from `q`.
+  - `label_copy(q, src_label1, dst_label1, ... src_labelN, dst_labelN)` for copying label values from `src_*` to `dst_*`.
+  - `label_move(q, src_label1, dst_label1, ... src_labelN, dst_labelN)` for moving label values from `src_*` to `dst_*`.
+  - `label_transform(q, label, regexp, replacement)` for replacing all the `regexp` occurences with `replacement` in the `label` values from `q`.
+  - `label_value(q, label)` - returns numeric values for the given `label` from `q`.
+- `step()` function for returning the step in seconds used in the query.
+- `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
+- `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
+- `ideriv(m)` - for calculating `instant` derivative for `m`.
+- `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`.
+- `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range.
+- `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range.
+- `smooth_exponential(q, sf)` - smooths `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) with the given smooth factor `sf`.
+- `remove_resets(q)` - removes counter resets from `q`.
+- `lag(q[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `q` over `d`.
+- `lifetime(q[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `q`.
+- `scrape_interval(q[d])` - returns the average interval in seconds between data points of `q` over `d` aka `scrape interval`.
+- Trigonometric functions - `sin(q)`, `cos(q)`, `asin(q)`, `acos(q)` and `pi()`.
+- `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`.
+- `median(q)` - median aggregate. Shorthand to `quantile(0.5, q)`.
+- `limitk(k, q)` - limits the number of time series returned from `q` to `k`.
+- `keep_last_value(q)` - fills missing data (gaps) in `q` with the previous value.
+- `distinct_over_time(m[d])` - returns distinct number of values for `m` data points over `d` duration.
+- `distinct(q)` - returns a time series with the number of unique values for each timestamp in `q`.
+- `sum2_over_time(m[d])` - returns sum of squares for all the `m` values over `d` duration.
+- `sum2(q)` - returns a time series with sum of square values for each timestamp in `q`.
+- `geomean_over_time(m[d])` - returns [geomean](https://en.wikipedia.org/wiki/Geometric_mean) value for all the `m` value over `d` duration.
+- `geomean(q)` - returns a time series with [geomean](https://en.wikipedia.org/wiki/Geometric_mean) value for each timestamp in `q`.
+- `rand()`, `rand_normal()` and `rand_exponential()` functions - for generating pseudo-random series with even, normal and exponential distribution.
+- `increases_over_time(m[d])` and `decreases_over_time(m[d])` - returns the number of `m` increases or decreases over the given duration `d`.
+- `prometheus_buckets(q)` - converts [VictoriaMetrics histogram](https://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) buckets to Prometheus buckets with `le` labels.
+- `histogram(q)` - calculates aggregate histogram over `q` time series for each point on the graph. See [this article](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) for more details.
+- `topk_*` and `bottomk_*` aggregate functions, which return up to K time series. Note that the standard `topk` function may return more than K time series -
+   see [this article](https://www.robustperception.io/graph-top-n-time-series-in-grafana) for details.
+   - `topk_min(k, q)` - returns top K time series with the max minimums on the given time range
+   - `topk_max(k, q)` - returns top K time series with the max maximums on the given time range
+   - `topk_avg(k, q)` - returns top K time series with the max averages on the given time range
+   - `topk_median(k, q)` - returns top K time series with the max medians on the given time range
+   - `bottomk_min(k, q)` - returns bottom K time series with the min minimums on the given time range
+   - `bottomk_max(k, q)` - returns bottom K time series with the min maximums on the given time range
+   - `bottomk_avg(k, q)` - returns bottom K time series with the min averages on the given time range
+   - `bottomk_median(k, q)` - returns bottom K time series with the min medians on the given time range

docs/FAQ.md

@@ -0,0 +1,160 @@
+# FAQ
+
+### What is the main purpose of VictoriaMetrics?
+
+To provide the best long-term [remote storage](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) solution for [Prometheus](https://prometheus.io/).
+
+
+### Which features does VictoriaMetrics have?
+
+* Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
+  Additionally, VictoriaMetrics extends PromQL with opt-in [useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
+* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+  and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
+  [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+* [Uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) when working with millions of unique time series (aka high cardinality).
+* High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+  may be crammed into a limited storage comparing to TimescaleDB.
+* Optimized for storage with high-latency IO and low iops (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
+  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+  and [comparing Thanos to VictoriaMetrics](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+* Easy operation:
+  * VictoriaMetrics consists of a single executable without external dependencies.
+  * All the configuration is done via explicit command-line flags with reasonable defaults.
+  * All the data is stored in a single directory pointed by `-storageDataPath` flag.
+  * Easy backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+* Storage is protected from corruption on unclean shutdown (i.e. hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+* Supports metrics' ingestion and backfilling via the following protocols:
+  * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+  * [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+  * [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
+    if `-graphiteListenAddr` is set.
+  * [OpenTSDB put message](http://opentsdb.net/docs/build/html/api_telnet/put.html) if `-opentsdbListenAddr` is set.
+* Ideally works with big amounts of time series data from IoT sensors, connected car sensors and industrial sensors.
+* Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+### Which clients do you target?
+
+The following Prometheus users may be interested in VictoriaMetrics:
+- Users who don't want to bother with Prometheus' local storage operational burden - backups, replication, capacity planning, scalability, etc.
+- Users with multiple Prometheus instances who want performing arbitrary queries over all the metrics collected by their Prometheus instances (aka `global querying view`).
+- Users who want reducing costs for storing huge amounts of time series data.
+
+
+### How to start using VictoriaMetrics?
+
+Start with [single-node version](Single-server-VictoriaMetrics). It is easy to configure and operate. It should fit the majority of use cases.
+
+
+### Is it safe to enable [remote write storage](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) in Prometheus?
+
+Yes. Prometheus continues writing data to local storage after enabling remote storage write, so all the existing local storage data
+and new data is available for querying via Prometheus as usual.
+
+
+### How does VictoriaMetrics compare to other clustered TSDBs on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/improbable-eng/thanos), [Cortex](https://github.com/cortexproject/cortex), etc.?
+
+VictoriaMetrics is simpler, faster, more cost-effective and it provides [useful extensions for PromQL](ExtendedPromQL). The simplicity is twofold:
+- It is simpler to configure and operate. There is no need in configuring third-party [sidecars](https://github.com/improbable-eng/thanos/blob/master/docs/components/sidecar.md)
+  or fighting with [gossip protocol](https://github.com/improbable-eng/thanos/blob/master/docs/proposals/completed/201809_gossip-removal.md).
+- VictoriaMetrics has simpler architecture, which means less bugs and more useful features in a long run comparing to competing TSDBs.
+
+See [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+
+
+### How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)?
+
+VictoriaMetrics requires [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and it [works faster](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+It is easier to configure and operate. It provides [better query language](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085) than InfluxQL or Flux.
+
+
+### How does VictoriaMetrics compare to [TimescaleDB](https://www.timescale.com/)?
+
+TimescaleDB insists on using SQL as a query language. While SQL is more powerful than PromQL, this power is rarely required during typical TSDB usage. Real-world queries usually [look clearer and simpler when written in PromQL than in SQL](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085).
+Additionally, VictoriaMetrics requires [up to 70x less storage space comparing to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data.
+
+
+### Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/improbable-eng/thanos), [Cortex](https://github.com/cortexproject/cortex)?
+
+No. VictoriaMetrics core is written in Go from scratch by [fasthttp](https://github.com/valyala/fasthttp) [author](https://github.com/valyala).
+The architecture is [optimized for storing and querying large amounts of time series data with high cardinality](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac). VictoriaMetrics storage uses [certain ideas from ClickHouse](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282). Special thanks to [Alexey Milovidov](https://github.com/alexey-milovidov).
+
+
+### Are there performance comparisons with other solutions?
+
+Yes:
+
+* [Measuring vertical scalability for time series databases: VictoriaMetrics vs InfluxDB vs TimescaleDB](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+* [Measuring insert performance on high-cardinality time series: VictoriaMetrics vs InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
+* [TSBS benchmark on high-cardinality time series: VictoriaMetrics vs InfluxDB vs TimescaleDB](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+* [Standard TSBS benchmark: VictoriaMetrics vs InfluxDB vs TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+
+
+### What is the pricing for VictoriaMetrics?
+
+The following versions are open source and free:
+* [Single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics).
+* [Cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+We provide commercial support for both versions. [Contact us](mailto:info@victoriametrics.com) for the pricing.
+
+The following versions are commercial:
+* Managed cluster in the Cloud.
+* SaaS version.
+
+[Contact us](mailto:info@victoriametrics.com) for the pricing.
+
+
+### Why VictoriaMetrics doesn't support [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)?
+
+Remote read API requires transferring all the raw data for all the requested metrics over the given time range. For instance,
+if a query covers 1000 metrics with 10K values each, then the remote read API had to return `1000*10K`=10M metric values to Prometheus.
+This is slow and expensive.
+Prometheus remote read API isn't intended for querying foreign data aka `global query view`. See [this issue](https://github.com/prometheus/prometheus/issues/4456) for details.
+
+So just query VictoriaMetrics directly via [Prometheus Querying API](https://prometheus.io/docs/prometheus/latest/querying/api/)
+or via [Prometheus datasoruce in Grafana](http://docs.grafana.org/features/datasources/prometheus/).
+
+
+### Does VictoriaMetrics deduplicate data from Prometheus instances scraping the same targets (aka `HA pairs`)?
+
+Data from all the Prometheus instances is saved in VictoriaMetrics without deduplication.
+
+The deduplication for Prometheus HA pair may be easily implemented on top of VictoriaMetrics with the following steps:
+
+1) Run multiple VictoriaMetrics instances in multiple availability zones (datacenters).
+2) Configure each Prometheus from each HA pair to write data to VictoriaMetrics in distinct availability zone.
+3) Put [Promxy](https://github.com/jacksontj/promxy) in front of all the VictoriaMetrics instances.
+4) Send queries to Promxy - it will deduplicate data from VictoriaMetrics instances behind it.
+
+
+### Where is the source code of VictoriaMetrics?
+
+Source code for the following versions is available in the following places:
+* [Single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics).
+* [Cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+### Does VictoriaMetrics fit for data from IoT sensors and industrial sensors?
+
+VictoriaMetrics is able to handle data from hundreds of millions of IoT sensors and industrial sensors.
+It supports [high cardinality data](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b),
+perfectly [scales up on a single node](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+and scales horizontally to multiple nodes.
+
+
+### Where can I ask questions about VictoriaMetrics?
+
+See [VictoriaMetrics-users group](https://groups.google.com/forum/#!forum/victorametrics-users).
+
+
+### Where can I file bugs and feature requests regarding VictoriaMetrics?
+
+File bugs and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+
+
+### Are you looking for investors?
+
+Yes. [Mail us](mailto:info@victoriametrics.com) if you are interested in.

docs/Home.md

@@ -0,0 +1,13 @@
+# Docs
+
+* [Quick start](Quick-Start)
+* [`WITH` templates playground](https://play.victoriametrics.com/promql/expand-with-exprs)
+* [Grafana playground](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo)
+* [MetricsQL](ExtendedPromQL)
+* [Single-node version](Single-server-VictoriaMetrics)
+* [FAQ](FAQ)
+* [Cluster version](Cluster-VictoriaMetrics)
+* [Articles](Articles)
+* [Case Studies](CaseStudies)
+* [vmbackup](vmbackup)
+* [vmrestore](vmrestore)

docs/Quick-Start.md

@@ -0,0 +1,27 @@
+# Quick Start
+
+1. Download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
+   from [Docker hub](https://hub.docker.com/r/valyala/victoria-metrics/)
+   or [build it from sources](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics#how-to-build-from-sources).
+
+2. Run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
+   and their default values. Default flag values should fit the majoirty of cases. The minimum required flags to configure are:
+
+   * `-storageDataPath` - path to directory where VictoriaMetrics stores all the data.
+   * `-retentionPeriod` - data retention in months.
+
+   For instance:
+
+   `./victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics-data -retentionPeriod=3`
+
+   See [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) in order to configure VictoriaMetrics as OS service.
+   It is recommended setting up [VictoriaMetrics monitoring](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#monitoring).
+
+3. Configure all the Prometheus instances to write data to VictoriaMetrics.
+   See [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics#prometheus-setup).
+
+4. Configure Grafana to query VictoriaMetrics instead of Prometheus.
+   See [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics#grafana-setup).
+
+
+There is also [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) and [SaaS playground](https://play.victoriametrics.com/signIn).

docs/Release-Guide.md

@@ -0,0 +1,44 @@
+Release process guidance
+
+## Release version and Docker images
+
+1. Create release tag with `git tag v1.xx.y`.
+2. Run `make release` for creating `*.tar.gz` release archive with the corresponding `_checksums.txt` inside `bin` directory.
+3. Run `make publish` for creating and publishing Docker images.
+4. Push release tag to https://github.com/VictoriaMetrics/VictoriaMetrics : `git push origin v1.xx.y`.
+5. Go to https://github.com/VictoriaMetrics/VictoriaMetrics/releases , create new release from the pushed tag on step 4
+   and upload `*.tar.gz` archive with the corresponding `_checksums.txt` from step 2.
+
+
+## Helm Charts
+
+The helm chart repository [https://github.com/VictoriaMetrics/helm-charts/](https://github.com/VictoriaMetrics/helm-charts/)
+
+
+### Bump the version of images. 
+In that case, don't need to bump the helm chart version
+
+1. Need to update [`values.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/values.yaml), bump version for `vmselect`, `vminsert` and `vmstorage`
+2. Specify the correct version in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/Chart.yaml)
+3. Update version [README.md](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/README.md), specify the new version in the documentation
+4. Push changes to master. `master` is a source of truth
+5. Rebase `master` into `gh-pages` branch
+6. Run `make package` which creates or updates zip file with the packed chart
+7. Run `make merge`. It creates or updates metadata for charts in index.yaml 
+8. Push the changes to `gh-pages` branch 
+
+### Updating the chart.
+1. Update chart version in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/Chart.yaml)
+2. Update [README.md](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/README.md) file, reflect changes in the documentation.
+3. Repeat the procedure from step _4_ previous section.
+
+
+## Wiki pages
+
+All changes from `docs` folder and `.md` extension automatically push to Wiki
+
+**_Note_**: no vice versa, direct changes on Wiki will be overitten after any changes in `docs/*.md` 
+
+## Github pages
+
+All changes in `README.md`, `docs` folder and `.md` extension automatically push to Wiki

docs/Single-server-VictoriaMetrics.md

@@ -0,0 +1,880 @@
+## VictoriaMetrics
+
+VictoriaMetrics is fast, cost-effective and scalable time-series database. It can be used as long-term remote storage for Prometheus.
+It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
+[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and
+in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
+
+Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+## Case studies
+
+* [Wix.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wixcom)
+* [Wedos.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wedoscom)
+* [Dreamteam](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#dreamteam)
+
+
+## Prominent features
+
+* Supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
+  VictoriaMetrics implements [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL) query language, which is inspired by PromQL.
+* Supports global query view. Multiple Prometheus instances may write data into VictoriaMetrics. Later this data may be used in a single query.
+* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+  and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
+  [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+* [Uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) when working with millions of unique time series (aka high cardinality).
+* Optimized for time series with high churn rate. Think about [prometheus-operator](https://github.com/coreos/prometheus-operator) metrics from frequent deployments in Kubernetes.
+* High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+  may be crammed into limited storage comparing to TimescaleDB.
+* Optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
+  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
+  and [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+* Easy operation:
+  * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
+  * All the configuration is done via explicit command-line flags with reasonable defaults.
+  * All the data is stored in a single directory pointed by `-storageDataPath` flag.
+  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+  to S3 or GCS with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) / [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+  See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+* Storage is protected from corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+* Supports metrics' ingestion and [backfilling](#backfilling) via the following protocols:
+  * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+  * [InfluxDB line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+  * [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
+    if `-graphiteListenAddr` is set.
+  * [OpenTSDB put message](#sending-data-via-telnet-put-protocol) if `-opentsdbListenAddr` is set.
+  * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests) if `-opentsdbHTTPListenAddr` is set.
+  * [/api/v1/import](#how-to-import-time-series-data)
+* Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various Enterprise workloads.
+* Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+
+## Operation
+
+
+### Table of contents
+
+  - [How to start VictoriaMetrics](#how-to-start-victoriametrics)
+  - [Prometheus setup](#prometheus-setup)
+  - [Grafana setup](#grafana-setup)
+  - [How to upgrade VictoriaMetrics?](#how-to-upgrade-victoriametrics)
+  - [How to apply new config to VictoriaMetrics?](#how-to-apply-new-config-to-victoriametrics)
+  - [How to send data from InfluxDB-compatible agents such as Telegraf?](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+  - [How to send data from Graphite-compatible agents such as StatsD?](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
+  - [Querying Graphite data](#querying-graphite-data)
+  - [How to send data from OpenTSDB-compatible agents?](#how-to-send-data-from-opentsdb-compatible-agents)
+  - [Prometheus querying API usage](#prometheus-querying-api-usage)
+  - [How to build from sources](#how-to-build-from-sources)
+    - [Development build](#development-build)
+    - [Production build](#production-build)
+    - [ARM build](#arm-build)
+    - [Pure Go build (CGO_ENABLED=0)](#pure-go-build-cgo_enabled0)
+    - [Building docker images](#building-docker-images)
+  - [Start with docker-compose](#start-with-docker-compose)
+  - [Setting up service](#setting-up-service)
+  - [Third-party contributions](#third-party-contributions)
+  - [How to work with snapshots?](#how-to-work-with-snapshots)
+  - [How to delete time series?](#how-to-delete-time-series)
+  - [How to export time series?](#how-to-export-time-series)
+  - [How to import time series data?](#how-to-import-time-series-data)
+  - [Federation](#federation)
+  - [Capacity planning](#capacity-planning)
+  - [High availability](#high-availability)
+  - [Multiple retentions](#multiple-retentions)
+  - [Downsampling](#downsampling)
+  - [Multi-tenancy](#multi-tenancy)
+  - [Scalability and cluster version](#scalability-and-cluster-version)
+  - [Alerting](#alerting)
+  - [Security](#security)
+  - [Tuning](#tuning)
+  - [Monitoring](#monitoring)
+  - [Troubleshooting](#troubleshooting)
+  - [Backfilling](#backfilling)
+  - [Profiling](#profiling)
+- [Integrations](#integrations)
+- [Roadmap](#roadmap)
+- [Contacts](#contacts)
+- [Community and contributions](#community-and-contributions)
+- [Reporting bugs](#reporting-bugs)
+- [Victoria Metrics Logo](#victoria-metrics-logo)
+  - [Logo Usage Guidelines](#logo-usage-guidelines)
+    - [Font used:](#font-used)
+    - [Color Palette:](#color-palette)
+  - [We kindly ask:](#we-kindly-ask)
+
+
+### How to start VictoriaMetrics
+
+Just start VictoriaMetrics [executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+or [docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) with the desired command-line flags.
+
+The following command-line flags are used the most:
+
+* `-storageDataPath` - path to data directory. VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in current working directory.
+* `-retentionPeriod` - retention period in months for the data. Older data is automatically deleted. Default period is 1 month.
+* `-httpListenAddr` - TCP address to listen to for http requests. By default, it listens port `8428` on all the network interfaces.
+* `-graphiteListenAddr` - TCP and UDP address to listen to for Graphite data. By default, it is disabled.
+* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data over telnet protocol. By default, it is disabled.
+* `-opentsdbHTTPListenAddr` - TCP address to listen to for HTTP OpenTSDB data over `/api/put`. By default, it is disabled.
+
+Pass `-help` to see all the available flags with description and default values.
+
+It is recommended setting up [monitoring](#monitoring) for VictoriaMetrics.
+
+
+### Prometheus setup
+
+Add the following lines to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`):
+
+```yml
+remote_write:
+  - url: http://<victoriametrics-addr>:8428/api/v1/write
+    queue_config:
+      max_samples_per_send: 10000
+      max_shards: 30
+```
+
+Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
+Then apply the new config via the following command:
+
+```
+kill -HUP `pidof prometheus`
+```
+
+Prometheus writes incoming data to local storage and replicates it to remote storage in parallel.
+This means the data remains available in local storage for `--storage.tsdb.retention.time` duration
+even if remote storage is unavailable.
+
+If you plan to send data to VictoriaMetrics from multiple Prometheus instances, then add the following lines into `global` section
+of [Prometheus config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file):
+
+```yml
+global:
+  external_labels:
+    datacenter: dc-123
+```
+
+This instructs Prometheus to add `datacenter=dc-123` label to each time series sent to remote storage.
+The label name may be arbitrary - `datacenter` is just an example. The label value must be unique
+across Prometheus instances, so those time series may be filtered and grouped by this label.
+
+
+It is recommended upgrading Prometheus to [v2.12.0](https://github.com/prometheus/prometheus/releases) or newer,
+since the previous versions may have issues with `remote_write`.
+
+
+### Grafana setup
+
+Create [Prometheus datasource](http://docs.grafana.org/features/datasources/prometheus/) in Grafana with the following Url:
+
+```
+http://<victoriametrics-addr>:8428
+```
+
+Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
+
+Then build graphs with the created datasource using [Prometheus query language](https://prometheus.io/docs/prometheus/latest/querying/basics/).
+VictoriaMetrics supports native PromQL and [extends it with useful features](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL).
+
+
+### How to upgrade VictoriaMetrics?
+
+It is safe upgrading VictoriaMetrics to new versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+say otherwise. It is recommended performing regular upgrades to the latest version,
+since it may contain important bug fixes, performance optimizations or new features.
+
+Follow the following steps during the upgrade:
+
+1) Send `SIGINT` signal to VictoriaMetrics process in order to gracefully stop it.
+2) Wait until the process stops. This can take a few seconds.
+3) Start the upgraded VictoriaMetrics.
+
+Prometheus doesn't drop data during VictoriaMetrics restart.
+See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+
+
+### How to apply new config to VictoriaMetrics?
+
+VictoriaMetrics must be restarted for applying new config:
+
+1) Send `SIGINT` signal to VictoriaMetrics process in order to gracefully stop it.
+2) Wait until the process stops. This can take a few seconds.
+3) Start VictoriaMetrics with the new config.
+
+Prometheus doesn't drop data during VictoriaMetrics restart.
+See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+
+
+### How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)?
+
+Just use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
+For instance, put the following lines into `Telegraf` config, so it sends data to VictoriaMetrics instead of InfluxDB:
+
+```
+[[outputs.influxdb]]
+  urls = ["http://<victoriametrics-addr>:8428"]
+```
+
+Do not forget substituting `<victoriametrics-addr>` with the real address where VictoriaMetrics runs.
+
+VictoriaMetrics maps Influx data using the following rules:
+* [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
+  unless `db` tag exists in the Influx line.
+* Field names are mapped to time series names prefixed with `{measurement}{separator}` value,
+  where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag.
+  See also `-influxSkipSingleField` command-line flag. If `{measurement}` is empty, then time series names correspond to field names.
+* Field values are mapped to time series values.
+* Tags are mapped to Prometheus labels as-is.
+
+For example, the following Influx line:
+
+```
+foo,tag1=value1,tag2=value2 field1=12,field2=40
+```
+
+is converted into the following Prometheus data points:
+
+```
+foo_field1{tag1="value1", tag2="value2"} 12
+foo_field2{tag1="value1", tag2="value2"} 40
+```
+
+Example for writing data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+to local VictoriaMetrics using `curl`:
+
+```
+curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'http://localhost:8428/write'
+```
+
+An arbitrary number of lines delimited by '\n' may be sent in a single request.
+After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+
+```
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"measurement_.*"}'
+```
+
+The `/api/v1/export` endpoint should return the following response:
+
+```
+{"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560272508147]}
+{"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1560272508147]}
+```
+
+Note that Influx line protocol expects [timestamps in *nanoseconds* by default](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/#timestamp),
+while VictoriaMetrics stores them with *milliseconds* precision.
+
+
+### How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)?
+
+1) Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
+the following command will enable Graphite receiver in VictoriaMetrics on TCP and UDP port `2003`:
+
+```
+/path/to/victoria-metrics-prod -graphiteListenAddr=:2003
+```
+
+2) Use the configured address in Graphite-compatible agents. For instance, set `graphiteHost`
+to the VictoriaMetrics host in `StatsD` configs.
+
+
+Example for writing data with Graphite plaintext protocol to local VictoriaMetrics using `nc`:
+
+```
+echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N localhost 2003
+```
+
+VictoriaMetrics sets the current time if the timestamp is omitted.
+An arbitrary number of lines delimited by `\n` may be sent in one go.
+After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+
+```
+curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
+```
+
+The `/api/v1/export` endpoint should return the following response:
+
+```
+{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277406000]}
+```
+
+
+### Querying Graphite data
+
+Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read either via
+[Prometheus querying API](#prometheus-querying-api-usage)
+or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+
+
+
+### How to send data from OpenTSDB-compatible agents?
+
+VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
+and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data.
+
+#### Sending data via `telnet put` protocol
+
+1) Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
+the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
+
+```
+/path/to/victoria-metrics-prod -opentsdbListenAddr=:4242
+```
+
+2) Send data to the given address from OpenTSDB-compatible agents.
+
+
+Example for writing data with OpenTSDB protocol to local VictoriaMetrics using `nc`:
+
+```
+echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N localhost 4242
+```
+
+An arbitrary number of lines delimited by `\n` may be sent in one go.
+After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+
+```
+curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
+```
+
+The `/api/v1/export` endpoint should return the following response:
+
+```
+{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277292000]}
+```
+
+
+#### Sending OpenTSDB data via HTTP `/api/put` requests
+
+1) Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance,
+the following command enables OpenTSDB HTTP server on port `4242`:
+
+```
+/path/to/victoria-metrics-prod -opentsdbHTTPListenAddr=:4242
+```
+
+2) Send data to the given address from OpenTSDB-compatible agents.
+
+Example for writing a single data point:
+
+```
+curl -H 'Content-Type: application/json' -d '{"metric":"x.y.z","value":45.34,"tags":{"t1":"v1","t2":"v2"}}' http://localhost:4242/api/put
+```
+
+Example for writing multiple data points in a single request:
+
+```
+curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
+```
+
+After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+
+```
+curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar'
+```
+
+The `/api/v1/export` endpoint should return the following response:
+
+```
+{"metric":{"__name__":"foo"},"values":[45.34],"timestamps":[1566464846000]}
+{"metric":{"__name__":"bar"},"values":[43],"timestamps":[1566464846000]}
+{"metric":{"__name__":"x.y.z","t1":"v1","t2":"v2"},"values":[45.34],"timestamps":[1566464763000]}
+```
+
+
+### Prometheus querying API usage
+
+VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
+
+* [/api/v1/query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries)
+* [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)
+* [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
+* [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
+* [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
+
+These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
+
+VictoriaMetrics accepts additional args for `/api/v1/labels` and `/api/v1/label/.../values` handlers.
+See [this feature request](https://github.com/prometheus/prometheus/issues/6178) for details:
+
+* Any number [time series selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) via `match[]` query arg.
+* Optional `start` and `end` query args for limiting the time range for the selected labels or label values.
+
+Additionally VictoriaMetrics provides the following handlers:
+
+* `/api/v1/series/count` - it returns the total number of time series in the database. Note that this handler scans all the inverted index,
+  so it can be slow if the database contains tens of millions of time series.
+* `/api/v1/labels/count` - it returns a list of `label: values_count` entries. It can be used for determining labels with the maximum number of values.
+
+
+### How to build from sources
+
+We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
+[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) instead of building VictoriaMetrics
+from sources. Building from sources is reasonable when developing additional features specific
+to your needs.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make victoria-metrics` from the root folder of the repository.
+   It builds `victoria-metrics` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make victoria-metrics-prod` from the root folder of the repository.
+   It builds `victoria-metrics-prod` binary and puts it into the `bin` folder.
+
+#### ARM build
+
+ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
+
+#### Development ARM build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make victoria-metrics-arm` or `make victoria-metrics-arm64` from the root folder of the repository.
+   It builds `victoria-metrics-arm` or `victoria-metrics-arm64` binary respectively and puts it into the `bin` folder.
+
+#### Production ARM build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make victoria-metrics-arm-prod` or `make victoria-metrics-arm64-prod` from the root folder of the repository.
+   It builds `victoria-metrics-arm-prod` or `victoria-metrics-arm64-prod` binary respectively and puts it into the `bin` folder.
+
+#### Pure Go build (CGO_ENABLED=0)
+
+`Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
+This is an experimental mode, which may result in a lower compression ratio and slower decompression performance.
+Use it with caution!
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make victoria-metrics-pure` from the root folder of the repository.
+   It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-victoria-metrics`. It builds `victoriametrics/victoria-metrics:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-victoria-metrics`.
+
+
+### Start with docker-compose
+
+[Docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/docker-compose.yml)
+helps to spin up VictoriaMetrics, Prometheus and Grafana with one command.
+More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#folder-contains-basic-images-and-tools-for-building-and-running-victoria-metrics-in-docker).
+
+
+### Setting up service
+
+Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
+
+
+### Third-party contributions
+
+* [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
+
+
+### How to work with snapshots?
+
+VictoriaMetrics can create [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+for all the data stored under `-storageDataPath` directory.
+Navigate to `http://<victoriametrics-addr>:8428/snapshot/create` in order to create an instant snapshot.
+The page will return the following JSON response:
+
+```
+{"status":"ok","snapshot":"<snapshot-name>"}
+```
+
+Snapshots are created under `<-storageDataPath>/snapshots` directory, where `<-storageDataPath>`
+is the command-line flag value. Snapshots can be archived to backup storage at any time
+with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+
+The `http://<victoriametrics-addr>:8428/snapshot/list` page contains the list of available snapshots.
+
+Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete?snapshot=<snapshot-name>` in order
+to delete `<snapshot-name>` snapshot.
+
+Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete_all` in order to delete all the snapshots.
+
+Steps for restoring from a snapshot:
+1. Stop VictoriaMetrics with `kill -INT`.
+2. Restore snapshot contents from backup with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md)
+   to the directory pointed by `-storageDataPath`.
+3. Start VictoriaMetrics.
+
+
+### How to delete time series?
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`,
+where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to delete. After that all the time series matching the given selector are deleted. Storage space for
+the deleted time series isn't freed instantly - it is freed during subsequent merges of data files.
+
+It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
+before actually deleting the metrics.
+
+
+### How to export time series?
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
+where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
+The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
+Each JSON line would contain data for a single time series. An example output:
+
+```
+{"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
+{"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
+```
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-time-series-data).
+
+
+### How to import time series data?
+
+Time series data can be imported via any supported ingestion protocol:
+
+* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
+* [Influx line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
+* [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
+* [OpenTSDB telnet put protocol](#sending-data-via-telnet-put-protocol)
+* [OpenTSDB http /api/put](#sending-opentsdb-data-via-http-apiput-requests)
+* `/api/v1/import` http POST handler, which accepts data from [/api/v1/export](#how-to-export-time-series).
+
+The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import`. Example for importing data obtained via `/api/v1/export`:
+
+```
+# Export the data from <source-victoriametrics>:
+curl -s 'http://source-victoriametrics:8428/api/v1/export' -d 'match={__name__!=""}' > exported_data.jsonl
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST 'http://destination-victoriametrics:8428/api/v1/import' -T exported_data.jsonl
+```
+
+Each request to `/api/v1/import` can load up to a single vCPU core on VictoriaMetrics. Import speed can be improved by splitting the original file into smaller parts
+and importing them concurrently. Note that the original file must be split on newlines.
+
+
+### Federation
+
+VictoriaMetrics exports [Prometheus-compatible federation data](https://prometheus.io/docs/prometheus/latest/federation/)
+at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
+
+Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
+`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. By default, the last point
+on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden.
+For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
+with scrape intervals exceeding `5m`.
+
+
+### Capacity planning
+
+A rough estimation of the required resources for ingestion path:
+
+* RAM size: less than 1KB per active time series. So, ~1GB of RAM is required for 1M active time series.
+  Time series is considered active if new data points have been added to it recently or if it has been recently queried.
+  The number of active time series may be obtained from `vm_cache_entries{type="storage/hour_metric_ids"}` metric
+  exproted on the `/metrics` page.
+  VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited by `-memory.allowedPercent` flag.
+
+* CPU cores: a CPU core per 300K inserted data points per second. So, ~4 CPU cores are required for processing
+  the insert stream of 1M data points per second. The ingestion rate may be lower for high cardinality data or for time series with high number of labels.
+  See [this article](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) for details.
+  If you see lower numbers per CPU core, then it is likely active time series info doesn't fit caches,
+  so you need more RAM for lowering CPU usage.
+
+* Storage space: less than a byte per data point on average. So, ~260GB is required for storing a month-long insert stream
+  of 100K data points per second.
+  The actual storage size heavily depends on data randomness (entropy). Higher randomness means higher storage size requirements.
+  Read [this article](https://medium.com/faun/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
+  for details.
+
+* Network usage: outbound traffic is negligible. Ingress traffic is ~100 bytes per ingested data point via
+  [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
+  The actual ingress bandwidth usage depends on the average number of labels per ingested metric and the average size
+  of label values. The higher number of per-metric labels and longer label values mean the higher ingress bandwidth.
+
+
+The required resources for query path:
+
+* RAM size: depends on the number of time series to scan in each query and the `step`
+  argument passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).
+  The higher number of scanned time series and lower `step` argument results in the higher RAM usage.
+
+* CPU cores: a CPU core per 30 millions of scanned data points per second.
+
+* Network usage: depends on the frequency and the type of incoming requests. Typical Grafana dashboards usually
+  require negligible network bandwidth.
+
+
+### High availability
+
+1) Install multiple VictoriaMetrics instances in distinct datacenters (availability zones).
+2) Add addresses of these instances to `remote_write` section in Prometheus config:
+
+```yml
+remote_write:
+  - url: http://<victoriametrics-addr-1>:8428/api/v1/write
+    queue_config:
+      max_samples_per_send: 10000
+  # ...
+  - url: http://<victoriametrics-addr-N>:8428/api/v1/write
+    queue_config:
+      max_samples_per_send: 10000
+```
+
+3) Apply the updated config:
+
+```
+kill -HUP `pidof prometheus`
+```
+
+4) Now Prometheus should write data into all the configured `remote_write` urls in parallel.
+5) Set up [Promxy](https://github.com/jacksontj/promxy) in front of all the VictoriaMetrics replicas.
+6) Set up Prometheus datasource in Grafana that points to Promxy.
+
+
+If you have Prometheus HA pairs with replicas `r1` and `r2` in each pair, then configure each `r1`
+to write data to `victoriametrics-addr-1`, while each `r2` should write data to `victoriametrics-addr-2`.
+
+
+### Multiple retentions
+
+Just start multiple VictoriaMetrics instances with distinct values for the following flags:
+
+* `-retentionPeriod`
+* `-storageDataPath`, so the data for each retention period is saved in a separate directory
+* `-httpListenAddr`, so clients may reach VictoriaMetrics instance with proper retention
+
+
+### Downsampling
+
+There is no downsampling support at the moment, but:
+- VictoriaMetrics is optimized for querying big amounts of raw data. See benchmark results for heavy queries
+  in [this article](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+- VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
+  for details.
+
+These properties reduce the need in downsampling. We plan to implement downsampling in the future.
+See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
+
+
+### Multi-tenancy
+
+Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) instead.
+
+
+### Scalability and cluster version
+
+Though single-node VictoriaMetrics cannot scale to multiple nodes, it is optimized for resource usage - storage size / bandwidth / IOPS, RAM, CPU.
+This means that a single-node VictoriaMetrics may scale vertically and substitute a moderately sized cluster built with competing solutions
+such as Thanos, Uber M3, InfluxDB or TimescaleDB. See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+
+So try single-node VictoriaMetrics at first and then [switch to cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) if you still need
+horizontally scalable long-term remote storage for really large Prometheus deployments.
+[Contact us](mailto:info@victoriametrics.com) for paid support.
+
+
+### Alerting
+
+VictoriaMetrics doesn't support rule evaluation and alerting yet, so these actions must be performed either
+on [Prometheus side](https://prometheus.io/docs/alerting/overview/) or on [Grafana side](https://grafana.com/docs/alerting/rules/).
+
+
+### Security
+
+Do not forget protecting sensitive endpoints in VictoriaMetrics when exposing it to untrusted networks such as the internet.
+Consider setting the following command-line flags:
+
+* `-tls`, `-tlsCertFile` and `-tlsKeyFile` for switching from HTTP to HTTPS.
+* `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
+  with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
+* `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
+* `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
+
+Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
+For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`.
+
+
+### Tuning
+
+* There is no need in VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
+  which are automatically adjusted for the available CPU and RAM resources.
+* There is no need in Operating System tuning since VictoriaMetrics is optimized for default OS settings.
+  The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
+  so Prometheus instances could establish more connections to VictoriaMetrics.
+* The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
+  since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
+  If you plan storing more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
+  then the following options are recommended to pass to `mkfs.ext4`:
+
+```
+mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
+```
+
+
+### Monitoring
+
+VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page.
+Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics.
+There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
+
+The most interesting metrics are:
+
+* `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
+  aka active time series.
+* `rate(vm_new_timeseries_created_total[5m])` - time series churn rate.
+* `vm_rows{type="indexdb"}` - the number of rows in inverted index. High value for this number usually mean high churn rate for time series.
+* Sum of `vm_rows{type="storage/big"}` and `vm_rows{type="storage/small"}` - total number of `(timestamp, value)` data points
+  in the database.
+* Sum of all the `vm_cache_size_bytes` metrics - the total size of all the caches in the database.
+* `vm_allowed_memory_bytes` - the maximum allowed size for caches in the database. It is calculated as `system_memory * <-memory.allowedPercent> / 100`,
+  where `system_memory` is the amount of system memory and `-memory.allowedPercent` is the corresponding flag value.
+* `vm_rows_inserted_total` - the total number of inserted rows since VictoriaMetrics start.
+
+
+### Troubleshooting
+
+* It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
+  in tweaking these flag values arises.
+
+* If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
+  then it is likely you have too many active time series for the current amount of RAM.
+  It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
+  ingestion performance.
+  Another option is to increase `-memory.allowedPercent` command-line flag value. Be careful with this
+  option, since too big value for `-memory.allowedPercent` may result in high I/O usage.
+
+* VictoriaMetrics requires free disk space for [merging data files to bigger ones](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  It may slow down when there is no enough free space left. So make sure `-storageDataPath` directory
+  has at least 20% of free space comparing to disk size.
+
+* If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
+  then just remove directoreis with broken parts. This will recover VictoriaMetrics at the cost
+  of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
+  for automatic recovering from such errors.
+
+
+### Backfilling
+
+Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
+
+It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
+historical data with timestamps from the past, since the cache assumes that the data is written with
+the current timestamps. Query cache can be enabled after the backfilling is complete.
+
+
+### Profiling
+
+VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+- Memory profile. It can be collected with the following command:
+```
+curl -s http://<victoria-metrics-host>:8428/debug/pprof/heap > mem.pprof
+```
+
+- CPU profile. It can be collected with the following command:
+```
+curl -s http://<victoria-metrics-host>:8428/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+
+
+## Integrations
+
+* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
+  See [these docs](https://github.com/netdata/netdata#integrations).
+* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
+  See [this example](/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+* [Ansible role for installing VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
+
+
+## Roadmap
+
+- [ ] Replication [#118](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/118)
+- [ ] Support of Object Storages (GCS, S3, Azure Storage) [#38](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/38)
+- [ ] Data downsampling [#36](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36)
+- [ ] Alert Manager Integration [#119](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/119) 
+- [ ] CLI tool for data migration, re-balancing and adding/removing nodes [#103](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/103)
+
+
+The discussion happens [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/129). Feel free to comment any item or add own one.
+
+
+## Contacts
+
+Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics.com](mailto:info@victoriametrics.com).
+
+
+## Community and contributions
+
+Feel free asking any questions regarding VictoriaMetrics:
+
+- [slack](http://slack.victoriametrics.com/)
+- [telegram-en](https://t.me/VictoriaMetrics_en)
+- [telegram-ru](https://t.me/VictoriaMetrics_ru1)
+- [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
+
+
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
+
+
+## Reporting bugs
+
+Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+
+
+## Victoria Metrics Logo
+
+[Zip](VM_logo.zip) contains three folders with different image orientation (main color and inverted version).
+
+Files included in each folder:
+
+* 2 JPEG Preview files
+* 2 PNG Preview files with transparent background
+* 2 EPS Adobe Illustrator EPS10 files
+
+
+### Logo Usage Guidelines
+
+#### Font used:
+
+* Lato Black
+* Lato Regular
+
+#### Color Palette:
+
+* HEX [#110f0f](https://www.color-hex.com/color/110f0f)
+* HEX [#ffffff](https://www.color-hex.com/color/ffffff)
+
+### We kindly ask:
+
+- Please don't use any other font instead of suggested.
+- There should be sufficient clear space around the logo.
+- Do not change spacing, alignment, or relative locations of the design elements.
+- Do not change the proportions of any of the design elements or the design itself. You    may resize as needed but must retain all proportions.

docs/vmbackup.md

@@ -0,0 +1,181 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.

docs/vmrestore.md

@@ -0,0 +1,86 @@
+## vmrestore
+
+`vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
+
+Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+when restarting `vmrestore` with the same args.
+
+
+### Usage
+
+VictoriaMetrics must be stopped during the restore process.
+
+```
+vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+
+```
+
+* `<bucket>` is [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets) name.
+* `<path/to/backup>` is the path to backup made with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) on GCS bucket.
+* `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
+  to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
+
+The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup.
+
+
+### Troubleshooting
+
+* If `vmrestore` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
+
+
+### Advanced usage
+
+Run `vmrestore -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum download speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -src string
+    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -storageDataPath string
+    	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmrestore` from the root folder of the repository.
+   It builds `vmrestore` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmrestore-prod` from the root folder of the repository.
+   It builds `vmrestore-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.

go.mod

@@ -1,18 +1,32 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
 require (
-	github.com/VictoriaMetrics/fastcache v1.5.2
-	github.com/VictoriaMetrics/metrics v1.7.2
-	github.com/cespare/xxhash/v2 v2.1.0
+	cloud.google.com/go v0.49.0 // indirect
+	cloud.google.com/go/storage v1.4.0
+	github.com/VictoriaMetrics/fastcache v1.5.4
+	github.com/VictoriaMetrics/metrics v1.9.3
+	github.com/aws/aws-sdk-go v1.26.0
+	github.com/cespare/xxhash/v2 v2.1.1
+	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
 	github.com/golang/snappy v0.0.1
-	github.com/google/go-cmp v0.3.0 // indirect
-	github.com/klauspost/compress v1.9.1
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/klauspost/compress v1.9.4
 	github.com/valyala/fastjson v1.4.1
 	github.com/valyala/fastrand v1.0.0
-	github.com/valyala/gozstd v1.6.2
+	github.com/valyala/gozstd v1.6.3
 	github.com/valyala/histogram v1.0.1
-	github.com/valyala/quicktemplate v1.3.1
-	golang.org/x/sys v0.0.0-20191027211539-f8518d3b3627
+	github.com/valyala/quicktemplate v1.4.1
+	go.opencensus.io v0.22.2 // indirect
+	golang.org/x/exp v0.0.0-20191129062945-2f5052295587 // indirect
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
+	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
+	golang.org/x/tools v0.0.0-20191212051200-825cb0626375 // indirect
+	google.golang.org/api v0.14.0
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f // indirect
+	google.golang.org/grpc v1.25.1 // indirect
 )
 
 go 1.12

go.sum

@@ -1,39 +1,99 @@
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
-github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
-github.com/VictoriaMetrics/fastcache v1.5.2 h1:Erd8iIuBAL9kke8JzM4+WxkKuFkHh3ktwLanJvDgR44=
-github.com/VictoriaMetrics/fastcache v1.5.2/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
-github.com/VictoriaMetrics/metrics v1.7.2 h1:PzC0SEo5lbbNK7xaYwclCCdoaIGRmXOfflIMF3LpSW4=
-github.com/VictoriaMetrics/metrics v1.7.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ=
+cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY=
+cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.4.0 h1:KDdqY5VTXBTqpSbctVTt0mVvfanP6JZzNzLE0qNY100=
+cloud.google.com/go/storage v1.4.0/go.mod h1:ZusYJWlOshgSBGbt6K3GnB3MT3H1xs2id9+TCl4fDBA=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/VictoriaMetrics/fastcache v1.5.4 h1:0BaXbRH01RycJk79OOBwMCXlNryko9z4yEf6RqbP+Xo=
+github.com/VictoriaMetrics/fastcache v1.5.4/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
+github.com/VictoriaMetrics/metrics v1.9.3 h1:+1kZnOIb8RY825Nb9q9yMrPcOYuPE2GrZWxUh59XnHI=
+github.com/VictoriaMetrics/metrics v1.9.3/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0=
-github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
-github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
-github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/aws/aws-sdk-go v1.26.0 h1:uddjCeunG1nroOMjcMO57g/vfwY7jJ1zYSEGmdeCeEs=
+github.com/aws/aws-sdk-go v1.26.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.9.1 h1:TWy0o9J9c6LK9C8t7Msh6IAJNXbsU/nvKLTQUU5HdaY=
-github.com/klauspost/compress v1.9.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.4 h1:xhvAeUPQ2drNUhKtrGdTGNvV9nNafHMUkRyLkzxJoB4=
+github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
@@ -41,13 +101,144 @@ github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/y
 github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
-github.com/valyala/gozstd v1.6.2 h1:MgBfNm0I8IKm51LUTTKfO9vi4BtmoH7kBXeUvgaiZVU=
-github.com/valyala/gozstd v1.6.2/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/valyala/gozstd v1.6.3 h1:kr3oF/F1RvxYr8wgPjrH04gvHuMEL99pPC9e+5pPQdU=
+github.com/valyala/gozstd v1.6.3/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg=
 github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto=
-github.com/valyala/quicktemplate v1.3.1 h1:V9Ixd/ONuoT6C1ipx8XR2dNGSDgIVnvT4ezZ38ZWllU=
-github.com/valyala/quicktemplate v1.3.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/quicktemplate v1.4.1 h1:tEtkSN6mTCJlYVT7As5x4wjtkk2hj2thsb0M+AcAVeM=
+github.com/valyala/quicktemplate v1.4.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587 h1:5Uz0rkjCFu9BC9gCRN7EkwVvhNyQgGWb8KNJrPwBoHY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sys v0.0.0-20191027211539-f8518d3b3627 h1:/FZUR3d/QsXe4AcJyJFCc40TOj3y6Hs23Y3YJlvVkWo=
-golang.org/x/sys v0.0.0-20191027211539-f8518d3b3627/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191212051200-825cb0626375 h1:hrBCt+pkb1irxFjg/GeadSn24gLxi5/Z861I53OkQbc=
+golang.org/x/tools v0.0.0-20191212051200-825cb0626375/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f h1:naitw5DILWPQvG0oG04mR9jF8fmKpRdW3E3zzKA4D0Y=
+google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

lib/backup/actions/backup.go

@@ -0,0 +1,167 @@
+package actions
+
+import (
+	"fmt"
+	"io"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Backup performs backup according to the provided settings.
+//
+// Note that the backup works only for VictoriaMetrics snapshots
+// made via `/snapshot/create`. It works improperly on mutable files.
+type Backup struct {
+	// Concurrency is the number of concurrent workers during the backup.
+	// Concurrency=1 by default.
+	Concurrency int
+
+	// Src is backup source
+	Src *fslocal.FS
+
+	// Dst is backup destination.
+	//
+	// If dst contains the previous backup data, then incremental backup
+	// is made, i.e. only the changed data is uploaded.
+	//
+	// If dst points to empty dir, then full backup is made.
+	// Origin can be set to the previous backup in order to reduce backup duration
+	// and reduce network bandwidth usage.
+	Dst common.RemoteFS
+
+	// Origin is optional origin for speeding up full backup if Dst points
+	// to empty dir.
+	Origin common.OriginFS
+}
+
+// Run runs b with the provided settings.
+func (b *Backup) Run() error {
+	startTime := time.Now()
+
+	concurrency := b.Concurrency
+	src := b.Src
+	dst := b.Dst
+	origin := b.Origin
+
+	if origin == nil {
+		origin = &fsnil.FS{}
+	}
+
+	logger.Infof("starting backup from %s to %s using origin %s", src, dst, origin)
+
+	logger.Infof("obtaining list of parts at %s", src)
+	srcParts, err := src.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list src parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", dst)
+	dstParts, err := dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", origin)
+	originParts, err := origin.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list origin parts: %s", err)
+	}
+
+	backupSize := getPartsSize(srcParts)
+
+	partsToDelete := common.PartsDifference(dstParts, srcParts)
+	deleteSize := getPartsSize(partsToDelete)
+	if len(partsToDelete) > 0 {
+		logger.Infof("deleting %d parts from %s", len(partsToDelete), dst)
+		deletedParts := uint64(0)
+		err = runParallel(concurrency, partsToDelete, func(p common.Part) error {
+			logger.Infof("deleting %s from %s", &p, dst)
+			if err := dst.DeletePart(p); err != nil {
+				return fmt.Errorf("cannot delete %s from %s: %s", &p, dst, err)
+			}
+			atomic.AddUint64(&deletedParts, 1)
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&deletedParts)
+			logger.Infof("deleted %d out of %d parts from %s in %s", n, len(partsToDelete), dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+		if err := dst.RemoveEmptyDirs(); err != nil {
+			return fmt.Errorf("cannot remove empty directories at %s: %s", dst, err)
+		}
+	}
+
+	partsToCopy := common.PartsDifference(srcParts, dstParts)
+	originCopyParts := common.PartsIntersect(originParts, partsToCopy)
+	copySize := getPartsSize(originCopyParts)
+	if len(originCopyParts) > 0 {
+		logger.Infof("server-side copying %d parts from %s to %s", len(originCopyParts), origin, dst)
+		copiedParts := uint64(0)
+		err = runParallel(concurrency, originCopyParts, func(p common.Part) error {
+			logger.Infof("server-side copying %s from %s to %s", &p, origin, dst)
+			if err := dst.CopyPart(origin, p); err != nil {
+				return fmt.Errorf("cannot copy %s from %s to %s: %s", &p, origin, dst, err)
+			}
+			atomic.AddUint64(&copiedParts, 1)
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&copiedParts)
+			logger.Infof("server-side copied %d out of %d parts from %s to %s in %s", n, len(originCopyParts), origin, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	srcCopyParts := common.PartsDifference(partsToCopy, originParts)
+	uploadSize := getPartsSize(srcCopyParts)
+	if len(srcCopyParts) > 0 {
+		logger.Infof("uploading %d parts from %s to %s", len(srcCopyParts), src, dst)
+		bytesUploaded := uint64(0)
+		err = runParallel(concurrency, srcCopyParts, func(p common.Part) error {
+			logger.Infof("uploading %s from %s to %s", &p, src, dst)
+			rc, err := src.NewReadCloser(p)
+			if err != nil {
+				return fmt.Errorf("cannot create reader for %s from %s: %s", &p, src, err)
+			}
+			sr := &statReader{
+				r:         rc,
+				bytesRead: &bytesUploaded,
+			}
+			if err := dst.UploadPart(p, sr); err != nil {
+				return fmt.Errorf("cannot upload %s to %s: %s", &p, dst, err)
+			}
+			if err = rc.Close(); err != nil {
+				return fmt.Errorf("cannot close reader for %s from %s: %s", &p, src, err)
+			}
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&bytesUploaded)
+			logger.Infof("uploaded %d out of %d bytes from %s to %s in %s", n, uploadSize, src, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("backed up %d bytes in %s; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
+		backupSize, time.Since(startTime), deleteSize, copySize, uploadSize)
+
+	return nil
+}
+
+type statReader struct {
+	r         io.Reader
+	bytesRead *uint64
+}
+
+func (sr *statReader) Read(p []byte) (int, error) {
+	n, err := sr.r.Read(p)
+	atomic.AddUint64(sr.bytesRead, uint64(n))
+	return n, err
+}

lib/backup/actions/restore.go

@@ -0,0 +1,181 @@
+package actions
+
+import (
+	"fmt"
+	"io"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Restore restores data according to the provided settings.
+//
+// Note that the restore works only for VictoriaMetrics backups made from snapshots.
+// It works improperly on mutable files.
+type Restore struct {
+	// Concurrency is the number of concurrent workers to run during restore.
+	// Concurrency=1 is used by default.
+	Concurrency int
+
+	// Src is the source containing backed up data.
+	Src common.RemoteFS
+
+	// Dst is destination to restore the data.
+	//
+	// If dst points to existing directory, then incremental restore is performed,
+	// i.e. only new data is downloaded from src.
+	Dst *fslocal.FS
+}
+
+// Run runs r with the provided settings.
+func (r *Restore) Run() error {
+	startTime := time.Now()
+
+	// Make sure VictoriaMetrics doesn't run during the restore process.
+	if err := fs.MkdirAllIfNotExist(r.Dst.Dir); err != nil {
+		return fmt.Errorf("cannot create dir %q: %s", r.Dst.Dir, err)
+	}
+	flockF, err := fs.CreateFlockFile(r.Dst.Dir)
+	if err != nil {
+		return fmt.Errorf("cannot create lock file in %q; make sure VictoriaMetrics doesn't use the dir; error: %s", r.Dst.Dir, err)
+	}
+	defer fs.MustClose(flockF)
+
+	concurrency := r.Concurrency
+	src := r.Src
+	dst := r.Dst
+	logger.Infof("starting restore from %s to %s", src, dst)
+
+	logger.Infof("obtaining list of parts at %s", src)
+	srcParts, err := src.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list src parts: %s", err)
+	}
+	logger.Infof("obtaining list of parts at %s", dst)
+	dstParts, err := dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts: %s", err)
+	}
+
+	backupSize := getPartsSize(srcParts)
+
+	// Validate srcParts. They must cover the whole files.
+	common.SortParts(srcParts)
+	offset := uint64(0)
+	var pOld common.Part
+	var path string
+	for _, p := range srcParts {
+		if p.Path != path {
+			if offset != pOld.FileSize {
+				return fmt.Errorf("invalid size for %q; got %d; want %d", path, offset, pOld.FileSize)
+			}
+			pOld = p
+			path = p.Path
+			offset = 0
+		}
+		if p.Offset < offset {
+			return fmt.Errorf("there is an overlap in %d bytes between %s and %s", offset-p.Offset, &pOld, &p)
+		}
+		if p.Offset > offset {
+			if offset == 0 {
+				return fmt.Errorf("there is a gap in %d bytes from file start to %s", p.Offset, &p)
+			}
+			return fmt.Errorf("there is a gap in %d bytes between %s and %s", p.Offset-offset, &pOld, &p)
+		}
+		if p.Size != p.ActualSize {
+			return fmt.Errorf("invalid size for %s; got %d; want %d", &p, p.ActualSize, p.Size)
+		}
+		offset += p.Size
+	}
+
+	partsToDelete := common.PartsDifference(dstParts, srcParts)
+	deleteSize := uint64(0)
+	if len(partsToDelete) > 0 {
+		// Fully remove local file if certain parts from the remote part are missing.
+		pathsToDelete := make(map[string]bool)
+		for _, p := range partsToDelete {
+			pathsToDelete[p.Path] = true
+		}
+		logger.Infof("deleting %d files from %s", len(pathsToDelete), dst)
+		for path := range pathsToDelete {
+			logger.Infof("deleting %s from %s", path, dst)
+			size, err := dst.DeletePath(path)
+			if err != nil {
+				return fmt.Errorf("cannot delete %s from %s: %s", path, dst, err)
+			}
+			deleteSize += size
+		}
+		if err != nil {
+			return err
+		}
+		if err := dst.RemoveEmptyDirs(); err != nil {
+			return fmt.Errorf("cannot remove empty directories at %s: %s", dst, err)
+		}
+	}
+
+	// Re-read dstParts, since additional parts may be removed on the previous step.
+	dstParts, err = dst.ListParts()
+	if err != nil {
+		return fmt.Errorf("cannot list dst parts after the deletion: %s", err)
+	}
+
+	partsToCopy := common.PartsDifference(srcParts, dstParts)
+	downloadSize := getPartsSize(partsToCopy)
+	if len(partsToCopy) > 0 {
+		perPath := make(map[string][]common.Part)
+		for _, p := range partsToCopy {
+			parts := perPath[p.Path]
+			parts = append(parts, p)
+			perPath[p.Path] = parts
+		}
+		logger.Infof("downloading %d parts from %s to %s", len(partsToCopy), src, dst)
+		bytesDownloaded := uint64(0)
+		err = runParallelPerPath(concurrency, perPath, func(parts []common.Part) error {
+			// Sort partsToCopy in order to properly grow file size during downloading.
+			common.SortParts(parts)
+			for _, p := range parts {
+				logger.Infof("downloading %s from %s to %s", &p, src, dst)
+				wc, err := dst.NewWriteCloser(p)
+				if err != nil {
+					return fmt.Errorf("cannot create writer for %q to %s: %s", &p, dst, err)
+				}
+				sw := &statWriter{
+					w:            wc,
+					bytesWritten: &bytesDownloaded,
+				}
+				if err := src.DownloadPart(p, sw); err != nil {
+					return fmt.Errorf("cannot download %s to %s: %s", &p, dst, err)
+				}
+				if err := wc.Close(); err != nil {
+					return fmt.Errorf("cannot close reader fro %s from %s: %s", &p, src, err)
+				}
+			}
+			return nil
+		}, func(elapsed time.Duration) {
+			n := atomic.LoadUint64(&bytesDownloaded)
+			logger.Infof("downloaded %d out of %d bytes from %s to %s in %s", n, downloadSize, src, dst, elapsed)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("restored %d bytes from backup in %s; deleted %d bytes; downloaded %d bytes", backupSize, time.Since(startTime), deleteSize, downloadSize)
+
+	return nil
+}
+
+type statWriter struct {
+	w            io.Writer
+	bytesWritten *uint64
+}
+
+func (sw *statWriter) Write(p []byte) (int, error) {
+	n, err := sw.w.Write(p)
+	atomic.AddUint64(sw.bytesWritten, uint64(n))
+	return n, err
+}

lib/backup/actions/util.go

@@ -0,0 +1,235 @@
+package actions
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
+)
+
+var (
+	credsFilePath = flag.String("credsFilePath", "", "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.\n"+
+		"See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
+	configFilePath = flag.String("configFilePath", "", "Path to file with S3 configs. Configs are loaded from default location if not set.\n"+
+		"See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
+	configProfile    = flag.String("configProfile", "default", "Profile name for S3 configs")
+	customS3Endpoint = flag.String("customS3Endpoint", "", "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set")
+)
+
+func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
+	var err error
+	runWithProgress(progress, func() {
+		err = runParallelInternal(concurrency, parts, f)
+	})
+	return err
+}
+
+func runParallelPerPath(concurrency int, perPath map[string][]common.Part, f func(parts []common.Part) error, progress func(elapsed time.Duration)) error {
+	var err error
+	runWithProgress(progress, func() {
+		err = runParallelPerPathInternal(concurrency, perPath, f)
+	})
+	return err
+}
+
+func runParallelPerPathInternal(concurrency int, perPath map[string][]common.Part, f func(parts []common.Part) error) error {
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	if len(perPath) == 0 {
+		return nil
+	}
+
+	// len(perPath) capacity guarantees non-blocking behavior below.
+	resultCh := make(chan error, len(perPath))
+	workCh := make(chan []common.Part, len(perPath))
+	stopCh := make(chan struct{})
+
+	// Start workers
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for parts := range workCh {
+				select {
+				case <-stopCh:
+					return
+				default:
+				}
+				resultCh <- f(parts)
+			}
+		}()
+	}
+
+	// Feed workers with work.
+	for _, parts := range perPath {
+		workCh <- parts
+	}
+	close(workCh)
+
+	// Read results.
+	var err error
+	for i := 0; i < len(perPath); i++ {
+		err = <-resultCh
+		if err != nil {
+			// Stop the work.
+			close(stopCh)
+			break
+		}
+	}
+
+	// Wait for all the workers to stop.
+	wg.Wait()
+
+	return err
+}
+
+func runParallelInternal(concurrency int, parts []common.Part, f func(p common.Part) error) error {
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	if len(parts) == 0 {
+		return nil
+	}
+
+	// len(parts) capacity guarantees non-blocking behavior below.
+	resultCh := make(chan error, len(parts))
+	workCh := make(chan common.Part, len(parts))
+	stopCh := make(chan struct{})
+
+	// Start workers
+	var wg sync.WaitGroup
+	wg.Add(concurrency)
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			defer wg.Done()
+			for p := range workCh {
+				select {
+				case <-stopCh:
+					return
+				default:
+				}
+				resultCh <- f(p)
+			}
+		}()
+	}
+
+	// Feed workers with work.
+	for _, p := range parts {
+		workCh <- p
+	}
+	close(workCh)
+
+	// Read results.
+	var err error
+	for i := 0; i < len(parts); i++ {
+		err = <-resultCh
+		if err != nil {
+			// Stop the work.
+			close(stopCh)
+			break
+		}
+	}
+
+	// Wait for all the workers to stop.
+	wg.Wait()
+
+	return err
+}
+
+func runWithProgress(progress func(elapsed time.Duration), f func()) {
+	startTime := time.Now()
+	doneCh := make(chan struct{})
+	go func() {
+		f()
+		close(doneCh)
+	}()
+
+	tc := time.NewTicker(10 * time.Second)
+	for {
+		select {
+		case <-doneCh:
+			tc.Stop()
+			// The last progress call.
+			progress(time.Since(startTime))
+			return
+		case <-tc.C:
+			progress(time.Since(startTime))
+		}
+	}
+}
+
+func getPartsSize(parts []common.Part) uint64 {
+	n := uint64(0)
+	for _, p := range parts {
+		n += p.Size
+	}
+	return n
+}
+
+// NewRemoteFS returns new remote fs from the given path.
+func NewRemoteFS(path string) (common.RemoteFS, error) {
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	n := strings.Index(path, "://")
+	if n < 0 {
+		return nil, fmt.Errorf("Missing scheme in path %q. Supported schemes: `gcs://`, `s3://`, `fs://`", path)
+	}
+	scheme := path[:n]
+	dir := path[n+len("://"):]
+	switch scheme {
+	case "fs":
+		if !strings.HasPrefix(dir, "/") {
+			return nil, fmt.Errorf("dir must be absolute; got %q", dir)
+		}
+		fs := &fsremote.FS{
+			Dir: dir,
+		}
+		return fs, nil
+	case "gcs":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fs := &gcsremote.FS{
+			CredsFilePath: *credsFilePath,
+			Bucket:        bucket,
+			Dir:           dir,
+		}
+		if err := fs.Init(); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to gcs: %s", err)
+		}
+		return fs, nil
+	case "s3":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fs := &s3remote.FS{
+			CredsFilePath:  *credsFilePath,
+			ConfigFilePath: *configFilePath,
+			CustomEndpoint: *customS3Endpoint,
+			ProfileName:    *configProfile,
+			Bucket:         bucket,
+			Dir:            dir,
+		}
+		if err := fs.Init(); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to s3: %s", err)
+		}
+		return fs, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q in `-dst`", scheme)
+	}
+}

lib/backup/common/fs.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"io"
+)
+
+// OriginFS is an interface for remote origin filesystem.
+//
+// This filesystem is used for performing server-side file copies
+// instead of uploading data from local filesystem.
+type OriginFS interface {
+	// String must return human-readable representation of OriginFS.
+	String() string
+
+	// ListParts must return all the parts for the OriginFS.
+	ListParts() ([]Part, error)
+}
+
+// RemoteFS is a filesystem where backups are stored.
+type RemoteFS interface {
+	// String must return human-readable representation of RemoteFS.
+	String() string
+
+	// ListParts must return all the parts for the RemoteFS.
+	ListParts() ([]Part, error)
+
+	// DeletePart must delete part p from RemoteFS.
+	DeletePart(p Part) error
+
+	// RemoveEmptyDirs must recursively remove empty directories in RemoteFS.
+	RemoveEmptyDirs() error
+
+	// CopyPart must copy part p from dstFS to RemoteFS.
+	CopyPart(dstFS OriginFS, p Part) error
+
+	// DownloadPart must download part p from RemoteFS to w.
+	DownloadPart(p Part, w io.Writer) error
+
+	// UploadPart must upload part p from r to RemoteFS.
+	UploadPart(p Part, r io.Reader) error
+}

lib/backup/common/part.go

@@ -0,0 +1,136 @@
+package common
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Part is an atomic unit for transfer during backup / restore.
+//
+// Each source file can be split into parts with up to MaxPartSize sizes.
+type Part struct {
+	// Path is the path to file for backup.
+	Path string
+
+	// FileSize is the size of the whole file for the given part.
+	FileSize uint64
+
+	// Offset is offset in the file to backup.
+	Offset uint64
+
+	// Size is the size of the part to backup starting from Offset.
+	Size uint64
+
+	// ActualSize is the actual size of the part.
+	//
+	// The part is considered broken if it isn't equal to Size.
+	// Such a part must be removed from remote storage.
+	ActualSize uint64
+}
+
+func (p *Part) key() string {
+	return fmt.Sprintf("%s%016X%016X%016X%016X", p.Path, p.FileSize, p.Offset, p.Size, p.ActualSize)
+}
+
+// String returns human-readable representation of the part.
+func (p *Part) String() string {
+	return fmt.Sprintf("part{path: %q, file_size: %d, offset: %d, size: %d}", p.Path, p.FileSize, p.Offset, p.Size)
+}
+
+// RemotePath returns remote path for the part p and the given prefix.
+func (p *Part) RemotePath(prefix string) string {
+	for strings.HasSuffix(prefix, "/") {
+		prefix = prefix[:len(prefix)-1]
+	}
+	return fmt.Sprintf("%s/%s/%016X_%016X_%016X", prefix, p.Path, p.FileSize, p.Offset, p.Size)
+}
+
+var partNameRegexp = regexp.MustCompile(`^(.+)/([0-9A-F]{16})_([0-9A-F]{16})_([0-9A-F]{16})$`)
+
+// ParseFromRemotePath parses p from remotePath.
+//
+// Returns true on success.
+func (p *Part) ParseFromRemotePath(remotePath string) bool {
+	tmp := partNameRegexp.FindStringSubmatch(remotePath)
+	if len(tmp) != 5 {
+		return false
+	}
+	path := tmp[1]
+	for strings.HasPrefix(path, "/") {
+		path = path[1:]
+	}
+	fileSize, err := strconv.ParseUint(tmp[2], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse fileSize from %q: %s", tmp[2], err)
+	}
+	offset, err := strconv.ParseUint(tmp[3], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse offset from %q: %s", tmp[3], err)
+	}
+	size, err := strconv.ParseUint(tmp[4], 16, 64)
+	if err != nil {
+		logger.Panicf("BUG: cannot parse size from %q: %s", tmp[4], err)
+	}
+	p.Path = path
+	p.FileSize = fileSize
+	p.Offset = offset
+	p.Size = size
+	return true
+}
+
+// MaxPartSize is the maximum size for each part.
+//
+// The MaxPartSize reduces bandwidth usage during retires on network errors
+// when transferring multi-TB files.
+const MaxPartSize = 128 * 1024 * 1024
+
+// SortParts sorts parts by (Path, Offset)
+func SortParts(parts []Part) {
+	sort.Slice(parts, func(i, j int) bool {
+		a := parts[i]
+		b := parts[j]
+		if a.Path != b.Path {
+			return a.Path < b.Path
+		}
+		return a.Offset < b.Offset
+	})
+}
+
+// PartsDifference returns a - b
+func PartsDifference(a, b []Part) []Part {
+	m := make(map[string]bool, len(b))
+	for _, p := range b {
+		k := p.key()
+		m[k] = true
+	}
+	var d []Part
+	for _, p := range a {
+		k := p.key()
+		if !m[k] {
+			d = append(d, p)
+		}
+	}
+	return d
+}
+
+// PartsIntersect returns the intersection of a and b
+func PartsIntersect(a, b []Part) []Part {
+	m := make(map[string]bool, len(a))
+	for _, p := range a {
+		k := p.key()
+		m[k] = true
+	}
+	var d []Part
+	for _, p := range b {
+		k := p.key()
+		if m[k] {
+			d = append(d, p)
+		}
+	}
+	return d
+}

lib/backup/fscommon/fscommon.go

@@ -0,0 +1,262 @@
+package fscommon
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FsyncFile fsyncs path contents and the parent directory contents.
+func FsyncFile(path string) error {
+	if err := fsync(path); err != nil {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("cannot fsync file %q: %s", path, err)
+	}
+	dir := filepath.Dir(path)
+	if err := fsync(dir); err != nil {
+		return fmt.Errorf("cannot fsync dir %q: %s", dir, err)
+	}
+	return nil
+}
+
+// FsyncDir fsyncs dir contents.
+func FsyncDir(dir string) error {
+	return fsync(dir)
+}
+
+func fsync(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+// AppendFiles appends all the files from dir to dst.
+//
+// All the appended files will have dir prefix.
+func AppendFiles(dst []string, dir string) ([]string, error) {
+	d, err := os.Open(dir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open %q: %s", dir, err)
+	}
+	dst, err = appendFilesInternal(dst, d)
+	if err1 := d.Close(); err1 != nil {
+		err = err1
+	}
+	return dst, err
+}
+
+func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
+	dir := d.Name()
+	dfi, err := d.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", dir, err)
+	}
+	if !dfi.IsDir() {
+		return nil, fmt.Errorf("%q isn't a directory", dir)
+	}
+	fis, err := d.Readdir(-1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read directory contents in %q: %s", dir, err)
+	}
+	for _, fi := range fis {
+		name := fi.Name()
+		if name == "." || name == ".." {
+			continue
+		}
+		if name == "flock.lock" {
+			// Do not take into account flock.lock files, since they are used
+			// for preventing from concurrent access.
+			continue
+		}
+		path := dir + "/" + name
+		if fi.IsDir() {
+			// Process directory
+			dst, err = AppendFiles(dst, path)
+			if err != nil {
+				return nil, fmt.Errorf("cannot list %q: %s", path, err)
+			}
+			continue
+		}
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Process file
+			dst = append(dst, path)
+			continue
+		}
+		pathOrig := path
+	again:
+		// Process symlink
+		pathReal, err := filepath.EvalSymlinks(pathOrig)
+		if err != nil {
+			if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
+				// Skip symlink that points to nowhere.
+				continue
+			}
+			return nil, fmt.Errorf("cannot resolve symlink %q: %s", pathOrig, err)
+		}
+		sfi, err := os.Stat(pathReal)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat %q from symlink %q: %s", pathReal, path, err)
+		}
+		if sfi.IsDir() {
+			// Symlink points to directory
+			dstNew, err := AppendFiles(dst, pathReal)
+			if err != nil {
+				return nil, fmt.Errorf("cannot list files at %q from symlink %q: %s", pathReal, path, err)
+			}
+			pathReal += "/"
+			for i := len(dst); i < len(dstNew); i++ {
+				x := dstNew[i]
+				if !strings.HasPrefix(x, pathReal) {
+					return nil, fmt.Errorf("unexpected prefix for path %q; want %q", x, pathReal)
+				}
+				dstNew[i] = path + "/" + x[len(pathReal):]
+			}
+			dst = dstNew
+			continue
+		}
+		if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Symlink points to file
+			dst = append(dst, path)
+			continue
+		}
+		// Symlink points to symlink. Process it again.
+		pathOrig = pathReal
+		goto again
+	}
+	return dst, nil
+}
+
+// RemoveEmptyDirs recursively removes empty directories under the given dir.
+func RemoveEmptyDirs(dir string) error {
+	_, err := removeEmptyDirs(dir)
+	return err
+}
+
+func removeEmptyDirs(dir string) (bool, error) {
+	d, err := os.Open(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return true, nil
+		}
+		return false, err
+	}
+	ok, err := removeEmptyDirsInternal(d)
+	if err1 := d.Close(); err1 != nil {
+		err = err1
+	}
+	if err != nil {
+		return false, err
+	}
+	return ok, nil
+}
+
+func removeEmptyDirsInternal(d *os.File) (bool, error) {
+	dir := d.Name()
+	dfi, err := d.Stat()
+	if err != nil {
+		return false, fmt.Errorf("cannot stat %q: %s", dir, err)
+	}
+	if !dfi.IsDir() {
+		return false, fmt.Errorf("%q isn't a directory", dir)
+	}
+	fis, err := d.Readdir(-1)
+	if err != nil {
+		return false, fmt.Errorf("cannot read directory contents in %q: %s", dir, err)
+	}
+	dirEntries := 0
+	hasFlock := false
+	for _, fi := range fis {
+		name := fi.Name()
+		if name == "." || name == ".." {
+			continue
+		}
+		path := dir + "/" + name
+		if fi.IsDir() {
+			// Process directory
+			ok, err := removeEmptyDirs(path)
+			if err != nil {
+				return false, fmt.Errorf("cannot list %q: %s", path, err)
+			}
+			if !ok {
+				dirEntries++
+			}
+			continue
+		}
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			if name == "flock.lock" {
+				hasFlock = true
+				continue
+			}
+			// Skip plain files.
+			dirEntries++
+			continue
+		}
+		pathOrig := path
+	again:
+		// Process symlink
+		pathReal, err := filepath.EvalSymlinks(pathOrig)
+		if err != nil {
+			if os.IsNotExist(err) || strings.Contains(err.Error(), "no such file or directory") {
+				// Remove symlink that points to nowere.
+				logger.Infof("removing broken symlink %q", pathOrig)
+				if err := os.Remove(pathOrig); err != nil {
+					return false, fmt.Errorf("cannot remove %q: %s", pathOrig, err)
+				}
+				continue
+			}
+			return false, fmt.Errorf("cannot resolve symlink %q: %s", pathOrig, err)
+		}
+		sfi, err := os.Stat(pathReal)
+		if err != nil {
+			return false, fmt.Errorf("cannot stat %q from symlink %q: %s", pathReal, path, err)
+		}
+		if sfi.IsDir() {
+			// Symlink points to directory
+			ok, err := removeEmptyDirs(pathReal)
+			if err != nil {
+				return false, fmt.Errorf("cannot list files at %q from symlink %q: %s", pathReal, path, err)
+			}
+			if !ok {
+				dirEntries++
+			} else {
+				// Remove the symlink
+				logger.Infof("removing symlink that points to empty dir %q", pathOrig)
+				if err := os.Remove(pathOrig); err != nil {
+					return false, fmt.Errorf("cannot remove %q: %s", pathOrig, err)
+				}
+			}
+			continue
+		}
+		if sfi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			// Symlink points to file. Skip it.
+			dirEntries++
+			continue
+		}
+		// Symlink points to symlink. Process it again.
+		pathOrig = pathReal
+		goto again
+	}
+	if dirEntries > 0 {
+		return false, nil
+	}
+	logger.Infof("removing empty dir %q", dir)
+	if hasFlock {
+		flockFilepath := dir + "/flock.lock"
+		if err := os.Remove(flockFilepath); err != nil {
+			return false, fmt.Errorf("cannot remove %q: %s", flockFilepath, err)
+		}
+	}
+	if err := os.Remove(dir); err != nil {
+		return false, fmt.Errorf("cannot remove %q: %s", dir, err)
+	}
+	return true, nil
+}

lib/backup/fslocal/bandwidth_limiter.go

@@ -0,0 +1,116 @@
+package fslocal
+
+import (
+	"io"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+type bandwidthLimiter struct {
+	perSecondLimit int
+
+	c *sync.Cond
+
+	// quota for the current second
+	quota int
+}
+
+func newBandwidthLimiter(perSecondLimit int) *bandwidthLimiter {
+	if perSecondLimit <= 0 {
+		logger.Panicf("BUG: perSecondLimit must be positive; got %d", perSecondLimit)
+	}
+	var bl bandwidthLimiter
+	bl.perSecondLimit = perSecondLimit
+	var mu sync.Mutex
+	bl.c = sync.NewCond(&mu)
+	go bl.perSecondUpdater()
+	return &bl
+}
+
+func (bl *bandwidthLimiter) NewReadCloser(rc io.ReadCloser) *bandwidthLimitedReader {
+	return &bandwidthLimitedReader{
+		rc: rc,
+		bl: bl,
+	}
+}
+
+func (bl *bandwidthLimiter) NewWriteCloser(wc io.WriteCloser) *bandwidthLimitedWriter {
+	return &bandwidthLimitedWriter{
+		wc: wc,
+		bl: bl,
+	}
+}
+
+type bandwidthLimitedReader struct {
+	rc io.ReadCloser
+	bl *bandwidthLimiter
+}
+
+func (blr *bandwidthLimitedReader) Read(p []byte) (int, error) {
+	quota := blr.bl.GetQuota(len(p))
+	return blr.rc.Read(p[:quota])
+}
+
+func (blr *bandwidthLimitedReader) Close() error {
+	return blr.rc.Close()
+}
+
+type bandwidthLimitedWriter struct {
+	wc io.WriteCloser
+	bl *bandwidthLimiter
+}
+
+func (blw *bandwidthLimitedWriter) Write(p []byte) (int, error) {
+	nn := 0
+	for len(p) > 0 {
+		quota := blw.bl.GetQuota(len(p))
+		n, err := blw.wc.Write(p[:quota])
+		nn += n
+		if err != nil {
+			return nn, err
+		}
+		p = p[quota:]
+	}
+	return nn, nil
+}
+
+func (blw *bandwidthLimitedWriter) Close() error {
+	return blw.wc.Close()
+}
+
+func (bl *bandwidthLimiter) perSecondUpdater() {
+	tc := time.NewTicker(time.Second)
+	c := bl.c
+	for range tc.C {
+		c.L.Lock()
+		bl.quota = bl.perSecondLimit
+		c.Signal()
+		c.L.Unlock()
+	}
+}
+
+// GetQuota returns the number in the range [1..n] - the allowed quota for now.
+//
+// The function blocks until at least 1 can be returned from it.
+func (bl *bandwidthLimiter) GetQuota(n int) int {
+	if n <= 0 {
+		logger.Panicf("BUG: n must be positive; got %d", n)
+	}
+	c := bl.c
+	c.L.Lock()
+	for bl.quota <= 0 {
+		c.Wait()
+	}
+	quota := bl.quota
+	if quota > n {
+		quota = n
+	}
+	bl.quota -= quota
+	if bl.quota > 0 {
+		c.Signal()
+	}
+	c.L.Unlock()
+	return quota
+}

lib/backup/fslocal/fslocal.go

@@ -0,0 +1,232 @@
+package fslocal
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represents local filesystem.
+//
+// Backups are made from local fs.
+// Data is restored from backups to local fs.
+type FS struct {
+	// Dir is a path to local directory to work with.
+	Dir string
+
+	// MaxBytesPerSecond is the maximum bandwidth usage during backups or restores.
+	MaxBytesPerSecond int
+
+	bl *bandwidthLimiter
+}
+
+// Init initializes fs
+func (fs *FS) Init() error {
+	if fs.MaxBytesPerSecond > 0 {
+		fs.bl = newBandwidthLimiter(fs.MaxBytesPerSecond)
+	}
+	return nil
+}
+
+// String returns user-readable representation for the fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fslocal %q", fs.Dir)
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			// Return empty part list for non-existing directory.
+			// The directory will be created later.
+			return nil, nil
+		}
+		return nil, err
+	}
+	files, err := fscommon.AppendFiles(nil, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var parts []common.Part
+	dir += "/"
+	for _, file := range files {
+		if !strings.HasPrefix(file, dir) {
+			logger.Panicf("BUG: unexpected prefix for file %q; want %q", file, dir)
+		}
+		fi, err := os.Stat(file)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat %q: %s", file, err)
+		}
+		path := file[len(dir):]
+		size := uint64(fi.Size())
+		if size == 0 {
+			parts = append(parts, common.Part{
+				Path:   path,
+				Offset: 0,
+				Size:   0,
+			})
+			continue
+		}
+		offset := uint64(0)
+		for offset < size {
+			n := size - offset
+			if n > common.MaxPartSize {
+				n = common.MaxPartSize
+			}
+			parts = append(parts, common.Part{
+				Path:       path,
+				FileSize:   size,
+				Offset:     offset,
+				Size:       n,
+				ActualSize: n,
+			})
+			offset += n
+		}
+	}
+	return parts, nil
+}
+
+// NewReadCloser returns io.ReadCloser for the given part p located in fs.
+func (fs *FS) NewReadCloser(p common.Part) (io.ReadCloser, error) {
+	path := fs.path(p)
+	r, err := filestream.OpenReaderAt(path, int64(p.Offset), true)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open %q at %q: %s", p.Path, fs.Dir, err)
+	}
+	lrc := &limitedReadCloser{
+		r: r,
+		n: p.Size,
+	}
+	if fs.bl == nil {
+		return lrc, nil
+	}
+	blrc := fs.bl.NewReadCloser(lrc)
+	return blrc, nil
+}
+
+// NewWriteCloser returns io.WriteCloser for the given part p located in fs.
+func (fs *FS) NewWriteCloser(p common.Part) (io.WriteCloser, error) {
+	path := fs.path(p)
+	if err := fs.mkdirAll(path); err != nil {
+		return nil, err
+	}
+	w, err := filestream.OpenWriterAt(path, int64(p.Offset), true)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open writer for %q at offset %d: %s", path, p.Offset, err)
+	}
+	wc := &writeCloser{
+		w:    w,
+		n:    p.Size,
+		path: path,
+	}
+	if fs.bl == nil {
+		return wc, nil
+	}
+	blwc := fs.bl.NewWriteCloser(wc)
+	return blwc, nil
+}
+
+// DeletePath deletes the given path from fs and returns the size
+// for the deleted file.
+func (fs *FS) DeletePath(path string) (uint64, error) {
+	p := common.Part{
+		Path: path,
+	}
+	fullPath := fs.path(p)
+	f, err := os.Open(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// The file could be deleted earlier via symlink.
+			return 0, nil
+		}
+		return 0, fmt.Errorf("cannot open %q at %q: %s", path, fullPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return 0, fmt.Errorf("cannot stat %q at %q: %s", path, fullPath, err)
+	}
+	size := uint64(fi.Size())
+	if err := os.Remove(fullPath); err != nil {
+		return 0, fmt.Errorf("cannot remove %q: %s", fullPath, err)
+	}
+	return size, nil
+}
+
+// RemoveEmptyDirs recursively removes all the empty directories in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	return fscommon.RemoveEmptyDirs(fs.Dir)
+}
+
+func (fs *FS) mkdirAll(filePath string) error {
+	dir := filepath.Dir(filePath)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return fmt.Errorf("cannot create directory %q: %s", dir, err)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	dir := fs.Dir
+	for strings.HasSuffix(dir, "/") {
+		dir = dir[:len(dir)-1]
+	}
+	return fs.Dir + "/" + p.Path
+}
+
+type limitedReadCloser struct {
+	r *filestream.Reader
+	n uint64
+}
+
+func (lrc *limitedReadCloser) Read(p []byte) (int, error) {
+	if lrc.n == 0 {
+		return 0, io.EOF
+	}
+	if uint64(len(p)) > lrc.n {
+		p = p[:lrc.n]
+	}
+	n, err := lrc.r.Read(p)
+	if n > len(p) {
+		return n, fmt.Errorf("too much data read; got %d bytes; want %d bytes", n, len(p))
+	}
+	lrc.n -= uint64(n)
+	return n, err
+}
+
+func (lrc *limitedReadCloser) Close() error {
+	lrc.r.MustClose()
+	return nil
+}
+
+type writeCloser struct {
+	w    *filestream.Writer
+	n    uint64
+	path string
+}
+
+func (wc *writeCloser) Write(p []byte) (int, error) {
+	n, err := wc.w.Write(p)
+	if uint64(n) > wc.n {
+		return n, fmt.Errorf("too much data written; got %d bytes; want %d bytes", n, wc.n)
+	}
+	wc.n -= uint64(n)
+	return n, err
+}
+
+func (wc *writeCloser) Close() error {
+	wc.w.MustClose()
+	if wc.n != 0 {
+		return fmt.Errorf("missing data writes for %d bytes", wc.n)
+	}
+	return fscommon.FsyncFile(wc.path)
+}

lib/backup/fsnil/fsnil.go

@@ -0,0 +1,20 @@
+package fsnil
+
+import (
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+// FS represents nil remote filesystem.
+type FS struct{}
+
+// String returns human-readable string representation for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fsnil")
+}
+
+// ListParts returns all the parts from fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	return nil, nil
+}

lib/backup/fsremote/fsremote.go

@@ -0,0 +1,190 @@
+package fsremote
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represents remote filesystem.
+//
+// Backups are uploaded there.
+// Data is downloaded from there during restore.
+type FS struct {
+	// Dir is a path to remote directory with backup data.
+	Dir string
+}
+
+// String returns human-readable string representation for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("fsremote %q", fs.Dir)
+}
+
+// ListParts returns all the parts from fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			// Return empty part list for non-existing directory.
+			// The directory will be created later.
+			return nil, nil
+		}
+		return nil, err
+	}
+	files, err := fscommon.AppendFiles(nil, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var parts []common.Part
+	dir += "/"
+	for _, file := range files {
+		if !strings.HasPrefix(file, dir) {
+			logger.Panicf("BUG: unexpected prefix for file %q; want %q", file, dir)
+		}
+		var p common.Part
+		if !p.ParseFromRemotePath(file[len(dir):]) {
+			logger.Infof("skipping unknown file %s", file)
+			continue
+		}
+		// Check for correct part size.
+		fi, err := os.Stat(file)
+		if err != nil {
+			return nil, fmt.Errorf("cannot stat file %q for part %q: %s", file, p.Path, err)
+		}
+		p.ActualSize = uint64(fi.Size())
+		parts = append(parts, p)
+	}
+	return parts, nil
+}
+
+// DeletePart deletes the given part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	path := fs.path(p)
+	if err := os.Remove(path); err != nil {
+		return fmt.Errorf("cannot remove %q: %s", path, err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes all the empty directories in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	return fscommon.RemoveEmptyDirs(fs.Dir)
+}
+
+// CopyPart copies the part p from srcFS to fs.
+//
+// srcFS must have *FS type.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be fsremote", srcFS, fs)
+	}
+	srcPath := src.path(p)
+	dstPath := fs.path(p)
+	if err := fs.mkdirAll(dstPath); err != nil {
+		return err
+	}
+	// Attempt to create hardlink from srcPath to dstPath.
+	if err := os.Link(srcPath, dstPath); err == nil {
+		return fscommon.FsyncFile(dstPath)
+	}
+
+	// Cannot create hardlink. Just copy file contents
+	srcFile, err := os.Open(srcPath)
+	if err != nil {
+		return fmt.Errorf("cannot open file %q: %s", srcPath, err)
+	}
+	dstFile, err := os.Create(dstPath)
+	if err != nil {
+		_ = srcFile.Close()
+		return fmt.Errorf("cannot create file %q: %s", dstPath, err)
+	}
+	n, err := io.Copy(dstFile, srcFile)
+	if err1 := dstFile.Close(); err1 != nil {
+		err = err1
+	}
+	if err1 := srcFile.Close(); err1 != nil {
+		err = err1
+	}
+	if err != nil {
+		_ = os.RemoveAll(dstPath)
+		return err
+	}
+	if uint64(n) != p.Size {
+		_ = os.RemoveAll(dstPath)
+		return fmt.Errorf("unexpected number of bytes copied from %q to %q; got %d bytes; want %d bytes", srcPath, dstPath, n, p.Size)
+	}
+	if err := fscommon.FsyncFile(dstPath); err != nil {
+		_ = os.RemoveAll(dstPath)
+		return err
+	}
+	return nil
+}
+
+// DownloadPart download part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	path := fs.path(p)
+	r, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("cannot open %q: %s", path, err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download data from %q: %s", path, err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q; got %d bytes; want %d bytes", path, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	path := fs.path(p)
+	if err := fs.mkdirAll(path); err != nil {
+		return err
+	}
+	w, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("cannot create file %q: %s", path, err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := w.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("cannot upload data to %q: %s", path, err)
+	}
+	if uint64(n) != p.Size {
+		_ = os.RemoveAll(path)
+		return fmt.Errorf("wrong data size uploaded to %q; got %d bytes; want %d bytes", path, n, p.Size)
+	}
+	if err := fscommon.FsyncFile(path); err != nil {
+		_ = os.RemoveAll(path)
+		return err
+	}
+	return nil
+}
+
+func (fs *FS) mkdirAll(filePath string) error {
+	dir := filepath.Dir(filePath)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return fmt.Errorf("cannot create directory %q: %s", dir, err)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	return p.RemotePath(fs.Dir)
+}

lib/backup/gcsremote/gcs.go

@@ -0,0 +1,189 @@
+package gcsremote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+// FS represents filesystem for backups in GCS.
+//
+// Init must be called before calling other FS methods.
+type FS struct {
+	// Path to GCP credentials file.
+	//
+	// Default credentials are used if empty.
+	CredsFilePath string
+
+	// GCS bucket to use.
+	Bucket string
+
+	// Directory in the bucket to write to.
+	Dir string
+
+	bkt *storage.BucketHandle
+}
+
+// Init initializes fs.
+func (fs *FS) Init() error {
+	if fs.bkt != nil {
+		logger.Panicf("BUG: fs.Init has been already called")
+	}
+	for strings.HasPrefix(fs.Dir, "/") {
+		fs.Dir = fs.Dir[1:]
+	}
+	if !strings.HasSuffix(fs.Dir, "/") {
+		fs.Dir += "/"
+	}
+	ctx := context.Background()
+	var client *storage.Client
+	if len(fs.CredsFilePath) > 0 {
+		creds := option.WithCredentialsFile(fs.CredsFilePath)
+		c, err := storage.NewClient(ctx, creds)
+		if err != nil {
+			return fmt.Errorf("cannot create gcs client with credsFile %q: %s", fs.CredsFilePath, err)
+		}
+		client = c
+	} else {
+		c, err := storage.NewClient(ctx)
+		if err != nil {
+			return fmt.Errorf("cannot create default gcs client: %q", err)
+		}
+		client = c
+	}
+	fs.bkt = client.Bucket(fs.Bucket)
+	return nil
+}
+
+// String returns human-readable description for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("GCS{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)
+}
+
+// selectAttrs contains object attributes to select in ListParts.
+var selectAttrs = []string{
+	"Name",
+	"Size",
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	ctx := context.Background()
+	q := &storage.Query{
+		Prefix: dir,
+	}
+	if err := q.SetAttrSelection(selectAttrs); err != nil {
+		return nil, fmt.Errorf("error in SetAttrSelection: %s", err)
+	}
+	it := fs.bkt.Objects(ctx, q)
+	var parts []common.Part
+	for {
+		attr, err := it.Next()
+		if err == iterator.Done {
+			return parts, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error when iterating objects at %q: %s", dir, err)
+		}
+		file := attr.Name
+		if !strings.HasPrefix(file, dir) {
+			return nil, fmt.Errorf("unexpected prefix for gcs key %q; want %q", file, dir)
+		}
+		var p common.Part
+		if !p.ParseFromRemotePath(file[len(dir):]) {
+			logger.Infof("skipping unknown object %q", file)
+			continue
+		}
+		p.ActualSize = uint64(attr.Size)
+		parts = append(parts, p)
+	}
+}
+
+// DeletePart deletes part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	if err := o.Delete(ctx); err != nil {
+		return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes empty dirs in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	// GCS has no directories, so nothing to remove.
+	return nil
+}
+
+// CopyPart copies p from srcFS to fs.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be GCS", srcFS, fs)
+	}
+	srcObj := src.object(p)
+	dstObj := fs.object(p)
+
+	copier := dstObj.CopierFrom(srcObj)
+	ctx := context.Background()
+	attr, err := copier.Run(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot copy %q from %s to %s: %s", p.Path, src, fs, err)
+	}
+	if uint64(attr.Size) != p.Size {
+		return fmt.Errorf("unexpected %q size after copying from %s to %s; got %d bytes; want %d bytes", p.Path, src, fs, attr.Size, p.Size)
+	}
+	return nil
+}
+
+// DownloadPart downloads part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	r, err := o.NewReader(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot open reader for %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download %q from at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads part p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	o := fs.object(p)
+	ctx := context.Background()
+	w := o.NewWriter(ctx)
+	n, err := io.Copy(w, r)
+	if err1 := w.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot upload data to %q at %s (remote path %q): %s", p.Path, fs, o.ObjectName(), err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size uploaded to %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+func (fs *FS) object(p common.Part) *storage.ObjectHandle {
+	path := p.RemotePath(fs.Dir)
+	return fs.bkt.Object(path)
+}

lib/backup/s3remote/s3.go

@@ -0,0 +1,236 @@
+package s3remote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// FS represents filesystem for backups in S3.
+//
+// Init must be called before calling other FS methods.
+type FS struct {
+	// Path to S3 credentials file.
+	CredsFilePath string
+
+	// Path to S3 configs file.
+	ConfigFilePath string
+
+	// GCS bucket to use.
+	Bucket string
+
+	// Directory in the bucket to write to.
+	Dir string
+
+	// Set for using S3-compatible enpoint such as MinIO etc.
+	CustomEndpoint string
+
+	// The name of S3 config profile to use.
+	ProfileName string
+
+	s3       *s3.S3
+	uploader *s3manager.Uploader
+}
+
+// Init initializes fs.
+func (fs *FS) Init() error {
+	if fs.s3 != nil {
+		logger.Panicf("BUG: Init is already called")
+	}
+	for strings.HasPrefix(fs.Dir, "/") {
+		fs.Dir = fs.Dir[1:]
+	}
+	if !strings.HasSuffix(fs.Dir, "/") {
+		fs.Dir += "/"
+	}
+	opts := session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Profile:           fs.ProfileName,
+	}
+	if len(fs.CredsFilePath) > 0 {
+		opts.SharedConfigFiles = []string{
+			fs.ConfigFilePath,
+			fs.CredsFilePath,
+		}
+	}
+	sess, err := session.NewSessionWithOptions(opts)
+	if err != nil {
+		return fmt.Errorf("cannot create S3 session: %s", err)
+	}
+
+	if len(fs.CustomEndpoint) > 0 {
+		// Use provided custom endpoint for S3
+		logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint)
+		sess.Config.WithEndpoint(fs.CustomEndpoint)
+
+		// Disable prefixing endpoint with bucket name
+		sess.Config.WithS3ForcePathStyle(true)
+	} else {
+		// Determine bucket region.
+		ctx := context.Background()
+		region, err := s3manager.GetBucketRegion(ctx, sess, fs.Bucket, "")
+		if err != nil {
+			return fmt.Errorf("cannot determine region for bucket %q: %s", fs.Bucket, err)
+		}
+		sess.Config.WithRegion(region)
+		logger.Infof("bucket %q is stored at region %q; switching to this region", fs.Bucket, region)
+	}
+
+	fs.s3 = s3.New(sess)
+	fs.uploader = s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
+		// We manage upload concurrency by ourselves.
+		u.Concurrency = 1
+	})
+	return nil
+}
+
+// String returns human-readable description for fs.
+func (fs *FS) String() string {
+	return fmt.Sprintf("S3{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)
+}
+
+// ListParts returns all the parts for fs.
+func (fs *FS) ListParts() ([]common.Part, error) {
+	dir := fs.Dir
+	input := &s3.ListObjectsV2Input{
+		Bucket: aws.String(fs.Bucket),
+		Prefix: aws.String(dir),
+	}
+	var errOuter error
+	var parts []common.Part
+	err := fs.s3.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, o := range page.Contents {
+			file := *o.Key
+			if !strings.HasPrefix(file, dir) {
+				errOuter = fmt.Errorf("unexpected prefix for s3 key %q; want %q", file, dir)
+				return false
+			}
+			var p common.Part
+			if !p.ParseFromRemotePath(file[len(dir):]) {
+				logger.Infof("skipping unknown object %q", file)
+				continue
+			}
+			p.ActualSize = uint64(*o.Size)
+			parts = append(parts, p)
+		}
+		return !lastPage
+	})
+	if errOuter != nil && err == nil {
+		err = errOuter
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error when listing s3 objects inside dir %q: %s", dir, err)
+	}
+	return parts, nil
+}
+
+// DeletePart deletes part p from fs.
+func (fs *FS) DeletePart(p common.Part) error {
+	path := fs.path(p)
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	_, err := fs.s3.DeleteObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	return nil
+}
+
+// RemoveEmptyDirs recursively removes empty dirs in fs.
+func (fs *FS) RemoveEmptyDirs() error {
+	// S3 has no directories, so nothing to remove.
+	return nil
+}
+
+// CopyPart copies p from srcFS to fs.
+func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
+	src, ok := srcFS.(*FS)
+	if !ok {
+		return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be S3", srcFS, fs)
+	}
+	srcPath := src.path(p)
+	dstPath := fs.path(p)
+	copySource := fmt.Sprintf("/%s/%s", src.Bucket, srcPath)
+
+	input := &s3.CopyObjectInput{
+		Bucket:     aws.String(fs.Bucket),
+		CopySource: aws.String(copySource),
+		Key:        aws.String(dstPath),
+	}
+	_, err := fs.s3.CopyObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot copy %q from %s to %s (copySource %q): %s", p.Path, src, fs, copySource, err)
+	}
+	return nil
+}
+
+// DownloadPart downloads part p from fs to w.
+func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
+	path := fs.path(p)
+	input := &s3.GetObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	o, err := fs.s3.GetObject(input)
+	if err != nil {
+		return fmt.Errorf("cannot open %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	r := o.Body
+	n, err := io.Copy(w, r)
+	if err1 := r.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	if err != nil {
+		return fmt.Errorf("cannot download %q from at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	if uint64(n) != p.Size {
+		return fmt.Errorf("wrong data size downloaded from %q at %s; got %d bytes; want %d bytes", p.Path, fs, n, p.Size)
+	}
+	return nil
+}
+
+// UploadPart uploads part p from r to fs.
+func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
+	path := fs.path(p)
+	sr := &statReader{
+		r: r,
+	}
+	input := &s3manager.UploadInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+		Body:   sr,
+	}
+	_, err := fs.uploader.Upload(input)
+	if err != nil {
+		return fmt.Errorf("cannot upoad data to %q at %s (remote path %q): %s", p.Path, fs, path, err)
+	}
+	if uint64(sr.size) != p.Size {
+		return fmt.Errorf("wrong data size uploaded to %q at %s; got %d bytes; want %d bytes", p.Path, fs, sr.size, p.Size)
+	}
+	return nil
+}
+
+func (fs *FS) path(p common.Part) string {
+	return p.RemotePath(fs.Dir)
+}
+
+type statReader struct {
+	r    io.Reader
+	size int64
+}
+
+func (sr *statReader) Read(p []byte) (int, error) {
+	n, err := sr.r.Read(p)
+	sr.size += int64(n)
+	return n, err
+}

lib/bytesutil/bytesutil_test.go

@@ -1,6 +1,7 @@
 package bytesutil
 
 import (
+	"bytes"
 	"testing"
 )
 
@@ -20,3 +21,10 @@ func TestResize(t *testing.T) {
 		}
 	}
 }
+
+func TestToUnsafeString(t *testing.T) {
+	s := "str"
+	if !bytes.Equal([]byte("str"), ToUnsafeBytes(s)) {
+		t.Fatalf(`[]bytes(%s) doesnt equal to %s `, s, s)
+	}
+}

lib/decimal/decimal.go

@@ -3,6 +3,8 @@ package decimal
 import (
 	"math"
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fastnum"
 )
 
 // CalibrateScale calibrates a and b with the corresponding exponents ae, be
@@ -43,10 +45,6 @@ func CalibrateScale(a []int64, ae int16, b []int64, be int16) (e int16) {
 	}
 	if downExp > 0 {
 		for i, v := range b {
-			if v == vInfPos || v == vInfNeg {
-				// Special case for these values - do not touch them.
-				continue
-			}
 			adjExp := downExp
 			for adjExp > 0 {
 				v /= 10
@@ -84,17 +82,32 @@ func AppendDecimalToFloat(dst []float64, va []int64, e int16) []float64 {
 	// Extend dst capacity in order to eliminate memory allocations below.
 	dst = ExtendFloat64sCapacity(dst, len(va))
 
+	if fastnum.IsInt64Zeros(va) {
+		return fastnum.AppendFloat64Zeros(dst, len(va))
+	}
+	if e == 0 {
+		if fastnum.IsInt64Ones(va) {
+			return fastnum.AppendFloat64Ones(dst, len(va))
+		}
+		for _, v := range va {
+			f := float64(v)
+			dst = append(dst, f)
+		}
+		return dst
+	}
+
+	// increase conversion precision for negative exponents by dividing by e10
+	if e < 0 {
+		e10 := math.Pow10(int(-e))
+		for _, v := range va {
+			f := float64(v) / e10
+			dst = append(dst, f)
+		}
+		return dst
+	}
 	e10 := math.Pow10(int(e))
 	for _, v := range va {
-		// Manually inline ToFloat here for better performance
-		var f float64
-		if v == vInfPos {
-			f = infPos
-		} else if v == vInfNeg {
-			f = infNeg
-		} else {
-			f = float64(v) * e10
-		}
+		f := float64(v) * e10
 		dst = append(dst, f)
 	}
 	return dst
@@ -108,6 +121,14 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 	if len(src) == 0 {
 		return dst, 0
 	}
+	if fastnum.IsFloat64Zeros(src) {
+		dst = fastnum.AppendInt64Zeros(dst, len(src))
+		return dst, 0
+	}
+	if fastnum.IsFloat64Ones(src) {
+		dst = fastnum.AppendInt64Ones(dst, len(src))
+		return dst, 0
+	}
 
 	// Extend dst capacity in order to eliminate memory allocations below.
 	dst = ExtendInt64sCapacity(dst, len(src))
@@ -237,13 +258,12 @@ func maxUpExponent(v int64) int16 {
 
 // ToFloat returns f=v*10^e.
 func ToFloat(v int64, e int16) float64 {
-	if v == vInfPos {
-		return infPos
+	f := float64(v)
+	// increase conversion precision for negative exponents by dividing by e10
+	if e < 0 {
+		return f / math.Pow10(int(-e))
 	}
-	if v == vInfNeg {
-		return infNeg
-	}
-	return float64(v) * math.Pow10(int(e))
+	return f * math.Pow10(int(e))
 }
 
 const (
@@ -254,11 +274,6 @@ const (
 	vMin = -1<<63 + 1
 )
 
-var (
-	infPos = math.Inf(1)
-	infNeg = math.Inf(-1)
-)
-
 // FromFloat converts f to v*10^e.
 //
 // It tries minimizing v.
@@ -288,7 +303,7 @@ func FromFloat(f float64) (int64, int16) {
 }
 
 func fromFloatInf(f float64) (int64, int16) {
-	// Special case for Inf
+	// Limit infs by max and min values for int64
 	if math.IsInf(f, 1) {
 		return vInfPos, 0
 	}
@@ -343,7 +358,7 @@ func positiveFloatToDecimalSlow(f float64) (int64, int16) {
 			prec = 1e15
 		}
 		_, exp := math.Frexp(f)
-		scale = int16(float64(exp) * math.Ln2 / math.Ln10)
+		scale = int16(float64(exp) * (math.Ln2 / math.Ln10))
 		f *= math.Pow10(-int(scale))
 	}
 

lib/decimal/decimal_test.go

@@ -43,6 +43,9 @@ func TestPositiveFloatToDecimal(t *testing.T) {
 	f(1234567890123456789e-14, 1234567890123, -8)
 	f(1234567890123456789e-17, 12345678901234, -12)
 	f(1234567890123456789e-20, 1234567890123, -14)
+
+	f(0.000874957, 874957, -9)
+	f(0.001130435, 1130435, -9)
 }
 
 func TestAppendDecimalToFloat(t *testing.T) {
@@ -52,6 +55,16 @@ func TestAppendDecimalToFloat(t *testing.T) {
 	testAppendDecimalToFloat(t, []int64{0}, -10, []float64{0})
 	testAppendDecimalToFloat(t, []int64{-1, -10, 0, 100}, 2, []float64{-1e2, -1e3, 0, 1e4})
 	testAppendDecimalToFloat(t, []int64{-1, -10, 0, 100}, -2, []float64{-1e-2, -1e-1, 0, 1})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -5, []float64{8.74957, 1.130435e1})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -6, []float64{8.74957e-1, 1.130435})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -7, []float64{8.74957e-2, 1.130435e-1})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -8, []float64{8.74957e-3, 1.130435e-2})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -9, []float64{8.74957e-4, 1.130435e-3})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -10, []float64{8.74957e-5, 1.130435e-4})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -11, []float64{8.74957e-6, 1.130435e-5})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -12, []float64{8.74957e-7, 1.130435e-6})
+	testAppendDecimalToFloat(t, []int64{874957, 1130435}, -13, []float64{8.74957e-8, 1.130435e-7})
+	testAppendDecimalToFloat(t, []int64{vInfPos, vInfNeg, 1, 2}, 0, []float64{9.223372036854776e+18, -9.223372036854776e+18, 1, 2})
 }
 
 func testAppendDecimalToFloat(t *testing.T, va []int64, e int16, fExpected []float64) {
@@ -93,7 +106,7 @@ func TestCalibrateScale(t *testing.T) {
 	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{500, 100}, 0, 2, []int64{vInfPos, 1200}, []int64{500e2, 100e2}, 0)
 	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{500, 100}, 0, -2, []int64{vInfPos, 1200}, []int64{5, 1}, 0)
 	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{3500, 100}, 0, -3, []int64{vInfPos, 1200}, []int64{3, 0}, 0)
-	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{35, 1}, 0, 40, []int64{vInfPos, 0}, []int64{35e17, 1e17}, 23)
+	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{35, 1}, 0, 40, []int64{0, 0}, []int64{35e17, 1e17}, 23)
 	testCalibrateScale(t, []int64{vInfPos, 1200}, []int64{35, 1}, 40, 0, []int64{vInfPos, 1200}, []int64{0, 0}, 40)
 	testCalibrateScale(t, []int64{vInfNeg, 1200}, []int64{35, 1}, 35, -5, []int64{vInfNeg, 1200}, []int64{0, 0}, 35)
 	testCalibrateScale(t, []int64{vMax, vMin, 123}, []int64{100}, 0, 3, []int64{vMax, vMin, 123}, []int64{100e3}, 0)
@@ -159,6 +172,8 @@ func TestMaxUpExponent(t *testing.T) {
 		}
 	}
 
+	f(vInfPos, 0)
+	f(vInfNeg, 0)
 	f(0, 1024)
 	f(-1<<63, 0)
 	f((-1<<63)+1, 0)
@@ -206,6 +221,9 @@ func TestAppendFloatToDecimal(t *testing.T) {
 	// no-op
 	testAppendFloatToDecimal(t, []float64{}, nil, 0)
 	testAppendFloatToDecimal(t, []float64{0}, []int64{0}, 0)
+	testAppendFloatToDecimal(t, []float64{infPos, infNeg, 123}, []int64{vInfPos, vInfNeg, 123}, 0)
+	testAppendFloatToDecimal(t, []float64{infPos, infNeg, 123, 1e-4, 1e32}, []int64{92233, -92233, 0, 0, 1000000000000000000}, 14)
+	testAppendFloatToDecimal(t, []float64{float64(vInfPos), float64(vInfNeg), 123}, []int64{9223372036854775000, -9223372036854775000, 123}, 0)
 	testAppendFloatToDecimal(t, []float64{0, -0, 1, -1, 12345678, -123456789}, []int64{0, 0, 1, -1, 12345678, -123456789}, 0)
 
 	// upExp
@@ -323,11 +341,15 @@ func TestFloatToDecimalRoundtrip(t *testing.T) {
 	f(12.34567890125)
 	f(-1234567.8901256789)
 	f(15e18)
+	f(0.000874957)
+	f(0.001130435)
 
-	f(math.Inf(1))
-	f(math.Inf(-1))
-	f(1<<63 - 1)
-	f(-1 << 63)
+	f(2933434554455e245)
+	f(3439234258934e-245)
+	f(float64(vInfPos))
+	f(float64(vInfNeg))
+	f(infPos)
+	f(infNeg)
 
 	for i := 0; i < 1e4; i++ {
 		v := rand.NormFloat64()
@@ -351,9 +373,26 @@ func roundFloat(f float64, exp int) float64 {
 }
 
 func equalFloat(f1, f2 float64) bool {
+	f1 = adjustInf(f1)
+	f2 = adjustInf(f2)
 	if math.IsInf(f1, 0) {
 		return math.IsInf(f1, 1) == math.IsInf(f2, 1) || math.IsInf(f1, -1) == math.IsInf(f2, -1)
 	}
 	eps := math.Abs(f1 - f2)
 	return eps == 0 || eps*conversionPrecision < math.Abs(f1)+math.Abs(f2)
 }
+
+func adjustInf(f float64) float64 {
+	if f == float64(vInfPos) {
+		return infPos
+	}
+	if f == float64(vInfNeg) {
+		return infNeg
+	}
+	return f
+}
+
+var (
+	infPos = math.Inf(1)
+	infNeg = math.Inf(-1)
+)

lib/decimal/decimal_timing_test.go

@@ -8,17 +8,41 @@ import (
 )
 
 func BenchmarkAppendDecimalToFloat(b *testing.B) {
+	b.Run("RealFloat", func(b *testing.B) {
+		benchmarkAppendDecimalToFloat(b, testVA, vaScale)
+	})
+	b.Run("Integers", func(b *testing.B) {
+		benchmarkAppendDecimalToFloat(b, testIntegers, integersScale)
+	})
+	b.Run("Zeros", func(b *testing.B) {
+		benchmarkAppendDecimalToFloat(b, testZeros, 0)
+	})
+	b.Run("Ones", func(b *testing.B) {
+		benchmarkAppendDecimalToFloat(b, testOnes, 0)
+	})
+}
+
+func benchmarkAppendDecimalToFloat(b *testing.B, a []int64, scale int16) {
 	b.ReportAllocs()
-	b.SetBytes(int64(len(testVA)))
+	b.SetBytes(int64(len(a)))
 	b.RunParallel(func(pb *testing.PB) {
 		var fa []float64
 		for pb.Next() {
-			fa = AppendDecimalToFloat(fa[:0], testVA, 0)
+			fa = AppendDecimalToFloat(fa[:0], a, scale)
 			atomic.AddUint64(&Sink, uint64(len(fa)))
 		}
 	})
 }
 
+var testZeros = make([]int64, 8*1024)
+var testOnes = func() []int64 {
+	a := make([]int64, 8*1024)
+	for i := 0; i < len(a); i++ {
+		a[i] = 1
+	}
+	return a
+}()
+
 func BenchmarkAppendFloatToDecimal(b *testing.B) {
 	b.Run("RealFloat", func(b *testing.B) {
 		benchmarkAppendFloatToDecimal(b, testFAReal)
@@ -26,8 +50,23 @@ func BenchmarkAppendFloatToDecimal(b *testing.B) {
 	b.Run("Integers", func(b *testing.B) {
 		benchmarkAppendFloatToDecimal(b, testFAInteger)
 	})
+	b.Run("Zeros", func(b *testing.B) {
+		benchmarkAppendFloatToDecimal(b, testFZeros)
+	})
+	b.Run("Ones", func(b *testing.B) {
+		benchmarkAppendFloatToDecimal(b, testFOnes)
+	})
 }
 
+var testFZeros = make([]float64, 8*1024)
+var testFOnes = func() []float64 {
+	a := make([]float64, 8*1024)
+	for i := 0; i < len(a); i++ {
+		a[i] = 1
+	}
+	return a
+}()
+
 func benchmarkAppendFloatToDecimal(b *testing.B, fa []float64) {
 	b.ReportAllocs()
 	b.SetBytes(int64(len(fa)))
@@ -47,7 +86,7 @@ func benchmarkAppendFloatToDecimal(b *testing.B, fa []float64) {
 var testFAReal = func() []float64 {
 	fa := make([]float64, 8*1024)
 	for i := 0; i < len(fa); i++ {
-		fa[i] = rand.NormFloat64() * 1e6
+		fa[i] = rand.NormFloat64() * 1e-6
 	}
 	return fa
 }()
@@ -60,10 +99,8 @@ var testFAInteger = func() []float64 {
 	return fa
 }()
 
-var testVA = func() []int64 {
-	va, _ := AppendFloatToDecimal(nil, testFAReal)
-	return va
-}()
+var testVA, vaScale = AppendFloatToDecimal(nil, testFAReal)
+var testIntegers, integersScale = AppendFloatToDecimal(nil, testFAInteger)
 
 func BenchmarkFromFloat(b *testing.B) {
 	for _, f := range []float64{0, 1234, 12334345, 12343.4344, 123.45678901e12, 12.3454435e30} {

lib/encoding/encoding.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fastnum"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -201,6 +202,14 @@ func unmarshalInt64Array(dst []int64, src []byte, mt MarshalType, firstValue int
 		if len(src) > 0 {
 			return nil, fmt.Errorf("unexpected data left in const encoding: %d bytes", len(src))
 		}
+		if firstValue == 0 {
+			dst = fastnum.AppendInt64Zeros(dst, itemsCount)
+			return dst, nil
+		}
+		if firstValue == 1 {
+			dst = fastnum.AppendInt64Ones(dst, itemsCount)
+			return dst, nil
+		}
 		for itemsCount > 0 {
 			dst = append(dst, firstValue)
 			itemsCount--
@@ -267,6 +276,14 @@ func isConst(a []int64) bool {
 	if len(a) == 0 {
 		return false
 	}
+	if fastnum.IsInt64Zeros(a) {
+		// Fast path for array containing only zeros.
+		return true
+	}
+	if fastnum.IsInt64Ones(a) {
+		// Fast path for array containing only ones.
+		return true
+	}
 	v1 := a[0]
 	for _, v := range a {
 		if v != v1 {

lib/encoding/encoding_timing_test.go

@@ -32,7 +32,7 @@ func BenchmarkUnmarshalGaugeArray(b *testing.B) {
 		var dst []int64
 		var err error
 		for pb.Next() {
-			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledGaugeArray, MarshalTypeZSTDNearestDelta, 0, len(benchGaugeArray))
+			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledGaugeArray, MarshalTypeZSTDNearestDelta, benchGaugeArray[0], len(benchGaugeArray))
 			if err != nil {
 				panic(fmt.Errorf("cannot unmarshal gauge array: %s", err))
 			}
@@ -79,7 +79,7 @@ func BenchmarkUnmarshalDeltaConstArray(b *testing.B) {
 		var dst []int64
 		var err error
 		for pb.Next() {
-			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledDeltaConstArray, MarshalTypeDeltaConst, 0, len(benchDeltaConstArray))
+			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledDeltaConstArray, MarshalTypeDeltaConst, benchDeltaConstArray[0], len(benchDeltaConstArray))
 			if err != nil {
 				panic(fmt.Errorf("cannot unmarshal delta const array: %s", err))
 			}
@@ -126,7 +126,7 @@ func BenchmarkUnmarshalConstArray(b *testing.B) {
 		var dst []int64
 		var err error
 		for pb.Next() {
-			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledConstArray, MarshalTypeConst, 0, len(benchConstArray))
+			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledConstArray, MarshalTypeConst, benchConstArray[0], len(benchConstArray))
 			if err != nil {
 				panic(fmt.Errorf("cannot unmarshal const array: %s", err))
 			}
@@ -171,7 +171,7 @@ func BenchmarkUnmarshalZeroConstArray(b *testing.B) {
 		var dst []int64
 		var err error
 		for pb.Next() {
-			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledZeroConstArray, MarshalTypeConst, 0, len(benchZeroConstArray))
+			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledZeroConstArray, MarshalTypeConst, benchZeroConstArray[0], len(benchZeroConstArray))
 			if err != nil {
 				panic(fmt.Errorf("cannot unmarshal zero const array: %s", err))
 			}
@@ -210,7 +210,7 @@ func BenchmarkUnmarshalInt64Array(b *testing.B) {
 		var dst []int64
 		var err error
 		for pb.Next() {
-			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledInt64Array, benchMarshalType, 0, len(benchInt64Array))
+			dst, err = unmarshalInt64Array(dst[:0], benchMarshaledInt64Array, benchMarshalType, benchInt64Array[0], len(benchInt64Array))
 			if err != nil {
 				panic(fmt.Errorf("cannot unmarshal int64 array: %s", err))
 			}

lib/fastnum/fastnum.go

@@ -0,0 +1,144 @@
+package fastnum
+
+import (
+	"bytes"
+	"unsafe"
+)
+
+// AppendInt64Zeros appends items zeros to dst and returns the result.
+//
+// It is faster than the corresponding loop.
+func AppendInt64Zeros(dst []int64, items int) []int64 {
+	return appendInt64Data(dst, items, int64Zeros[:])
+}
+
+// AppendInt64Ones appends items ones to dst and returns the result.
+//
+// It is faster than the corresponding loop.
+func AppendInt64Ones(dst []int64, items int) []int64 {
+	return appendInt64Data(dst, items, int64Ones[:])
+}
+
+// AppendFloat64Zeros appends items zeros to dst and returns the result.
+//
+// It is faster than the corresponding loop.
+func AppendFloat64Zeros(dst []float64, items int) []float64 {
+	return appendFloat64Data(dst, items, float64Zeros[:])
+}
+
+// AppendFloat64Ones appends items ones to dst and returns the result.
+//
+// It is faster than the corresponding loop.
+func AppendFloat64Ones(dst []float64, items int) []float64 {
+	return appendFloat64Data(dst, items, float64Ones[:])
+}
+
+// IsInt64Zeros checks whether a contains only zeros.
+func IsInt64Zeros(a []int64) bool {
+	return isInt64Data(a, int64Zeros[:])
+}
+
+// IsInt64Ones checks whether a contains only ones.
+func IsInt64Ones(a []int64) bool {
+	return isInt64Data(a, int64Ones[:])
+}
+
+// IsFloat64Zeros checks whether a contains only zeros.
+func IsFloat64Zeros(a []float64) bool {
+	return isFloat64Data(a, float64Zeros[:])
+}
+
+// IsFloat64Ones checks whether a contains only ones.
+func IsFloat64Ones(a []float64) bool {
+	return isFloat64Data(a, float64Ones[:])
+}
+
+func appendInt64Data(dst []int64, items int, src []int64) []int64 {
+	for items > 0 {
+		n := len(src)
+		if n > items {
+			n = items
+		}
+		dst = append(dst, src[:n]...)
+		items -= n
+	}
+	return dst
+}
+
+func appendFloat64Data(dst []float64, items int, src []float64) []float64 {
+	for items > 0 {
+		n := len(src)
+		if n > items {
+			n = items
+		}
+		dst = append(dst, src[:n]...)
+		items -= n
+	}
+	return dst
+}
+
+func isInt64Data(a, data []int64) bool {
+	if len(a) == 0 {
+		return true
+	}
+	if len(data) != 8*1024 {
+		panic("len(data) must equal to 8*1024")
+	}
+	b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0]))
+	for len(a) > 0 {
+		n := len(data)
+		if n > len(a) {
+			n = len(a)
+		}
+		x := a[:n]
+		a = a[n:]
+		xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0]))
+		xbLen := len(x) * 8
+		if !bytes.Equal(xb[:xbLen], b[:xbLen]) {
+			return false
+		}
+	}
+	return true
+}
+
+func isFloat64Data(a, data []float64) bool {
+	if len(a) == 0 {
+		return true
+	}
+	if len(data) != 8*1024 {
+		panic("len(data) must equal to 8*1024")
+	}
+	b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0]))
+	for len(a) > 0 {
+		n := len(data)
+		if n > len(a) {
+			n = len(a)
+		}
+		x := a[:n]
+		a = a[n:]
+		xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0]))
+		xbLen := len(x) * 8
+		if !bytes.Equal(xb[:xbLen], b[:xbLen]) {
+			return false
+		}
+	}
+	return true
+}
+
+var (
+	int64Zeros [8 * 1024]int64
+	int64Ones  = func() (a [8 * 1024]int64) {
+		for i := 0; i < len(a); i++ {
+			a[i] = 1
+		}
+		return a
+	}()
+
+	float64Zeros [8 * 1024]float64
+	float64Ones  = func() (a [8 * 1024]float64) {
+		for i := 0; i < len(a); i++ {
+			a[i] = 1
+		}
+		return a
+	}()
+)

lib/fastnum/fastnum_test.go

@@ -0,0 +1,192 @@
+package fastnum
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestIsInt64Zeros(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := make([]int64, n)
+			if !IsInt64Zeros(a) {
+				t.Fatalf("IsInt64Zeros must return true")
+			}
+			if len(a) > 0 {
+				a[len(a)-1] = 1
+				if IsInt64Zeros(a) {
+					t.Fatalf("IsInt64Zeros must return false")
+				}
+			}
+		})
+	}
+}
+
+func TestIsInt64Ones(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := make([]int64, n)
+			for i := 0; i < n; i++ {
+				a[i] = 1
+			}
+			if !IsInt64Ones(a) {
+				t.Fatalf("IsInt64Ones must return true")
+			}
+			if len(a) > 0 {
+				a[len(a)-1] = 0
+				if IsInt64Ones(a) {
+					t.Fatalf("IsInt64Ones must return false")
+				}
+			}
+		})
+	}
+}
+
+func TestIsFloat64Zeros(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := make([]float64, n)
+			if !IsFloat64Zeros(a) {
+				t.Fatalf("IsInt64Zeros must return true")
+			}
+			if len(a) > 0 {
+				a[len(a)-1] = 1
+				if IsFloat64Zeros(a) {
+					t.Fatalf("IsInt64Zeros must return false")
+				}
+			}
+		})
+	}
+}
+
+func TestIsFloat64Ones(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := make([]float64, n)
+			for i := 0; i < n; i++ {
+				a[i] = 1
+			}
+			if !IsFloat64Ones(a) {
+				t.Fatalf("IsInt64Ones must return true")
+			}
+			if len(a) > 0 {
+				a[len(a)-1] = 0
+				if IsFloat64Ones(a) {
+					t.Fatalf("IsInt64Ones must return false")
+				}
+			}
+		})
+	}
+}
+
+func TestAppendInt64Zeros(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := AppendInt64Zeros(nil, n)
+			if len(a) != n {
+				t.Fatalf("unexpected len(a); got %d; want %d", len(a), n)
+			}
+			if !IsInt64Zeros(a) {
+				t.Fatalf("IsInt64Zeros must return true")
+			}
+
+			prefix := []int64{1, 2, 3}
+			a = AppendInt64Zeros(prefix, n)
+			if len(a) != len(prefix)+n {
+				t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n)
+			}
+			for i := 0; i < len(prefix); i++ {
+				if a[i] != prefix[i] {
+					t.Fatalf("unexpected prefix[%d]; got %d; want %d", i, a[i], prefix[i])
+				}
+			}
+			if !IsInt64Zeros(a[len(prefix):]) {
+				t.Fatalf("IsInt64Zeros for prefixed a must return true")
+			}
+		})
+	}
+}
+
+func TestAppendInt64Ones(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := AppendInt64Ones(nil, n)
+			if len(a) != n {
+				t.Fatalf("unexpected len(a); got %d; want %d", len(a), n)
+			}
+			if !IsInt64Ones(a) {
+				t.Fatalf("IsInt64Ones must return true")
+			}
+
+			prefix := []int64{1, 2, 3}
+			a = AppendInt64Ones(prefix, n)
+			if len(a) != len(prefix)+n {
+				t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n)
+			}
+			for i := 0; i < len(prefix); i++ {
+				if a[i] != prefix[i] {
+					t.Fatalf("unexpected prefix[%d]; got %d; want %d", i, a[i], prefix[i])
+				}
+			}
+			if !IsInt64Ones(a[len(prefix):]) {
+				t.Fatalf("IsInt64Ones for prefixed a must return true")
+			}
+		})
+	}
+}
+
+func TestAppendFloat64Zeros(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := AppendFloat64Zeros(nil, n)
+			if len(a) != n {
+				t.Fatalf("unexpected len(a); got %d; want %d", len(a), n)
+			}
+			if !IsFloat64Zeros(a) {
+				t.Fatalf("IsFloat64Zeros must return true")
+			}
+
+			prefix := []float64{1, 2, 3}
+			a = AppendFloat64Zeros(prefix, n)
+			if len(a) != len(prefix)+n {
+				t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n)
+			}
+			for i := 0; i < len(prefix); i++ {
+				if a[i] != prefix[i] {
+					t.Fatalf("unexpected prefix[%d]; got %f; want %f", i, a[i], prefix[i])
+				}
+			}
+			if !IsFloat64Zeros(a[len(prefix):]) {
+				t.Fatalf("IsFloat64Zeros for prefixed a must return true")
+			}
+		})
+	}
+}
+
+func TestAppendFloat64Ones(t *testing.T) {
+	for _, n := range []int{0, 1, 10, 100, 1000, 1e4, 1e5, 8*1024 + 1} {
+		t.Run(fmt.Sprintf("%d_items", n), func(t *testing.T) {
+			a := AppendFloat64Ones(nil, n)
+			if len(a) != n {
+				t.Fatalf("unexpected len(a); got %d; want %d", len(a), n)
+			}
+			if !IsFloat64Ones(a) {
+				t.Fatalf("IsFloat64Ones must return true")
+			}
+
+			prefix := []float64{1, 2, 3}
+			a = AppendFloat64Ones(prefix, n)
+			if len(a) != len(prefix)+n {
+				t.Fatalf("unexpected len(a) with prefix; got %d; want %d", len(a), len(prefix)+n)
+			}
+			for i := 0; i < len(prefix); i++ {
+				if a[i] != prefix[i] {
+					t.Fatalf("unexpected prefix[%d]; got %f; want %f", i, a[i], prefix[i])
+				}
+			}
+			if !IsFloat64Ones(a[len(prefix):]) {
+				t.Fatalf("IsFloat64Ones for prefixed a must return true")
+			}
+		})
+	}
+}

lib/filestream/filestream.go

@@ -3,6 +3,7 @@ package filestream
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"sync"
 
@@ -51,6 +52,26 @@ type Reader struct {
 	st streamTracker
 }
 
+// OpenReaderAt opens the file at the given path in nocache mode at the given offset.
+//
+// If nocache is set, then the reader doesn't pollute OS page cache.
+func OpenReaderAt(path string, offset int64, nocache bool) (*Reader, error) {
+	r, err := Open(path, nocache)
+	if err != nil {
+		return nil, err
+	}
+	n, err := r.f.Seek(offset, io.SeekStart)
+	if err != nil {
+		r.MustClose()
+		return nil, fmt.Errorf("cannot seek to offset=%d for %q: %s", offset, path, err)
+	}
+	if n != offset {
+		r.MustClose()
+		return nil, fmt.Errorf("invalid seek offset for %q; got %d; want %d", path, n, offset)
+	}
+	return r, nil
+}
+
 // Open opens the file from the given path in nocache mode.
 //
 // If nocache is set, then the reader doesn't pollute OS page cache.
@@ -143,6 +164,28 @@ type Writer struct {
 	st streamTracker
 }
 
+// OpenWriterAt opens the file at path in nocache mode for writing at the given offset.
+//
+// The file at path is created if it is missing.
+//
+// If nocache is set, the writer doesn't pollute OS page cache.
+func OpenWriterAt(path string, offset int64, nocache bool) (*Writer, error) {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open %q: %s", path, err)
+	}
+	n, err := f.Seek(offset, io.SeekStart)
+	if err != nil {
+		_ = f.Close()
+		return nil, fmt.Errorf("cannot seek to offset=%d in %q: %s", offset, path, err)
+	}
+	if n != offset {
+		_ = f.Close()
+		return nil, fmt.Errorf("invalid seek offset for %q; got %d; want %d", path, n, offset)
+	}
+	return newWriter(f, nocache), nil
+}
+
 // Create creates the file for the given path in nocache mode.
 //
 // If nocache is set, the writer doesn't pollute OS page cache.
@@ -151,6 +194,10 @@ func Create(path string, nocache bool) (*Writer, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot create file %q: %s", path, err)
 	}
+	return newWriter(f, nocache), nil
+}
+
+func newWriter(f *os.File, nocache bool) *Writer {
 	w := &Writer{
 		f:  f,
 		bw: getBufioWriter(f),
@@ -159,7 +206,7 @@ func Create(path string, nocache bool) (*Writer, error) {
 		w.st.fd = f.Fd()
 	}
 	writersCount.Inc()
-	return w, nil
+	return w
 }
 
 // MustClose syncs the underlying file to storage and then closes it.

lib/fs/dir_remover.go

@@ -11,12 +11,13 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func mustRemoveAll(path string) bool {
+func mustRemoveAll(path string, done func()) bool {
 	err := os.RemoveAll(path)
 	if err == nil {
 		// Make sure the parent directory doesn't contain references
 		// to the current directory.
 		mustSyncParentDirIfExists(path)
+		done()
 		return true
 	}
 	if !isTemporaryNFSError(err) {
@@ -26,8 +27,12 @@ func mustRemoveAll(path string) bool {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
 	// Schedule for later directory removal.
 	nfsDirRemoveFailedAttempts.Inc()
+	w := &removeDirWork{
+		path: path,
+		done: done,
+	}
 	select {
-	case removeDirCh <- path:
+	case removeDirCh <- w:
 	default:
 		logger.Panicf("FATAL: cannot schedule %s for removal, since the removal queue is full (%d entries)", path, cap(removeDirCh))
 	}
@@ -36,16 +41,21 @@ func mustRemoveAll(path string) bool {
 
 var nfsDirRemoveFailedAttempts = metrics.NewCounter(`vm_nfs_dir_remove_failed_attempts_total`)
 
-var removeDirCh = make(chan string, 1024)
+type removeDirWork struct {
+	path string
+	done func()
+}
+
+var removeDirCh = make(chan *removeDirWork, 1024)
 
 func dirRemover() {
 	const minSleepTime = 100 * time.Millisecond
 	const maxSleepTime = time.Second
 	sleepTime := minSleepTime
 	for {
-		var path string
+		var w *removeDirWork
 		select {
-		case path = <-removeDirCh:
+		case w = <-removeDirCh:
 		default:
 			if atomic.LoadUint64(&stopDirRemover) != 0 {
 				return
@@ -53,7 +63,7 @@ func dirRemover() {
 			time.Sleep(minSleepTime)
 			continue
 		}
-		if mustRemoveAll(path) {
+		if mustRemoveAll(w.path, w.done) {
 			sleepTime = minSleepTime
 			continue
 		}
@@ -67,7 +77,7 @@ func dirRemover() {
 		if sleepTime < maxSleepTime {
 			sleepTime *= 2
 		} else {
-			logger.Errorf("failed to remove directory %q due to NFS lock; retrying later", path)
+			logger.Errorf("failed to remove directory %q due to NFS lock; retrying later", w.path)
 		}
 	}
 }

lib/fs/fs.go

@@ -174,7 +174,7 @@ func mkdirSync(path string) error {
 	return nil
 }
 
-// RemoveDirContents removes all the contents of the given dir it it exists.
+// RemoveDirContents removes all the contents of the given dir if it exists.
 //
 // It doesn't remove the dir itself, so the dir may be mounted
 // to a separate partition.
@@ -246,7 +246,17 @@ func mustSyncParentDirIfExists(path string) {
 //
 // It properly handles NFS issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61 .
 func MustRemoveAll(path string) {
-	_ = mustRemoveAll(path)
+	_ = mustRemoveAll(path, func() {})
+}
+
+// MustRemoveAllWithDoneCallback removes path with all the contents.
+//
+// done is called after the path is successfully removed.
+//
+// done may be called after the function returns for NFS path.
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/61.
+func MustRemoveAllWithDoneCallback(path string, done func()) {
+	_ = mustRemoveAll(path, done)
 }
 
 // HardLinkFiles makes hard links for all the files from srcDir in dstDir.