CHANGELOG.md: cut v1.44.0 release

This should prevent from double counting for time series at the time when it changes label.
The most common case is in K8S, which changes pod uid label with each new deployment.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@master
+        uses: actions/setup-go@main
         with:
-          go-version: 1.14
+          go-version: 1.15
         id: go
       - name: Dependencies
         env:
@@ -43,7 +43,20 @@ jobs:
           make victoria-metrics-arm64
           make vmutils
           GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=freebsd go build -mod=vendor ./app/vmagent
+          GOOS=freebsd go build -mod=vendor ./app/vmalert
+          GOOS=freebsd go build -mod=vendor ./app/vmbackup
+          GOOS=freebsd go build -mod=vendor ./app/vmrestore
+          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=openbsd go build -mod=vendor ./app/vmagent
+          GOOS=openbsd go build -mod=vendor ./app/vmalert
+          GOOS=openbsd go build -mod=vendor ./app/vmbackup
+          GOOS=openbsd go build -mod=vendor ./app/vmrestore
           GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/vmagent
+          GOOS=darwin go build -mod=vendor ./app/vmalert
+          GOOS=darwin go build -mod=vendor ./app/vmbackup
+          GOOS=darwin go build -mod=vendor ./app/vmrestore
       - name: Publish coverage
         uses: codecov/codecov-action@v1.0.6
         with:

CHANGELOG.md

@@ -0,0 +1,115 @@
+# tip
+
+
+# [v1.44.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.44.0)
+
+* FEATURE: automatically add missing label filters to binary operands as described at https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization .
+  This should improve performance for queries with missing label filters in binary operands. For example, the following query should work faster now, because it shouldn't
+  fetch and discard time series for `node_filesystem_files_free` metric without matching labels for the left side of the expression:
+  ```
+     node_filesystem_files{ host="$host", mountpoint="/" } - node_filesystem_files_free
+  ```
+* FEATURE: vmagent: add Docker Swarm service discovery (aka [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/656
+* FEATURE: add ability to export data in CSV format. See [these docs](https://victoriametrics.github.io/#how-to-export-csv-data) for details.
+* FEATURE: vmagent: add `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag for suppressing `duplicate scrape target` errors.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651 and https://victoriametrics.github.io/vmagent.html#troubleshooting .
+* FEATURE: vmagent: show original labels before relabeling is applied on `duplicate scrape target` errors. This should simplify debugging for incorrect relabeling.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
+* FEATURE: vmagent: `/targets` page now accepts optional `show_original_labels=1` query arg for displaying original labels for each target before relabeling is applied.
+  This should simplify debugging for target relabeling configs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
+* FEATURE: add `-finalMergeDelay` command-line flag for configuring the delay before final merge for per-month partitions.
+  The final merge is started after no new data is ingested into per-month partition during `-finalMergeDelay`.
+* FEATURE: add `vm_rows_added_to_storage_total` metric, which shows the total number of rows added to storage since app start.
+  The `sum(rate(vm_rows_added_to_storage_total))` can be smaller than `sum(rate(vm_rows_inserted_total))` if certain metrics are dropped
+  due to [relabeling](https://victoriametrics.github.io/#relabeling). The `sum(rate(vm_rows_added_to_storage_total))` can be bigger
+  than `sum(rate(vm_rows_inserted_total))` if [replication](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#replication-and-data-safety) is enabled.
+* FEATURE: keep metric name after applying [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) functions, which don't change time series meaning.
+  The list of such functions:
+   * `keep_last_value`
+   * `keep_next_value`
+   * `interpolate`
+   * `running_min`
+   * `running_max`
+   * `running_avg`
+   * `range_min`
+   * `range_max`
+   * `range_avg`
+   * `range_first`
+   * `range_last`
+   * `range_quantile`
+   * `smooth_exponential`
+   * `ceil`
+   * `floor`
+   * `round`
+   * `clamp_min`
+   * `clamp_max`
+   * `max_over_time`
+   * `min_over_time`
+   * `avg_over_time`
+   * `quantile_over_time`
+   * `mode_over_time`
+   * `geomean_over_time`
+   * `holt_winters`
+   * `predict_linear`
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674
+
+* BUGFIX: properly handle stale time series after K8S deployment. Previously such time series could be double-counted.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
+* BUGFIX: return a single time series at max from `absent()` function like Prometheus does.
+* BUGFIX: vmalert: accept days, weeks and years in `for: ` part of config like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817
+* BUGFIX: fix `mode_over_time(m[d])` calculations. Previously the function could return incorrect results.
+
+
+# [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)
+
+* FEATURE: reduce CPU usage for repeated queries over sliding time window when no new time series are added to the database.
+  Typical use cases: repeated evaluation of alerting rules in [vmalert](https://victoriametrics.github.io/vmalert.html) or dashboard auto-refresh in Grafana.
+* FEATURE: vmagent: add OpenStack service discovery aka [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728 .
+* FEATURE: vmalert: make `-maxIdleConnections` configurable for datasource HTTP client. This option can be used for minimizing connection churn.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/795 .
+* FEATURE: add `-influx.maxLineSize` command-line flag for configuring the maximum size for a single Influx line during parsing.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/807
+
+* BUGFIX: properly handle `inf` values during [background merge of LSM parts](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  Previously `Inf` values could result in `NaN` values for adjancent samples in time series. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/805 .
+* BUGFIX: fill gaps on graphs for `range_*` and `running_*` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/806 .
+* BUGFIX: make a copy of label with new name during relabeling with `action: labelmap` in the same way as Prometheus does.
+  Previously the original label name has been replaced. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/812 .
+* BUGFIX: support parsing floating-point timestamp like Graphite Carbon does. Such timestmaps are truncated to seconds.
+
+
+# [v1.42.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.42.0)
+
+* FEATURE: use all the available CPU cores when accepting data via a single TCP connection
+  for [all the supported protocols](https://victoriametrics.github.io/#how-to-import-time-series-data).
+  Previously data ingested via a single TCP connection could use only a single CPU core. This could limit data ingestion performance.
+  The main benefit of this feature is that data can be imported at max speed via a single connection - there is no need to open multiple concurrent
+  connections to VictoriaMetrics or [vmagent](https://victoriametrics.github.io/vmagent.html) in order to achieve the maximum data ingestion speed.
+* FEATURE: cluster: improve performance for data ingestion path from `vminsert` to `vmstorage` nodes. The maximum data ingestion performance
+  for a single connection between `vminsert` and `vmstorage` node scales with the number of available CPU cores on `vmstorage` side.
+  This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/791 .
+* FEATURE: add ability to export / import data in native format via `/api/v1/export/native` and `/api/v1/import/native`.
+  This is the most optimized approach for data migration between VictoriaMetrics instances. Both single-node and cluster instances are supported.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/787#issuecomment-700632551 .
+* FEATURE: add `reduce_mem_usage` query option to `/api/v1/export` in order to reduce memory usage during data export / import.
+  See [these docs](https://victoriametrics.github.io/#how-to-export-data-in-json-line-format) for details.
+* FEATURE: improve performance for `/api/v1/series` handler when it returns big number of time series.
+* FEATURE: add `vm_merge_need_free_disk_space` metric, which can be used for estimating the number of deferred background data merges due to the lack of free disk space.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686 .
+* FEATURE: add OpenBSD support. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/785 .
+
+* BUGFIX: properly apply `-search.maxStalenessInterval` command-line flag value. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 .
+* BUGFIX: fix displaying data in Grafana tables. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 .
+* BUGFIX: do not adjust the number of detected CPU cores found at `/sys/devices/system/cpu/online`.
+  The adjustement was increasing the resulting GOMAXPROC by 1, which looked confusing to users.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-698595309 .
+* BUGFIX: vmagent: do not show `-remoteWrite.url` in initial logs if `-remoteWrite.showURL` isn't set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/773 .
+* BUGFIX: properly handle case when [/metrics/find](https://victoriametrics.github.io/#graphite-metrics-api-usage) finds both a leaf and a node for the given `query=prefix.*`.
+  In this case only the node must be returned with stripped dot in the end of id as carbonapi does.
+
+
+# Previous releases
+
+See [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).

README.md

@@ -77,7 +77,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
     if `-graphiteListenAddr` is set.
   * [OpenTSDB put message](#sending-data-via-telnet-put-protocol) if `-opentsdbListenAddr` is set.
   * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests) if `-opentsdbHTTPListenAddr` is set.
-  * [/api/v1/import](#how-to-import-time-series-data).
+  * [JSON line format](#how-to-import-data-in-json-line-format).
+  * [Native binary format](#how-to-import-data-in-native-format).
+  * [Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
   * [Arbitrary CSV data](#how-to-import-csv-data).
 * Supports metrics' relabeling. See [these docs](#relabeling) for details.
 * Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various Enterprise workloads.
@@ -99,9 +101,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
 * [How to send data from Graphite-compatible agents such as StatsD](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
 * [Querying Graphite data](#querying-graphite-data)
 * [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
-* [How to import data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format)
-* [How to import CSV data](#how-to-import-csv-data)
 * [Prometheus querying API usage](#prometheus-querying-api-usage)
+  * [Prometheus querying API enhancements](#prometheus-querying-api-enhancements)
+* [Graphite Metrics API usage](#graphite-metrics-api-usage)
 * [How to build from sources](#how-to-build-from-sources)
   * [Development build](#development-build)
   * [Production build](#production-build)
@@ -112,8 +114,16 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
 * [Setting up service](#setting-up-service)
 * [How to work with snapshots](#how-to-work-with-snapshots)
 * [How to delete time series](#how-to-delete-time-series)
+* [Forced merge](#forced-merge)
 * [How to export time series](#how-to-export-time-series)
+  * [How to export data in native format](#how-to-export-data-in-native-format)
+  * [How to export data in JSON line format](#how-to-export-data-in-json-line-format)
+  * [How to export CSV data](#how-to-export-csv-data)
 * [How to import time series data](#how-to-import-time-series-data)
+  * [How to import data in native format](#how-to-import-data-in-native-format)
+  * [How to import data in json line format](#how-to-import-data-in-json-line-format)
+  * [How to import CSV data](#how-to-import-csv-data)
+  * [How to import data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format)
 * [Relabeling](#relabeling)
 * [Federation](#federation)
 * [Capacity planning](#capacity-planning)
@@ -285,6 +295,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
 * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
 * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
+* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
+* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
 
 In the future other `*_sd_config` types will be supported.
 
@@ -341,7 +353,7 @@ curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'ht
 ```
 
 An arbitrary number of lines delimited by '\n' (aka newline char) may be sent in a single request.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"measurement_.*"}'
@@ -377,7 +389,7 @@ echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N localhost 2003
 
 VictoriaMetrics sets the current time if the timestamp is omitted.
 An arbitrary number of lines delimited by `\n` (aka newline char) may be sent in one go.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
@@ -391,9 +403,11 @@ The `/api/v1/export` endpoint should return the following response:
 
 ### Querying Graphite data
 
-Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read either via
-[Prometheus querying API](#prometheus-querying-api-usage)
-or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
+
+* [Prometheus querying API](#prometheus-querying-api-usage)
+* Metric names can be explored via [Graphite metrics API](#graphite-metrics-api-usage)
+* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml)
 
 ### How to send data from OpenTSDB-compatible agents
 
@@ -419,7 +433,7 @@ echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N localhost
 ```
 
 An arbitrary number of lines delimited by `\n` (aka newline char) may be sent in one go.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
@@ -454,7 +468,7 @@ Example for writing multiple data points in a single request:
 curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
 ```
 
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar'
@@ -469,82 +483,6 @@ The `/api/v1/export` endpoint should return the following response:
 ```
 
 
-### How to import CSV data
-
-Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
-The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
-
-```
-<column_pos>:<type>:<context>
-```
-
-* `<column_pos>` is the position of the CSV column (field). Column numbering starts from 1. The order of parsing rules may be arbitrary.
-* `<type>` describes the column type. Supported types are:
-  * `metric` - the corresponding CSV column at `<column_pos>` contains metric value, which must be integer or floating-point number.
-    The metric name is read from the `<context>`. CSV line must have at least a single metric field. Multiple metric fields per CSV line is OK.
-  * `label` - the corresponding CSV column at `<column_pos>` contains label value. The label name is read from the `<context>`.
-    CSV line may have arbitrary number of label fields. All these labels are attached to all the configured metrics.
-  * `time` - the corresponding CSV column at `<column_pos>` contains metric time. CSV line may contain either one or zero columns with time.
-    If CSV line has no time, then the current time is used. The time is applied to all the configured metrics.
-    The format of the time is configured via `<context>`. Supported time formats are:
-    * `unix_s` - unix timestamp in seconds.
-    * `unix_ms` - unix timestamp in milliseconds.
-    * `unix_ns` - unix timestamp in nanoseconds. Note that VictoriaMetrics rounds the timestamp to milliseconds.
-    * `rfc3339` - timestamp in [RFC3339](https://tools.ietf.org/html/rfc3339) format, i.e. `2006-01-02T15:04:05Z`.
-    * `custom:<layout>` - custom layout for the timestamp. The `<layout>` may contain arbitrary time layout according to [time.Parse rules in Go](https://golang.org/pkg/time/#Parse).
-
-Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
-
-Example for importing CSV data via `/api/v1/import/csv`:
-
-```bash
-curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
-curl -d "MSFT,3.21,1.67,NASDAQ" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
-```
-
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
-
-```bash
-curl -G 'http://localhost:8428/api/v1/export' -d 'match[]={ticker!=""}'
-```
-
-The following response should be returned:
-```bash
-{"metric":{"__name__":"bid","market":"NASDAQ","ticker":"MSFT"},"values":[1.67],"timestamps":[1583865146520]}
-{"metric":{"__name__":"bid","market":"NYSE","ticker":"GOOG"},"values":[4.56],"timestamps":[1583865146495]}
-{"metric":{"__name__":"ask","market":"NASDAQ","ticker":"MSFT"},"values":[3.21],"timestamps":[1583865146520]}
-{"metric":{"__name__":"ask","market":"NYSE","ticker":"GOOG"},"values":[1.23],"timestamps":[1583865146495]}
-```
-
-Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
-
-
-### How to import data in Prometheus exposition format
-
-VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
-via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
-
-```bash
-curl -d 'foo{bar="baz"} 123' -X POST 'http://localhost:8428/api/v1/import/prometheus'
-```
-
-The following command may be used for verifying the imported data:
-
-```bash
-curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"foo"}'
-```
-
-It should return somethins like the following:
-
-```
-{"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
-```
-
-VictoriaMetrics accepts arbitrary number of lines in a single request to `/api/v1/import/prometheus`, i.e. it supports data streaming.
-
-VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
-
-
 ### Prometheus querying API usage
 
 VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
@@ -558,6 +496,13 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
 
 These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
 
+#### Prometheus querying API enhancements
+
+Additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) VictoriaMetrics accepts relative times in `time`, `start` and `end` query args.
+For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
+
+By default, VictoriaMetrics returns time series for the last 5 minutes from /api/v1/series, while the Prometheus API defaults to all time.  Use `start` and `end` to select a different time range.
+
 VictoriaMetrics accepts additional args for `/api/v1/labels` and `/api/v1/label/.../values` handlers.
 See [this feature request](https://github.com/prometheus/prometheus/issues/6178) for details:
 
@@ -566,11 +511,27 @@ See [this feature request](https://github.com/prometheus/prometheus/issues/6178)
 
 Additionally VictoriaMetrics provides the following handlers:
 
-* `/api/v1/series/count` - it returns the total number of time series in the database. Note that this handler scans all the inverted index,
-  so it can be slow if the database contains tens of millions of time series.
+* `/api/v1/series/count` - it returns the total number of time series in the database. Some notes:
+  * the handler scans all the inverted index, so it can be slow if the database contains tens of millions of time series;
+  * the handler may count [deleted time series](#how-to-delete-time-series) additionally to normal time series due to internal implementation restrictions;
 * `/api/v1/labels/count` - it returns a list of `label: values_count` entries. It can be used for determining labels with the maximum number of values.
 * `/api/v1/status/active_queries` - it returns a list of currently running queries.
 
+
+### Graphite Metrics API usage
+
+VictoriaMetrics supports the following handlers from [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api):
+
+* [/metrics/find](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find)
+* [/metrics/expand](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand)
+* [/metrics/index.json](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json)
+
+VictoriaMetrics accepts the following additional query args at `/metrics/find` and `/metrics/expand`:
+  * `label` - for selecting arbitrary label values. By default `label=__name__`, i.e. metric names are selected.
+  * `delimiter` - for using different delimiters in metric name hierachy. For example, `/metrics/find?delimiter=_&query=node_*` would return all the metric name prefixes
+    that start with `node_`. By default `delimiter=.`.
+
+
 ### How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -622,8 +583,8 @@ Run `make package-victoria-metrics`. It builds `victoriametrics/victoria-metrics
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-victoria-metrics`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image for improved debuggability.
-It is possible to build the package on top of any other base image by setting it via `<ROOT_IMAGE>` environment variable.
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable.
 For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
@@ -675,9 +636,12 @@ Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_s
 where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to delete. After that all the time series matching the given selector are deleted. Storage space for
 the deleted time series isn't freed instantly - it is freed during subsequent [background merges of data files](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+Note that background merges may never occur for data from previous months, so storage space won't be freed for historical data.
+In this case [forced merge](#forced-merge) may help freeing up storage space.
 
 It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
-before actually deleting the metrics.
+before actually deleting the metrics.  By default this query will only scan active series in the past 5 minutes, so you may need to
+adjust `start` and `end` to a suitable range to achieve match hits.
 
 The `/api/v1/admin/tsdb/delete_series` handler may be protected with `authKey` if `-deleteAuthKey` command-line flag is set.
 
@@ -693,16 +657,58 @@ It isn't recommended using delete API for the following cases, since it brings n
   See [this article](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts) for details.
 * Reducing disk space usage by deleting unneeded time series. This doesn't work as expected, since the deleted
   time series occupy disk space until the next merge operation, which can never occur when deleting too old data.
+  [Forced merge](#forced-merge) may be used for freeing up disk space occupied by old data.
 
 It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
 
+
+### Forced merge
+
+VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
+This means that compactions are stopped for per-month partitions if no new data is ingested into these partitions.
+Sometimes it is necessary to trigger compactions for old partitions. For instance, in order to free up disk space occupied by [deleted time series](#how-to-delete-time-series).
+In this case forced compaction may be initiated on the specified per-month partition by sending request to `/internal/force_merge?partition_prefix=YYYY_MM`,
+where `YYYY_MM` is per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate forced
+merge for August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in background.
+
+Forced merges may require additional CPU, disk IO and storage space resources. It is unnecessary to run forced merge under normal conditions,
+since VictoriaMetrics automatically performs [optimal merges in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+when new data is ingested into it.
+
+
 ### How to export time series
 
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
+VictoriaMetrics provides the following handlers for exporting data:
+
+* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
+  See [these docs](#how-to-export-data-in-native-format) for details.
+* `/api/v1/export` for exporing data in JSON line format. See [these docs](#how-to-export-data-in-json-line-format) for details.
+* `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
+
+
+#### How to export data in native format
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
+where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
+
+
+#### How to export data in JSON line format
+
+Consider [exporting data in native format](#how-to-export-data-in-native-format) if big amounts of data must be migrated between VictoriaMetrics instances,
+since exporting in native format usually consumes lower amounts of CPU and memory resources, while the resulting exported data occupies lower amounts of disk space.
+
+In order to export data in JSON line format, send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
 where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
 The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
-Each JSON line would contain data for a single time series. An example output:
+Each JSON line contains samples for a single time series. An example output:
 
 ```jsonl
 {"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
@@ -712,8 +718,9 @@ Each JSON line would contain data for a single time series. An example output:
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
 
-Optional `max_rows_per_line` arg may be added to the request in order to limit the maximum number of rows exported per each JSON line.
-By default each JSON line contains all the rows for a single time series.
+Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
+Optional `reduce_mem_usage=1` arg may be added to the request for reducing memory usage when exporting big number of time series.
+In this case the output may contain multiple lines with distinct samples for the same time series.
 
 Pass `Accept-Encoding: gzip` HTTP header in the request to `/api/v1/export` in order to reduce network bandwidth during exporing big amounts
 of time series data. This enables gzip compression for the exported data. Example for exporting gzipped data:
@@ -724,22 +731,82 @@ curl -H 'Accept-Encoding: gzip' http://localhost:8428/api/v1/export -d 'match[]=
 
 The maximum duration for each request to `/api/v1/export` is limited by `-search.maxExportDuration` command-line flag.
 
-Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-time-series-data).
+Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
+
+
+#### How to export CSV data
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/csv?format=<format>&match=<timeseries_selector_for_export>`,
+where:
+
+* `<format>` must contain comma-delimited label names for the exported CSV. The following special label names are supported:
+  * `__name__` - metric name
+  * `__value__` - sample value
+  * `__timestamp__:<ts_format>` - sample timestamp. `<ts_format>` can have the following values:
+    * `unix_s` - unix seconds
+    * `unix_ms` - unix milliseconds
+    * `unix_ns` - unix nanoseconds
+    * `rfc3339` - [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) time
+    * `custom:<layout>` - custom layout for time that is supported by [time.Format](https://golang.org/pkg/time/#Time.Format) function from Go.
+
+* `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export.
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
+
 
 ### How to import time series data
 
 Time series data can be imported via any supported ingestion protocol:
 
-* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-* [Influx line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
-* [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
-* [OpenTSDB telnet put protocol](#sending-data-via-telnet-put-protocol)
-* [OpenTSDB http /api/put](#sending-opentsdb-data-via-http-apiput-requests)
-* `/api/v1/import` http POST handler, which accepts data from [/api/v1/export](#how-to-export-time-series).
-* `/api/v1/import/csv` http POST handler, which accepts CSV data. See [these docs](#how-to-import-csv-data) for details.
-* `/api/v1/import/prometheus` http POST handler, which accepts data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
+* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
+* Influx line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
+* Graphite plaintext protocol. See [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
+* OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
+* OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
+* `/api/v1/import` for importing data obtained from [/api/v1/export](#how-to-export-data-in-json-line-format).
+  See [these docs](##how-to-import-data-in-json-line-format) for details.
+* `/api/v1/import/native` for importing data obtained from [/api/v1/export/native](#how-to-export-data-in-native-format).
+  See [these docs](#how-to-import-data-in-native-format) for details.
+* `/api/v1/import/csv` for importing arbitrary CSV data. See [these docs](#how-to-import-csv-data) for details.
+* `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
 
-The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import`. Example for importing data obtained via `/api/v1/export`:
+
+#### How to import data in native format
+
+The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import/native`.
+Example for importing data obtained via [/api/v1/export/native](#how-to-export-data-in-native-format):
+
+```bash
+# Export the data from <source-victoriametrics>:
+curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
+```
+
+Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/native` for importing gzipped data:
+
+```bash
+# Export gzipped data from <source-victoriametrics>:
+curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin.gz
+
+# Import gzipped data to <destination-victoriametrics>:
+curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin.gz
+```
+
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
+#### How to import data in JSON line format
+
+Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
 
 ```bash
 # Export the data from <source-victoriametrics>:
@@ -759,16 +826,113 @@ curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export
 curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import -T exported_data.jsonl.gz
 ```
 
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
 Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
 
-Each request to `/api/v1/import` can load up to a single vCPU core on VictoriaMetrics. Import speed can be improved by splitting the original file into smaller parts
-and importing them concurrently. Note that the original file must be split on newlines.
+
+#### How to import CSV data
+
+Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
+The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
+
+```
+<column_pos>:<type>:<context>
+```
+
+* `<column_pos>` is the position of the CSV column (field). Column numbering starts from 1. The order of parsing rules may be arbitrary.
+* `<type>` describes the column type. Supported types are:
+  * `metric` - the corresponding CSV column at `<column_pos>` contains metric value, which must be integer or floating-point number.
+    The metric name is read from the `<context>`. CSV line must have at least a single metric field. Multiple metric fields per CSV line is OK.
+  * `label` - the corresponding CSV column at `<column_pos>` contains label value. The label name is read from the `<context>`.
+    CSV line may have arbitrary number of label fields. All these labels are attached to all the configured metrics.
+  * `time` - the corresponding CSV column at `<column_pos>` contains metric time. CSV line may contain either one or zero columns with time.
+    If CSV line has no time, then the current time is used. The time is applied to all the configured metrics.
+    The format of the time is configured via `<context>`. Supported time formats are:
+    * `unix_s` - unix timestamp in seconds.
+    * `unix_ms` - unix timestamp in milliseconds.
+    * `unix_ns` - unix timestamp in nanoseconds. Note that VictoriaMetrics rounds the timestamp to milliseconds.
+    * `rfc3339` - timestamp in [RFC3339](https://tools.ietf.org/html/rfc3339) format, i.e. `2006-01-02T15:04:05Z`.
+    * `custom:<layout>` - custom layout for the timestamp. The `<layout>` may contain arbitrary time layout according to [time.Parse rules in Go](https://golang.org/pkg/time/#Parse).
+
+Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
+
+Example for importing CSV data via `/api/v1/import/csv`:
+
+```bash
+curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+curl -d "MSFT,3.21,1.67,NASDAQ" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+```
+
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
+
+```bash
+curl -G 'http://localhost:8428/api/v1/export' -d 'match[]={ticker!=""}'
+```
+
+The following response should be returned:
+```bash
+{"metric":{"__name__":"bid","market":"NASDAQ","ticker":"MSFT"},"values":[1.67],"timestamps":[1583865146520]}
+{"metric":{"__name__":"bid","market":"NYSE","ticker":"GOOG"},"values":[4.56],"timestamps":[1583865146495]}
+{"metric":{"__name__":"ask","market":"NASDAQ","ticker":"MSFT"},"values":[3.21],"timestamps":[1583865146520]}
+{"metric":{"__name__":"ask","market":"NYSE","ticker":"GOOG"},"values":[1.23],"timestamps":[1583865146495]}
+```
+
+Extra labels may be added to all the imported lines by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/csv?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported lines.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
+#### How to import data in Prometheus exposition format
+
+VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
+via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
+
+```bash
+curl -d 'foo{bar="baz"} 123' -X POST 'http://localhost:8428/api/v1/import/prometheus'
+```
+
+The following command may be used for verifying the imported data:
+
+```bash
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"foo"}'
+```
+
+It should return something like the following:
+
+```
+{"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
+```
+
+Extra labels may be added to all the imported metrics by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/prometheus?extra_label=foo=bar` would add `{foo="bar"}` label to all the imported metrics.
+
+If timestamp is missing in `<metric> <value> <timestamp>` Prometheus exposition format line, then the current timestamp is used during data ingestion.
+It can be overriden by passing unix timestamp in *milliseconds* via `timestamp` query arg. For example, `/api/v1/import/prometheus?timestamp=1594370496905`.
+
+VictoriaMetrics accepts arbitrary number of lines in a single request to `/api/v1/import/prometheus`, i.e. it supports data streaming.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+
 
 
 ### Relabeling
 
 VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
 to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
+Example contents for `-relabelConfig` file:
+```yml
+# relabel_config.yml
+- target_label: cluster
+  replacement: dev
+- action: drop
+  source_labels: [__meta_kubernetes_pod_container_init]
+  regex: true
+```
 
 VictoriaMetrics provides the following extra actions for relabeling rules:
 
@@ -876,7 +1040,8 @@ with the enabled de-duplication. See [this section](#deduplication) for details.
 
 VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag
 is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points
-on the same time series if they are located closer than 60s to each other.
+on the same time series if they fall within the same discrete 60s bucket.  The earliest data point will be kept.  In the case of equal timestamps, an arbitrary data point will be kept.
+
 The de-duplication reduces disk space usage if multiple identically configured Prometheus instances in HA pair
 write data to the same VictoriaMetrics instance. Note that these Prometheus instances must have identical
 `external_labels` section in their configs, so they write data to the same time series.
@@ -956,6 +1121,7 @@ Consider setting the following command-line flags:
   with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
 * `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
 * `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
+* `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
 * `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
 
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
@@ -1037,6 +1203,9 @@ VictoriaMetrics also exposes currently running queries with their execution time
   has at least 20% of free space comparing to disk size. The remaining amount of free space
   can be [monitored](#monitoring) via `vm_free_disk_space_bytes` metric. The total size of data
   stored on the disk can be monitored via sum of `vm_data_size_bytes` metrics.
+  See also `vm_merge_need_free_disk_space` metrics, which are set to values higher than 0
+  if background merge cannot be initiated due to free disk space shortage. The value shows the number of per-month partitions,
+  which would start background merge if they had more free disk space.
 
 * If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
   then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
@@ -1063,6 +1232,8 @@ VictoriaMetrics also exposes currently running queries with their execution time
   This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
   metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
 
+* VictoriaMetrics ignores `NaN` values during data ingestion.
+
 
 ### Backfilling
 
@@ -1082,7 +1253,7 @@ for data with timestamps close to the current time.
 
 ### Data updates
 
-VictoriaMetrics doesn't support updating already exiting sample values to new ones. It stores all the ingested data points
+VictoriaMetrics doesn't support updating already existing sample values to new ones. It stores all the ingested data points
 for the same time series with identical timestamps. While is possible substituting old time series with new time series via
 [removal of old time series](#how-to-delete-timeseries) and then [writing new time series](#backfilling), this approach
 should be used only for one-off updates. It shouldn't be used for frequent updates because of non-zero overhead related to data removal.

app/victoria-metrics/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -66,6 +67,10 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+		return true
+	}
 	if vminsert.RequestHandler(w, r) {
 		return true
 	}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -9,6 +9,6 @@
   "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
   "result_query": {
     "status":"success",
-    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","1"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","3"]}]}
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","2"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","4"]}]}
   }
 }

app/vmagent/README.md

@@ -25,7 +25,8 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-native-format).
   * Data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
@@ -41,7 +42,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
 
@@ -135,7 +136,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `ec2_sd_configs` - for scraping targets in Amazon EC2.
   See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
-  `vmagent` doesn't support `role_arn` config param yet.
+  `vmagent` doesn't support `profile` config param and aws credentials file yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
   `vmagent` provides the following additional functionality for `gce_sd_config`:
@@ -147,6 +148,11 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
 * `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
   See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+* `openstack_sd_configs` - for scraping OpenStack targets.
+  See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
+  [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
+  See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
 
 File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -206,6 +212,8 @@ Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) f
 If you have suggestions, improvements or found a bug - feel free to open an issue on github or add review to the dashboard.
 
 `vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`/targets` handler accepts optional `show_original_labels=1` query arg, which shows the original labels per each target
+before applying relabeling. This information may be useful for debugging target relabeling.
 
 
 ### Troubleshooting
@@ -218,8 +226,7 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
 
-* It is recommended to increase `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
-  and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+* It is recommended to increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
 
 * If you see gaps on the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, then try increasing `-remoteWrite.queues`.
   Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage, so it starts dropping the buffered data
@@ -229,8 +236,13 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* By default `vmagent` masks `-remoteWrite.url` with `secret-url` values in logs and at `/metrics` page because
+  the url may contain sensitive information such as auth tokens or passwords.
+  Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
-  or they use init container.
+  or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
+  See available options below if you prefer fixing the root cause of the error:
 
   The following `relabel_configs` section may help determining `__meta_*` labels resulting in duplicate targets:
   ```yml
@@ -275,7 +287,7 @@ Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmagent/csvimport/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -18,12 +19,18 @@ var (
 
 // InsertHandler processes csv data from req.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, insertRows)
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -44,6 +51,7 @@ func insertRows(rows []parser.Row) error {
 				Value: tag.Value,
 			})
 		}
+		labels = append(labels, extraLabels...)
 		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,

app/vmagent/influx/request_handler.go

@@ -62,6 +62,7 @@ func insertRows(db string, rows []parser.Row) error {
 	buf := ctx.buf[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		commonLabels = commonLabels[:0]
 		hasDBKey := false
 		for j := range r.Tags {
@@ -111,7 +112,6 @@ func insertRows(db string, rows []parser.Row) error {
 				Samples: samples[len(samples)-1:],
 			})
 		}
-		rowsTotal += len(r.Fields)
 	}
 	ctx.buf = buf
 	ctx.ctx.WriteRequest.Timeseries = tssDst

app/vmagent/main.go

@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -28,6 +30,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -59,6 +62,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	cgroup.UpdateGOMAXPROCSToCPUQuota()
@@ -80,6 +84,7 @@ func main() {
 	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
 	remotewrite.Init()
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -127,12 +132,17 @@ func main() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
+		return true
+	}
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
@@ -171,6 +181,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
@@ -189,7 +208,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
@@ -213,6 +233,9 @@ var (
 	prometheusimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 	prometheusimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 
+	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
 

app/vmagent/native/request_handler.go

@@ -0,0 +1,85 @@
+package native
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	mn := &block.MetricName
+	labelsLen := len(labels)
+	labels = append(labels, prompbmarshal.Label{
+		Name:  "__name__",
+		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
+	})
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		labels = append(labels, prompbmarshal.Label{
+			Name:  bytesutil.ToUnsafeString(tag.Key),
+			Value: bytesutil.ToUnsafeString(tag.Value),
+		})
+	}
+	labels = append(labels, extraLabels...)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	samplesLen := len(samples)
+	for j, value := range values {
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     value,
+			Timestamp: timestamps[j],
+		})
+	}
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
+		Labels:  labels[labelsLen:],
+		Samples: samples[samplesLen:],
+	})
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	return nil
+}

app/vmagent/prometheusimport/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -18,13 +19,23 @@ var (
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
 		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, isGzipped, insertRows)
+		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -45,6 +56,7 @@ func insertRows(rows []parser.Row) error {
 				Value: tag.Value,
 			})
 		}
+		labels = append(labels, extraLabels...)
 		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,7 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
 		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
 		labelsLen := len(labels)
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
@@ -55,7 +56,6 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(ts.Samples)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmagent/remotewrite/client.go

@@ -44,7 +44,7 @@ var (
 )
 
 type client struct {
-	urlLabelValue  string
+	sanitizedURL   string
 	remoteWriteURL string
 	authHeader     string
 	fq             *persistentqueue.FastQueue
@@ -59,7 +59,7 @@ type client struct {
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	tlsCfg, err := getTLSConfig(argIdx)
 	if err != nil {
 		logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
@@ -101,7 +101,7 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 		authHeader = "Bearer " + token
 	}
 	c := &client{
-		urlLabelValue:  urlLabelValue,
+		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authHeader:     authHeader,
 		fq:             fq,
@@ -111,10 +111,10 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 		},
 		stopCh: make(chan struct{}),
 	}
-	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
-	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
-	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
-	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -122,14 +122,14 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 			c.runWorker()
 		}()
 	}
-	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 	return c
 }
 
 func (c *client) MustStop() {
 	close(c.stopCh)
 	c.wg.Wait()
-	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
 }
 
 func getTLSConfig(argIdx int) (*tls.Config, error) {
@@ -176,7 +176,7 @@ func (c *client) runWorker() {
 				// The block has been sent successfully.
 			case <-time.After(graceDuration):
 				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
-					len(block), c.remoteWriteURL, graceDuration.Seconds())
+					len(block), c.sanitizedURL, graceDuration.Seconds())
 			}
 			return
 		}
@@ -185,11 +185,12 @@ func (c *client) runWorker() {
 
 func (c *client) sendBlock(block []byte) {
 	retryDuration := time.Second
+	retriesCount := 0
 
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.remoteWriteURL, err)
+		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -210,7 +211,7 @@ again:
 			retryDuration = time.Minute
 		}
 		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := time.NewTimer(retryDuration)
 		select {
 		case <-c.stopCh:
@@ -229,7 +230,8 @@ again:
 	}
 
 	// Unexpected status code returned
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+	retriesCount++
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	retryDuration *= 2
 	if retryDuration > time.Minute {
 		retryDuration = time.Minute
@@ -237,10 +239,10 @@ again:
 	body, err := ioutil.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
-		logger.Errorf("cannot read response body from %q: %s", c.remoteWriteURL, err)
+		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
 	} else {
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
-			len(block), c.remoteWriteURL, statusCode, body, retryDuration.Seconds())
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
+			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
 	}
 	t := time.NewTimer(retryDuration)
 	select {

app/vmagent/remotewrite/pendingseries.go

@@ -3,10 +3,12 @@ package remotewrite
 import (
 	"flag"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -17,7 +19,7 @@ var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage. "+
 		"Minimum supported interval is 1 second")
-	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
 		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
 )
 
@@ -68,7 +70,7 @@ func (ps *pendingSeries) periodicFlusher() {
 		case <-ps.stopCh:
 			mustStop = true
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
@@ -79,10 +81,12 @@ func (ps *pendingSeries) periodicFlusher() {
 }
 
 type writeRequest struct {
-	wr            prompbmarshal.WriteRequest
-	pushBlock     func(block []byte)
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
 	lastFlushTime uint64
 
+	wr        prompbmarshal.WriteRequest
+	pushBlock func(block []byte)
+
 	tss []prompbmarshal.TimeSeries
 
 	labels  []prompbmarshal.Label
@@ -113,7 +117,7 @@ func (wr *writeRequest) reset() {
 
 func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
-	wr.lastFlushTime = fasttime.UnixTimestamp()
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
 	pushWriteRequest(&wr.wr, wr.pushBlock)
 	wr.reset()
 }
@@ -122,9 +126,9 @@ func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		dst := &tssDst[len(tssDst)-1]
-		wr.copyTimeSeries(dst, &src[i])
-		if len(wr.tss) >= maxRowsPerBlock {
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
+		if len(wr.samples) >= maxRowsPerBlock {
+			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss
 		}
@@ -164,7 +168,7 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 	}
 	bb := writeRequestBufPool.Get()
 	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
-	if len(bb.B) <= *maxUnpackedBlockSize {
+	if len(bb.B) <= maxUnpackedBlockSize.N {
 		zb := snappyBufPool.Get()
 		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		writeRequestBufPool.Put(bb)

app/vmagent/remotewrite/remotewrite.go

@@ -3,12 +3,12 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"runtime"
 	"sync"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -23,16 +23,16 @@ var (
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
-	queues      = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+	queues      = flag.Int("remoteWrite.queues", 4, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
-	maxPendingBytesPerURL = flag.Int("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
 		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
 		"Disk usage is unlimited if the value is set to 0")
-	decimalPlaces = flag.Int("remoteWrite.decimalPlaces", 0, "The number of significant decimal places to leave in metric values before writing them to remote storage. "+
-		"See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant decimal places. "+
+	significantFigures = flag.Int("remoteWrite.significantFigures", 0, "The number of significant figures to leave in metric values before writing them to remote storage. "+
+		"See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
 		"This option may be used for increasing on-disk compression level for the stored metrics")
 )
 
@@ -41,6 +41,18 @@ var rwctxs []*remoteWriteCtx
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Value
 
+// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
+// since it may lead to high memory usage due to big number of buffers.
+var maxQueues = runtime.GOMAXPROCS(-1) * 4
+
+// InitSecretFlags must be called after flag.Parse and before any logging.
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -48,12 +60,13 @@ var allRelabelConfigs atomic.Value
 // Stop must be called for graceful shutdown.
 func Init() {
 	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` must be set")
+		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
-
-	if !*showRemoteWriteURL {
-		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
-		httpserver.RegisterSecretFlag("remoteWrite.url")
+	if *queues > maxQueues {
+		*queues = maxQueues
+	}
+	if *queues <= 0 {
+		*queues = 1
 	}
 	initLabelsGlobal()
 	rcs, err := loadRelabelConfigs()
@@ -73,11 +86,11 @@ func Init() {
 		maxInmemoryBlocks = 2
 	}
 	for i, remoteWriteURL := range *remoteWriteURLs {
-		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if *showRemoteWriteURL {
-			urlLabelValue = remoteWriteURL
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, urlLabelValue)
+		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 		rwctxs = append(rwctxs, rwctx)
 	}
 
@@ -124,13 +137,13 @@ func Stop() {
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
 func Push(wr *prompbmarshal.WriteRequest) {
-	if *decimalPlaces > 0 {
-		// Round values according to decimalPlaces
+	if *significantFigures > 0 {
+		// Round values according to significantFigures
 		for i := range wr.Timeseries {
 			samples := wr.Timeseries[i].Samples
 			for j := range samples {
 				s := &samples[j]
-				s.Value = decimal.Round(s.Value, *decimalPlaces)
+				s.Value = decimal.Round(s.Value, *significantFigures)
 			}
 		}
 	}
@@ -144,10 +157,19 @@ func Push(wr *prompbmarshal.WriteRequest) {
 	tss := wr.Timeseries
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
+		samplesCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			i++
+			if samplesCount > maxRowsPerBlock {
+				break
+			}
+		}
 		tssBlock := tss
-		if len(tssBlock) > maxRowsPerBlock {
-			tssBlock = tss[:maxRowsPerBlock]
-			tss = tss[maxRowsPerBlock:]
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
 		} else {
 			tss = nil
 		}
@@ -180,17 +202,17 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
-	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, *maxPendingBytesPerURL)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, urlLabelValue, fq, *queues)
+	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
 	pss := make([]*pendingSeries, *queues)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq.MustWriteBlock)
@@ -201,7 +223,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
 	}
 }
 

app/vmagent/vmimport/request_handler.go

@@ -6,7 +6,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,12 +23,18 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, insertRows)
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -36,6 +44,7 @@ func insertRows(rows []parser.Row) error {
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Values)
 		labelsLen := len(labels)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
@@ -44,9 +53,12 @@ func insertRows(rows []parser.Row) error {
 				Value: bytesutil.ToUnsafeString(tag.Value),
 			})
 		}
+		labels = append(labels, extraLabels...)
 		values := r.Values
 		timestamps := r.Timestamps
-		_ = timestamps[len(values)-1]
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
 		samplesLen := len(samples)
 		for j, value := range values {
 			samples = append(samples, prompbmarshal.Sample{
@@ -58,7 +70,6 @@ func insertRows(rows []parser.Row) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(values)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmalert/README.md

@@ -166,6 +166,10 @@ The shortlist of configuration flags is the following:
     	Optional basic auth password for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.lookback duration
+        Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+  -datasource.maxIdleConnections int
+        Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
   -datasource.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
   -datasource.tlsCertFile string
@@ -194,8 +198,12 @@ The shortlist of configuration flags is the following:
     	Supports array of values separated by comma or specified via multiple flags.
   -external.url string
     	External URL is used as alert's source for sent alerts to the notifier
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
   -http.disableResponseCompression
     	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
     	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
   -http.pathPrefix string
@@ -216,8 +224,9 @@ The shortlist of configuration flags is the following:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -memory.allowedBytes int
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
@@ -293,8 +302,8 @@ The shortlist of configuration flags is the following:
     	Path to the file with alert rules. 
     	Supports patterns. Flag can be specified multiple times. 
     	Examples:
-    	 -rule /path/to/file. Path to a single file with alerting rules
-    	 -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+    	 -rule="/path/to/file". Path to a single file with alerting rules
+    	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, 
     	absolute path to all .yaml files in root.
     	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
     	Supports array of values separated by comma or specified via multiple flags.

app/vmalert/alerting.go

@@ -26,6 +26,7 @@ type AlertingRule struct {
 	Labels      map[string]string
 	Annotations map[string]string
 	GroupID     uint64
+	GroupName   string
 
 	// guard status fields
 	mu sync.RWMutex
@@ -52,10 +53,11 @@ func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
 		RuleID:      cfg.ID,
 		Name:        cfg.Alert,
 		Expr:        cfg.Expr,
-		For:         cfg.For,
+		For:         cfg.For.Duration(),
 		Labels:      cfg.Labels,
 		Annotations: cfg.Annotations,
 		GroupID:     group.ID(),
+		GroupName:   group.Name,
 		alerts:      make(map[uint64]*notifier.Alert),
 		metrics:     &alertingRuleMetrics{},
 	}
@@ -242,6 +244,9 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifie
 		Start:   start,
 		Expr:    ar.Expr,
 	}
+	// label defined here to make override possible by
+	// time series labels.
+	a.Labels[alertGroupNameLabel] = ar.GroupName
 	for _, l := range m.Labels {
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
@@ -336,15 +341,18 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 }
 
 const (
-	// AlertMetricName is the metric name for synthetic alert timeseries.
+	// alertMetricName is the metric name for synthetic alert timeseries.
 	alertMetricName = "ALERTS"
-	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	// alertForStateMetricName is the metric name for 'for' state of alert.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
-	// AlertNameLabel is the label name indicating the name of an alert.
+	// alertNameLabel is the label name indicating the name of an alert.
 	alertNameLabel = "alertname"
-	// AlertStateLabel is the label name indicating the state of an alert.
+	// alertStateLabel is the label name indicating the state of an alert.
 	alertStateLabel = "alertstate"
+
+	// alertGroupNameLabel defines the label name attached for generated time series.
+	alertGroupNameLabel = "alertgroup"
 )
 
 // alertToTimeSeries converts the given alert with the given timestamp to timeseries

app/vmalert/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"crypto/md5"
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
@@ -11,6 +12,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/metricsql"
 	"gopkg.in/yaml.v2"
 )
@@ -23,11 +25,30 @@ type Group struct {
 	Interval    time.Duration `yaml:"interval,omitempty"`
 	Rules       []Rule        `yaml:"rules"`
 	Concurrency int           `yaml:"concurrency"`
+	// Checksum stores the hash of yaml definition for this group.
+	// May be used to detect any changes like rules re-ordering etc.
+	Checksum string
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
 }
 
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type group Group
+	if err := unmarshal((*group)(g)); err != nil {
+		return err
+	}
+	b, err := yaml.Marshal(g)
+	if err != nil {
+		return fmt.Errorf("failed to marshal group configuration for checksum: %w", err)
+	}
+	h := md5.New()
+	h.Write(b)
+	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
 // Validate check for internal Group or Rule configuration errors
 func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
@@ -73,7 +94,7 @@ type Rule struct {
 	Record      string            `yaml:"record,omitempty"`
 	Alert       string            `yaml:"alert,omitempty"`
 	Expr        string            `yaml:"expr"`
-	For         time.Duration     `yaml:"for,omitempty"`
+	For         PromDuration      `yaml:"for,omitempty"`
 	Labels      map[string]string `yaml:"labels,omitempty"`
 	Annotations map[string]string `yaml:"annotations,omitempty"`
 
@@ -81,6 +102,37 @@ type Rule struct {
 	XXX map[string]interface{} `yaml:",inline"`
 }
 
+// PromDuration is Prometheus duration.
+type PromDuration struct {
+	milliseconds int64
+}
+
+// NewPromDuration returns PromDuration for given d.
+func NewPromDuration(d time.Duration) PromDuration {
+	return PromDuration{
+		milliseconds: d.Milliseconds(),
+	}
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler interface.
+func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	ms, err := metricsql.DurationValue(s, 0)
+	if err != nil {
+		return err
+	}
+	pd.milliseconds = ms
+	return nil
+}
+
+// Duration returns duration for pd.
+func (pd *PromDuration) Duration() time.Duration {
+	return time.Duration(pd.milliseconds) * time.Millisecond
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
@@ -100,7 +152,7 @@ func (r *Rule) Name() string {
 }
 
 // HashRule hashes significant Rule fields into
-// unique hash value
+// unique hash that supposed to define Rule uniqueness
 func HashRule(r Rule) uint64 {
 	h := fnv.New64a()
 	h.Write([]byte(r.Expr))
@@ -111,16 +163,7 @@ func HashRule(r Rule) uint64 {
 		h.Write([]byte("alerting"))
 		h.Write([]byte(r.Alert))
 	}
-	type item struct {
-		key, value string
-	}
-	var kv []item
-	for k, v := range r.Labels {
-		kv = append(kv, item{key: k, value: v})
-	}
-	sort.Slice(kv, func(i, j int) bool {
-		return kv[i].key < kv[j].key
-	})
+	kv := sortMap(r.Labels)
 	for _, i := range kv {
 		h.Write([]byte(i.key))
 		h.Write([]byte(i.value))
@@ -170,7 +213,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		}
 	}
 	if len(groups) < 1 {
-		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
 	return groups, nil
 }
@@ -203,3 +246,18 @@ func checkOverflow(m map[string]interface{}, ctx string) error {
 	}
 	return nil
 }
+
+type item struct {
+	key, value string
+}
+
+func sortMap(m map[string]string) []item {
+	var kv []item
+	for k, v := range m {
+		kv = append(kv, item{key: k, value: v})
+	}
+	sort.Slice(kv, func(i, j int) bool {
+		return kv[i].key < kv[j].key
+	})
+	return kv
+}

app/vmalert/config/config_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
@@ -51,10 +52,6 @@ func TestParseBad(t *testing.T) {
 			[]string{"testdata/dir/rules4-bad.rules"},
 			"either `record` or `alert` must be set",
 		},
-		{
-			[]string{"testdata/*.yaml"},
-			"no groups found",
-		},
 	}
 	for _, tc := range testCases {
 		_, err := Parse(tc.path, true, true)
@@ -273,7 +270,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: time.Minute},
+			Rule{Alert: "alert", Expr: "up == 1", For: NewPromDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -324,3 +321,36 @@ func TestHashRule(t *testing.T) {
 		}
 	}
 }
+
+func TestGroupChecksum(t *testing.T) {
+	data := `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`
+	var g Group
+	if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+		t.Fatalf("failed to unmarshal: %s", err)
+	}
+	if g.Checksum == "" {
+		t.Fatalf("expected to get non-empty checksum")
+	}
+	newData := `
+name: TestGroup
+rules:
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`
+	var ng Group
+	if err := yaml.Unmarshal([]byte(newData), &g); err != nil {
+		t.Fatalf("failed to unmarshal: %s", err)
+	}
+	if g.Checksum == ng.Checksum {
+		t.Fatalf("expected to get different checksums")
+	}
+}

app/vmalert/config/testdata/kube-good.rules

@@ -665,7 +665,7 @@
           /
         sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
         > 0.01
-      for: 15m
+      for: 1d
       labels:
         severity: critical
     - alert: KubeStateMetricsWatchErrors
@@ -1724,4 +1724,4 @@
         rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1
       for: 10m
       labels:
-        severity: warning
+        severity: warning

app/vmalert/datasource/init.go

@@ -21,18 +21,23 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
 		"By default the server name from -datasource.url is used")
+
+	lookBack = flag.Duration("datasource.lookback", 0, "Lookback defines how far to look into past when evaluating queries. "+
+		"For example, if datasource.lookback=5m then param \"time\" with value now()-5m will be added to every query.")
+	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, "Defines the number of idle (keep-alive connections) to configured datasource."+
+		"Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state.")
 )
 
 // Init creates a Querier from provided flag values.
 func Init() (Querier, error) {
 	if *addr == "" {
-		flag.PrintDefaults()
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
 	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.MaxIdleConns = *maxIdleConnections
 	c := &http.Client{Transport: tr}
-	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, *lookBack, c), nil
 }

app/vmalert/datasource/vm.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 )
 
 type response struct {
@@ -45,23 +46,25 @@ func (r response) metrics() ([]Metric, error) {
 	return ms, nil
 }
 
-const queryPath = "/api/v1/query?query="
-
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c             *http.Client
 	queryURL      string
 	basicAuthUser string
 	basicAuthPass string
+	lookBack      time.Duration
 }
 
+const queryPath = "/api/v1/query?query="
+
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, c *http.Client) *VMStorage {
 	return &VMStorage{
 		c:             c,
 		basicAuthUser: basicAuthUser,
 		basicAuthPass: basicAuthPass,
 		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+		lookBack:      lookBack,
 	}
 }
 
@@ -70,7 +73,12 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	const (
 		statusSuccess, statusError, rtVector = "success", "error", "vector"
 	)
-	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	q := s.queryURL + url.QueryEscape(query)
+	if s.lookBack > 0 {
+		lookBack := time.Now().Add(-s.lookBack)
+		q += fmt.Sprintf("&time=%d", lookBack.Unix())
+	}
+	req, err := http.NewRequest("POST", q, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -85,7 +93,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s with err %w. Reponse body %s", resp.StatusCode, req.URL, err, body)
+		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
 	}
 	r := &response{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
@@ -98,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
 	}
 	if r.Data.ResultType != rtVector {
-		return nil, fmt.Errorf("unknown restul type:%s. Expected vector", r.Data.ResultType)
+		return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
 	}
 	return r.metrics()
 }

app/vmalert/datasource/vm_test.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
+	"time"
 )
 
 var (
@@ -31,6 +33,13 @@ func TestVMSelectQuery(t *testing.T) {
 		if r.URL.Query().Get("query") != query {
 			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
+		timeParam := r.URL.Query().Get("time")
+		if timeParam == "" {
+			t.Errorf("expected 'time' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(timeParam, 10, 64); err != nil {
+			t.Errorf("failed to parse 'time' query param: %s", err)
+		}
 		switch c {
 		case 0:
 			conn, _, _ := w.(http.Hijacker).Hijack()
@@ -52,7 +61,7 @@ func TestVMSelectQuery(t *testing.T) {
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, srv.Client())
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
@@ -66,7 +75,7 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("expected error status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
-		t.Fatalf("expected unkown status got nil")
+		t.Fatalf("expected unknown status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
@@ -89,5 +98,4 @@ func TestVMSelectQuery(t *testing.T) {
 		m[0].Labels[0].Name != expected.Labels[0].Name {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
-
 }

app/vmalert/group.go

@@ -24,6 +24,7 @@ type Group struct {
 	Rules       []Rule
 	Interval    time.Duration
 	Concurrency int
+	Checksum    string
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -53,6 +54,7 @@ func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string
 		File:        cfg.File,
 		Interval:    cfg.Interval,
 		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
 		doneCh:      make(chan struct{}),
 		finishedCh:  make(chan struct{}),
 		updateCh:    make(chan *Group),
@@ -156,6 +158,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 		newRules = append(newRules, nr)
 	}
 	g.Concurrency = newGroup.Concurrency
+	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
 }
@@ -180,8 +183,31 @@ func (g *Group) close() {
 	}
 }
 
+var skipRandSleepOnGroupStart bool
+
 func (g *Group) start(ctx context.Context, querier datasource.Querier, nts []notifier.Notifier, rw *remotewrite.Client) {
 	defer func() { close(g.finishedCh) }()
+
+	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	if !skipRandSleepOnGroupStart {
+		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(g.Interval)
+		}
+		randSleep -= sleepOffset
+		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		select {
+		case <-ctx.Done():
+			sleepTimer.Stop()
+			return
+		case <-g.doneCh:
+			sleepTimer.Stop()
+			return
+		case <-sleepTimer.C:
+		}
+	}
+
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 	e := &executor{querier, nts, rw}
 	t := time.NewTicker(g.Interval)

app/vmalert/group_test.go

@@ -10,6 +10,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
 
+func init() {
+	// Disable rand sleep on group start during tests in order to speed up test execution.
+	// Rand sleep is needed only in prod code.
+	skipRandSleepOnGroupStart = true
+}
+
 func TestUpdateWith(t *testing.T) {
 	testCases := []struct {
 		name         string
@@ -26,7 +32,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   time.Second,
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -38,7 +44,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   time.Second,
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},

app/vmalert/main.go

@@ -29,8 +29,8 @@ var (
 	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules. 
 Supports patterns. Flag can be specified multiple times. 
 Examples:
- -rule /path/to/file. Path to a single file with alerting rules
- -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+ -rule="/path/to/file". Path to a single file with alerting rules
+ -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, 
 absolute path to all .yaml files in root.
 Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
 

app/vmalert/manager.go

@@ -93,31 +93,53 @@ func (m *manager) update(ctx context.Context, path []string, validateTpl, valida
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	type updateItem struct {
+		old *Group
+		new *Group
+	}
+	var toUpdate []updateItem
+
 	m.groupsMu.Lock()
 	for _, og := range m.groups {
 		ng, ok := groupsRegistry[og.ID()]
 		if !ok {
-			// old group is not present in new list
-			// and must be stopped and deleted
+			// old group is not present in new list,
+			// so must be stopped and deleted
 			og.close()
 			delete(m.groups, og.ID())
 			og = nil
 			continue
 		}
-		og.updateCh <- ng
 		delete(groupsRegistry, ng.ID())
+		if og.Checksum != ng.Checksum {
+			toUpdate = append(toUpdate, updateItem{old: og, new: ng})
+		}
 	}
-
 	for _, ng := range groupsRegistry {
 		m.startGroup(ctx, ng, restore)
 	}
 	m.groupsMu.Unlock()
+
+	if len(toUpdate) > 0 {
+		var wg sync.WaitGroup
+		for _, item := range toUpdate {
+			wg.Add(1)
+			go func(old *Group, new *Group) {
+				old.updateCh <- new
+				wg.Done()
+			}(item.old, item.new)
+		}
+		wg.Wait()
+	}
 	return nil
 }
 
 func (g *Group) toAPI() APIGroup {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
 	ag := APIGroup{
-		// encode as strings to avoid rounding
+		// encode as string to avoid rounding
 		ID:          fmt.Sprintf("%d", g.ID()),
 		Name:        g.Name,
 		File:        g.File,

app/vmalert/manager_test.go

@@ -5,7 +5,6 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -19,16 +18,15 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestManagerUpdateError(t *testing.T) {
+// TestManagerEmptyRulesDir tests
+// successful cases of
+// starting with empty rules folder
+func TestManagerEmptyRulesDir(t *testing.T) {
 	m := &manager{groups: make(map[uint64]*Group)}
 	path := []string{"foo/bar"}
 	err := m.update(context.Background(), path, true, true, false)
-	if err == nil {
-		t.Fatalf("expected to have err; got nil instead")
-	}
-	expErr := "no groups found"
-	if !strings.Contains(err.Error(), expErr) {
-		t.Fatalf("expected to got err %s; got %s", expErr, err)
+	if err != nil {
+		t.Fatalf("expected to load succesfully with empty rules dir; got err instead: %v", err)
 	}
 }
 
@@ -180,6 +178,27 @@ func TestManagerUpdate(t *testing.T) {
 					}},
 			},
 		},
+		{
+			name:       "update empty dir rules from 0 to 2 groups",
+			initPath:   "config/testdata/empty/*",
+			updatePath: "config/testdata/rules0-good.rules",
+			want: []*Group{
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Interval: defaultEvalInterval,
+					Rules:    []Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup", Rules: []Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
+			},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

app/vmalert/notifier/init.go

@@ -26,7 +26,6 @@ var (
 // Init creates a Notifier object based on provided flags.
 func Init(gen AlertURLGenerator) ([]Notifier, error) {
 	if len(*addrs) == 0 {
-		flag.PrintDefaults()
 		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
 	}
 

app/vmalert/remoteread/init.go

@@ -35,5 +35,5 @@ func Init() (datasource.Querier, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, c), nil
 }

app/vmauth/README.md

@@ -113,7 +113,7 @@ Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker i
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
@@ -140,3 +140,68 @@ curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
 The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+
+
+### Advanced usage
+
+Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
+
+```
+./vmauth -help
+
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md .
+
+  -auth.config string
+    	Path to auth config. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md for details on the format of this auth config
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address to listen for http connections (default ":8427")
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit (default 10)
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -memory.allowedBytes value
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It overrides httpAuth settings
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It overrides httpAuth settings
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+  -version
+    	Show VictoriaMetrics version
+```

app/vmbackup/README.md

@@ -6,12 +6,12 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. This saves time and costs on data transfer.
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
 
@@ -35,8 +35,8 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
 
@@ -49,13 +49,13 @@ with the following command:
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
 ```
 
-This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
 
 #### Incremental backups
 
-Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
-This saves time and network bandwidth costs when working with big backups:
+Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+It saves time and network bandwidth costs when working with big backups:
 
 ```
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
@@ -100,16 +100,16 @@ The backup algorithm is the following:
 2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
 3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
 4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
-   This are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+   These are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
+- Old files periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -129,7 +129,45 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ### Advanced usage
 
-Run `vmbackup -help` in order to see all the available options:
+
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+  You have to add a custom url endpoint via flag:
+```
+  # for minio
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+* Run `vmbackup -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -138,7 +176,7 @@ Run `vmbackup -help` in order to see all the available options:
     	Path to file with S3 configs. Configs are loaded from default location if not set.
     	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs (default "default")
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
     	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
@@ -161,18 +199,20 @@ Run `vmbackup -help` in order to see all the available options:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -maxBytesPerSecond int
+  -maxBytesPerSecond value
     	The maximum upload speed. There is no limit if it is set to 0
-  -memory.allowedBytes int
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -origin string
     	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
   -snapshot.createURL string
-    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snaphsot/create
   -snapshot.deleteURL string
-    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snaphosts will be automatically deleted. Example: http://victoriametrics:8428/snaphsot/delete
   -snapshotName string
     	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
   -storageDataPath string
@@ -205,7 +245,7 @@ Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` dock
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmbackup/main.go

@@ -13,22 +13,23 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
 	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
 	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
-	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup."+
+	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+
 		"Example: http://victoriametrics:8428/snaphsot/create")
-	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted."+
-		"Example: http://victoriametrics:8428/snaphsot/delete")
+	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+
+		"All created snaphosts will be automatically deleted. Example: http://victoriametrics:8428/snaphsot/delete")
 	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
 		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
 		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
 	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
 	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
-	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+	maxBytesPerSecond = flagutil.NewBytes("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
 )
 
 func main() {
@@ -89,6 +90,9 @@ func main() {
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot create backup: %s", err)
 	}
+	srcFS.MustStop()
+	dstFS.MustStop()
+	originFS.MustStop()
 }
 
 func usage() {
@@ -126,7 +130,7 @@ func newSrcFS() (*fslocal.FS, error) {
 
 	fs := &fslocal.FS{
 		Dir:               snapshotPath,
-		MaxBytesPerSecond: *maxBytesPerSecond,
+		MaxBytesPerSecond: maxBytesPerSecond.N,
 	}
 	if err := fs.Init(); err != nil {
 		return nil, fmt.Errorf("cannot initialize fs: %w", err)

app/vminsert/csvimport/request_handler.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -17,14 +19,18 @@ var (
 
 // InsertHandler processes /api/v1/import/csv requests.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows)
+			return insertRows(rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -38,6 +44,10 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
+		for j := range extraLabels {
+			label := &extraLabels[j]
+			ctx.AddLabel(label.Name, label.Value)
+		}
 		if hasRelabeling {
 			ctx.ApplyRelabeling()
 		}

app/vminsert/influx/request_handler.go

@@ -65,6 +65,7 @@ func insertRows(db string, rows []parser.Row) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		ic.Labels = ic.Labels[:0]
 		hasDBKey := false
 		for j := range r.Tags {
@@ -125,7 +126,6 @@ func insertRows(db string, rows []parser.Row) error {
 				}
 			}
 		}
-		rowsTotal += len(r.Fields)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/main.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheusimport"
@@ -24,6 +26,7 @@ import (
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -51,7 +54,7 @@ var (
 func Init() {
 	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
-
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -83,6 +86,7 @@ func Stop() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 }
 
 // RequestHandler is a handler for Prometheus remote storage write API
@@ -125,6 +129,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
@@ -143,7 +156,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
@@ -169,6 +183,9 @@ var (
 	prometheusimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 	prometheusimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 
+	nativeimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 

app/vminsert/native/request_handler.go

@@ -0,0 +1,115 @@
+package native
+
+import (
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import/native` request.
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	hasRelabeling := relabel.HasRelabeling()
+	mn := &block.MetricName
+	ic.Labels = ic.Labels[:0]
+	ic.AddLabelBytes(nil, mn.MetricGroup)
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		ic.AddLabelBytes(tag.Key, tag.Value)
+	}
+	for j := range extraLabels {
+		label := &extraLabels[j]
+		ic.AddLabel(label.Name, label.Value)
+	}
+	if hasRelabeling {
+		ic.ApplyRelabeling()
+	}
+	if len(ic.Labels) == 0 {
+		// Skip metric without labels.
+		return nil
+	}
+	ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	for j, value := range values {
+		timestamp := timestamps[j]
+		if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
+			return err
+		}
+	}
+	return ic.FlushBufs()
+}
+
+type pushCtx struct {
+	Common        common.InsertCtx
+	metricNameBuf []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/prometheusimport/request_handler.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -17,13 +19,23 @@ var (
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
 		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, isGzipped, insertRows)
+		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -37,6 +49,10 @@ func insertRows(rows []parser.Row) error {
 			tag := &r.Tags[j]
 			ctx.AddLabel(tag.Key, tag.Value)
 		}
+		for j := range extraLabels {
+			label := &extraLabels[j]
+			ctx.AddLabel(label.Name, label.Value)
+		}
 		if hasRelabeling {
 			ctx.ApplyRelabeling()
 		}

app/vminsert/prompush/push.go

@@ -22,10 +22,19 @@ func Push(wr *prompbmarshal.WriteRequest) {
 	tss := wr.Timeseries
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		samplesCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			i++
+			if samplesCount > maxRowsPerBlock {
+				break
+			}
+		}
 		tssBlock := tss
-		if len(tssBlock) > maxRowsPerBlock {
-			tssBlock = tss[:maxRowsPerBlock]
-			tss = tss[maxRowsPerBlock:]
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
 		} else {
 			tss = nil
 		}
@@ -42,6 +51,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 	rowsTotal := 0
 	for i := range tss {
 		ts := &tss[i]
+		rowsTotal += len(ts.Samples)
 		ctx.Labels = ctx.Labels[:0]
 		for j := range ts.Labels {
 			label := &ts.Labels[j]
@@ -62,7 +72,6 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 				return
 			}
 		}
-		rowsTotal += len(ts.Samples)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/promremotewrite/request_handler.go

@@ -36,6 +36,7 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range timeseries {
 		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
 		ctx.Labels = ctx.Labels[:0]
 		srcLabels := ts.Labels
 		for _, srcLabel := range srcLabels {
@@ -50,14 +51,14 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 		}
 		var metricNameRaw []byte
 		var err error
-		for i := range ts.Samples {
-			r := &ts.Samples[i]
+		samples := ts.Samples
+		for i := range samples {
+			r := &samples[i]
 			metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, r.Timestamp, r.Value)
 			if err != nil {
 				return err
 			}
 		}
-		rowsTotal += len(ts.Samples)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/vmimport/request_handler.go

@@ -7,6 +7,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
@@ -22,12 +25,18 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, insertRows)
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(rows []parser.Row) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -41,11 +50,16 @@ func insertRows(rows []parser.Row) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Values)
 		ic.Labels = ic.Labels[:0]
 		for j := range r.Tags {
 			tag := &r.Tags[j]
 			ic.AddLabelBytes(tag.Key, tag.Value)
 		}
+		for j := range extraLabels {
+			label := &extraLabels[j]
+			ic.AddLabel(label.Name, label.Value)
+		}
 		if hasRelabeling {
 			ic.ApplyRelabeling()
 		}
@@ -56,14 +70,15 @@ func insertRows(rows []parser.Row) error {
 		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
 		values := r.Values
 		timestamps := r.Timestamps
-		_ = timestamps[len(values)-1]
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
 		for j, value := range values {
 			timestamp := timestamps[j]
 			if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
 				return err
 			}
 		}
-		rowsTotal += len(values)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vmrestore/README.md

@@ -21,7 +21,7 @@ vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/r
 * `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
   to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
 
-The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup,
+The original `-storageDataPath` directory may contain old files. They will be substituted by the files from backup,
 i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/questions/476041/how-do-i-make-rsync-delete-files-that-have-been-deleted-from-the-source-folder).
 
 
@@ -33,7 +33,44 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 ### Advanced usage
 
-Run `vmrestore -help` in order to see all the available options:
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint.  It is possible to use `vmrestore` with s3 api compatible storages, like  minio, cloudian and other.
+  You have to add custom url endpoint with a flag:
+```
+  # for minio:
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+*  Run `vmrestore -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -42,7 +79,7 @@ Run `vmrestore -help` in order to see all the available options:
     	Path to file with S3 configs. Configs are loaded from default location if not set.
     	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs (default "default")
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
     	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
@@ -62,10 +99,12 @@ Run `vmrestore -help` in order to see all the available options:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -maxBytesPerSecond int
+  -maxBytesPerSecond value
     	The maximum download speed. There is no limit if it is set to 0
-  -memory.allowedBytes int
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -skipBackupCompleteCheck
@@ -102,7 +141,7 @@ Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` do
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmrestore/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -21,7 +22,7 @@ var (
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir "+
 		"is synchronized with -src contents, i.e. it works like 'rsync --delete'")
 	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
-	maxBytesPerSecond       = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+	maxBytesPerSecond       = flagutil.NewBytes("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
 	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file")
 )
 
@@ -51,6 +52,8 @@ func main() {
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot restore from backup: %s", err)
 	}
+	srcFS.MustStop()
+	dstFS.MustStop()
 }
 
 func usage() {
@@ -71,7 +74,7 @@ func newDstFS() (*fslocal.FS, error) {
 	}
 	fs := &fslocal.FS{
 		Dir:               *storageDataPath,
-		MaxBytesPerSecond: *maxBytesPerSecond,
+		MaxBytesPerSecond: maxBytesPerSecond.N,
 	}
 	if err := fs.Init(); err != nil {
 		return nil, fmt.Errorf("cannot initialize local fs: %w", err)

app/vmselect/bufferedwriter/bufferedwriter.go

@@ -0,0 +1,86 @@
+package bufferedwriter
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// Get returns buffered writer for the given w.
+//
+// The writer must be returned to the pool after use by calling Put().
+func Get(w io.Writer) *Writer {
+	v := writerPool.Get()
+	if v == nil {
+		v = &Writer{
+			// By default net/http.Server uses 4KB buffers, which are flushed to client with chunked responses.
+			// These buffers may result in visible overhead for responses exceeding a few megabytes.
+			// So allocate 64Kb buffers.
+			bw: bufio.NewWriterSize(w, 64*1024),
+		}
+	}
+	bw := v.(*Writer)
+	bw.bw.Reset(w)
+	return bw
+}
+
+// Put returns back bw to the pool.
+//
+// bw cannot be used after returning to the pool.
+func Put(bw *Writer) {
+	bw.reset()
+	writerPool.Put(bw)
+}
+
+var writerPool sync.Pool
+
+// Writer is buffered writer, which may be used in order to reduce overhead
+// when sending moderately big responses to http server.
+//
+// Writer methods can be called from concurrently running goroutines.
+// The writer remembers the first occurred error, which can be inspected with Error method.
+type Writer struct {
+	lock sync.Mutex
+	bw   *bufio.Writer
+	err  error
+}
+
+func (bw *Writer) reset() {
+	bw.bw.Reset(nil)
+	bw.err = nil
+}
+
+// Write writes p to bw.
+func (bw *Writer) Write(p []byte) (int, error) {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	if bw.err != nil {
+		return 0, bw.err
+	}
+	n, err := bw.bw.Write(p)
+	if err != nil {
+		bw.err = fmt.Errorf("cannot send %d bytes to client: %w", len(p), err)
+	}
+	return n, bw.err
+}
+
+// Flush flushes bw to the underlying writer.
+func (bw *Writer) Flush() error {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	if bw.err != nil {
+		return bw.err
+	}
+	if err := bw.bw.Flush(); err != nil {
+		bw.err = fmt.Errorf("cannot flush data to client: %w", err)
+	}
+	return bw.err
+}
+
+// Error returns the first occurred error in bw.
+func (bw *Writer) Error() error {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	return bw.err
+}

app/vmselect/graphite/graphite.go

@@ -0,0 +1,419 @@
+package graphite
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// MetricsFindHandler implements /metrics/find handler.
+//
+// See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
+func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	format := r.FormValue("format")
+	if format == "" {
+		format = "treejson"
+	}
+	switch format {
+	case "treejson", "completer":
+	default:
+		return fmt.Errorf(`unexpected "format" query arg: %q; expecting "treejson" or "completer"`, format)
+	}
+	query := r.FormValue("query")
+	if len(query) == 0 {
+		return fmt.Errorf("expecting non-empty `query` arg")
+	}
+	delimiter := r.FormValue("delimiter")
+	if delimiter == "" {
+		delimiter = "."
+	}
+	if len(delimiter) > 1 {
+		return fmt.Errorf("`delimiter` query arg must contain only a single char")
+	}
+	if searchutils.GetBool(r, "automatic_variants") {
+		// See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152
+		query = addAutomaticVariants(query, delimiter)
+	}
+	if format == "completer" {
+		// See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L148
+		query = strings.ReplaceAll(query, "..", ".*")
+		if !strings.HasSuffix(query, "*") {
+			query += "*"
+		}
+	}
+	leavesOnly := searchutils.GetBool(r, "leavesOnly")
+	wildcards := searchutils.GetBool(r, "wildcards")
+	label := r.FormValue("label")
+	if label == "__name__" {
+		label = ""
+	}
+	jsonp := r.FormValue("jsonp")
+	from, err := searchutils.GetTime(r, "from", 0)
+	if err != nil {
+		return err
+	}
+	ct := startTime.UnixNano() / 1e6
+	until, err := searchutils.GetTime(r, "until", ct)
+	if err != nil {
+		return err
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: from,
+		MaxTimestamp: until,
+	}
+	paths, err := metricsFind(tr, label, query, delimiter[0], false, deadline)
+	if err != nil {
+		return err
+	}
+	if leavesOnly {
+		paths = filterLeaves(paths, delimiter)
+	}
+	paths = deduplicatePaths(paths, delimiter)
+	sortPaths(paths, delimiter)
+	contentType := "application/json"
+	if jsonp != "" {
+		contentType = "text/javascript"
+	}
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsFindResponse(bw, paths, delimiter, format, wildcards, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	metricsFindDuration.UpdateDuration(startTime)
+	return nil
+}
+
+func deduplicatePaths(paths []string, delimiter string) []string {
+	if len(paths) == 0 {
+		return nil
+	}
+	sort.Strings(paths)
+	dst := paths[:1]
+	for _, path := range paths[1:] {
+		prevPath := dst[len(dst)-1]
+		if path == prevPath {
+			// Skip duplicate path.
+			continue
+		}
+		dst = append(dst, path)
+	}
+	return dst
+}
+
+// MetricsExpandHandler implements /metrics/expand handler.
+//
+// See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
+func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	queries := r.Form["query"]
+	if len(queries) == 0 {
+		return fmt.Errorf("missing `query` arg")
+	}
+	groupByExpr := searchutils.GetBool(r, "groupByExpr")
+	leavesOnly := searchutils.GetBool(r, "leavesOnly")
+	label := r.FormValue("label")
+	if label == "__name__" {
+		label = ""
+	}
+	delimiter := r.FormValue("delimiter")
+	if delimiter == "" {
+		delimiter = "."
+	}
+	if len(delimiter) > 1 {
+		return fmt.Errorf("`delimiter` query arg must contain only a single char")
+	}
+	jsonp := r.FormValue("jsonp")
+	from, err := searchutils.GetTime(r, "from", 0)
+	if err != nil {
+		return err
+	}
+	ct := startTime.UnixNano() / 1e6
+	until, err := searchutils.GetTime(r, "until", ct)
+	if err != nil {
+		return err
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: from,
+		MaxTimestamp: until,
+	}
+	m := make(map[string][]string, len(queries))
+	for _, query := range queries {
+		paths, err := metricsFind(tr, label, query, delimiter[0], true, deadline)
+		if err != nil {
+			return err
+		}
+		if leavesOnly {
+			paths = filterLeaves(paths, delimiter)
+		}
+		m[query] = paths
+	}
+	contentType := "application/json"
+	if jsonp != "" {
+		contentType = "text/javascript"
+	}
+	w.Header().Set("Content-Type", contentType)
+	if groupByExpr {
+		for _, paths := range m {
+			sortPaths(paths, delimiter)
+		}
+		WriteMetricsExpandResponseByQuery(w, m, jsonp)
+		return nil
+	}
+	paths := m[queries[0]]
+	if len(m) > 1 {
+		pathsSet := make(map[string]struct{})
+		for _, paths := range m {
+			for _, path := range paths {
+				pathsSet[path] = struct{}{}
+			}
+		}
+		paths = make([]string, 0, len(pathsSet))
+		for path := range pathsSet {
+			paths = append(paths, path)
+		}
+	}
+	sortPaths(paths, delimiter)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsExpandResponseFlat(bw, paths, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	metricsExpandDuration.UpdateDuration(startTime)
+	return nil
+}
+
+// MetricsIndexHandler implements /metrics/index.json handler.
+//
+// See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
+func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	jsonp := r.FormValue("jsonp")
+	metricNames, err := netstorage.GetLabelValues("__name__", deadline)
+	if err != nil {
+		return fmt.Errorf(`cannot obtain metric names: %w`, err)
+	}
+	contentType := "application/json"
+	if jsonp != "" {
+		contentType = "text/javascript"
+	}
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsIndexResponse(bw, metricNames, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	metricsIndexDuration.UpdateDuration(startTime)
+	return nil
+}
+
+// metricsFind searches for label values that match the given query.
+func metricsFind(tr storage.TimeRange, label, query string, delimiter byte, isExpand bool, deadline searchutils.Deadline) ([]string, error) {
+	n := strings.IndexAny(query, "*{[")
+	if n < 0 || n == len(query)-1 && strings.HasSuffix(query, "*") {
+		expandTail := n >= 0
+		if expandTail {
+			query = query[:len(query)-1]
+		}
+		suffixes, err := netstorage.GetTagValueSuffixes(tr, label, query, delimiter, deadline)
+		if err != nil {
+			return nil, err
+		}
+		if len(suffixes) == 0 {
+			return nil, nil
+		}
+		if !expandTail && len(query) > 0 && query[len(query)-1] == delimiter {
+			return []string{query}, nil
+		}
+		results := make([]string, 0, len(suffixes))
+		for _, suffix := range suffixes {
+			if expandTail || len(suffix) == 0 || len(suffix) == 1 && suffix[0] == delimiter {
+				results = append(results, query+suffix)
+			}
+		}
+		return results, nil
+	}
+	subquery := query[:n] + "*"
+	paths, err := metricsFind(tr, label, subquery, delimiter, isExpand, deadline)
+	if err != nil {
+		return nil, err
+	}
+	tail := ""
+	suffix := query[n:]
+	if m := strings.IndexByte(suffix, delimiter); m >= 0 {
+		tail = suffix[m+1:]
+		suffix = suffix[:m+1]
+	}
+	qPrefix := query[:n] + suffix
+	rePrefix, err := getRegexpForQuery(qPrefix, delimiter)
+	if err != nil {
+		return nil, fmt.Errorf("cannot convert query %q to regexp: %w", qPrefix, err)
+	}
+	results := make([]string, 0, len(paths))
+	for _, path := range paths {
+		if !rePrefix.MatchString(path) {
+			continue
+		}
+		if tail == "" {
+			results = append(results, path)
+			continue
+		}
+		subquery := path + tail
+		fullPaths, err := metricsFind(tr, label, subquery, delimiter, isExpand, deadline)
+		if err != nil {
+			return nil, err
+		}
+		if isExpand {
+			results = append(results, fullPaths...)
+		} else {
+			for _, fullPath := range fullPaths {
+				results = append(results, qPrefix+fullPath[len(path):])
+			}
+		}
+	}
+	return results, nil
+}
+
+var (
+	metricsFindDuration   = metrics.NewSummary(`vm_request_duration_seconds{path="/metrics/find"}`)
+	metricsExpandDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/metrics/expand"}`)
+	metricsIndexDuration  = metrics.NewSummary(`vm_request_duration_seconds{path="/metrics/index.json"}`)
+)
+
+func addAutomaticVariants(query, delimiter string) string {
+	// See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152
+	parts := strings.Split(query, delimiter)
+	for i, part := range parts {
+		if strings.Contains(part, ",") && !strings.Contains(part, "{") {
+			parts[i] = "{" + part + "}"
+		}
+	}
+	return strings.Join(parts, delimiter)
+}
+
+func filterLeaves(paths []string, delimiter string) []string {
+	leaves := paths[:0]
+	for _, path := range paths {
+		if !strings.HasSuffix(path, delimiter) {
+			leaves = append(leaves, path)
+		}
+	}
+	return leaves
+}
+
+func sortPaths(paths []string, delimiter string) {
+	sort.Slice(paths, func(i, j int) bool {
+		a, b := paths[i], paths[j]
+		isNodeA := strings.HasSuffix(a, delimiter)
+		isNodeB := strings.HasSuffix(b, delimiter)
+		if isNodeA == isNodeB {
+			return a < b
+		}
+		return isNodeA
+	})
+}
+
+func getRegexpForQuery(query string, delimiter byte) (*regexp.Regexp, error) {
+	regexpCacheLock.Lock()
+	defer regexpCacheLock.Unlock()
+
+	k := regexpCacheKey{
+		query:     query,
+		delimiter: delimiter,
+	}
+	if re := regexpCache[k]; re != nil {
+		return re.re, re.err
+	}
+	a := make([]string, 0, len(query))
+	quotedDelimiter := regexp.QuoteMeta(string([]byte{delimiter}))
+	tillNextDelimiter := "[^" + quotedDelimiter + "]*"
+	for i := 0; i < len(query); i++ {
+		switch query[i] {
+		case '*':
+			a = append(a, tillNextDelimiter)
+		case '{':
+			tmp := query[i+1:]
+			if n := strings.IndexByte(tmp, '}'); n < 0 {
+				a = append(a, regexp.QuoteMeta(query[i:]))
+				i = len(query)
+			} else {
+				a = append(a, "(?:")
+				opts := strings.Split(tmp[:n], ",")
+				for j, opt := range opts {
+					opts[j] = regexp.QuoteMeta(opt)
+				}
+				a = append(a, strings.Join(opts, "|"))
+				a = append(a, ")")
+				i += n + 1
+			}
+		case '[':
+			tmp := query[i:]
+			if n := strings.IndexByte(tmp, ']'); n < 0 {
+				a = append(a, regexp.QuoteMeta(query[i:]))
+				i = len(query)
+			} else {
+				a = append(a, tmp[:n+1])
+				i += n
+			}
+		default:
+			a = append(a, regexp.QuoteMeta(query[i:i+1]))
+		}
+	}
+	s := strings.Join(a, "")
+	if !strings.HasSuffix(s, quotedDelimiter) {
+		s += quotedDelimiter + "?"
+	}
+	s = "^(?:" + s + ")$"
+	re, err := regexp.Compile(s)
+	regexpCache[k] = &regexpCacheEntry{
+		re:  re,
+		err: err,
+	}
+	if len(regexpCache) >= maxRegexpCacheSize {
+		for k := range regexpCache {
+			if len(regexpCache) < maxRegexpCacheSize {
+				break
+			}
+			delete(regexpCache, k)
+		}
+	}
+	return re, err
+}
+
+type regexpCacheEntry struct {
+	re  *regexp.Regexp
+	err error
+}
+
+type regexpCacheKey struct {
+	query     string
+	delimiter byte
+}
+
+var regexpCache = make(map[regexpCacheKey]*regexpCacheEntry)
+var regexpCacheLock sync.Mutex
+
+const maxRegexpCacheSize = 10000

app/vmselect/graphite/graphite_test.go

@@ -0,0 +1,75 @@
+package graphite
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGetRegexpForQuery(t *testing.T) {
+	f := func(query string, delimiter byte, reExpected string) {
+		t.Helper()
+		re, err := getRegexpForQuery(query, delimiter)
+		if err != nil {
+			t.Fatalf("unexpected error in getRegexpForQuery(%q): %s", query, err)
+		}
+		reStr := re.String()
+		if reStr != reExpected {
+			t.Fatalf("unexpected regexp for query=%q, delimiter=%c; got %s; want %s", query, delimiter, reStr, reExpected)
+		}
+	}
+	f("", '.', `^(?:\.?)$`)
+	f("foobar", '.', `^(?:foobar\.?)$`)
+	f("*", '.', `^(?:[^\.]*\.?)$`)
+	f("*", '_', `^(?:[^_]*_?)$`)
+	f("foo.*.bar", '.', `^(?:foo\.[^\.]*\.bar\.?)$`)
+	f("fo*b{ar,aaa}[a-z]xx*.d", '.', `^(?:fo[^\.]*b(?:ar|aaa)[a-z]xx[^\.]*\.d\.?)$`)
+	f("fo*b{ar,aaa}[a-z]xx*_d", '_', `^(?:fo[^_]*b(?:ar|aaa)[a-z]xx[^_]*_d_?)$`)
+	f("foo.[ab]*z", '.', `^(?:foo\.[ab][^\.]*z\.?)$`)
+	f("foo_[ab]*", '_', `^(?:foo_[ab][^_]*_?)$`)
+	f("foo_[ab]_", '_', `^(?:foo_[ab]_)$`)
+	f("foo.[ab].", '.', `^(?:foo\.[ab]\.)$`)
+}
+
+func TestSortPaths(t *testing.T) {
+	f := func(paths []string, delimiter string, pathsSortedExpected []string) {
+		t.Helper()
+		sortPaths(paths, delimiter)
+		if !reflect.DeepEqual(paths, pathsSortedExpected) {
+			t.Fatalf("unexpected sortPaths result;\ngot\n%q\nwant\n%q", paths, pathsSortedExpected)
+		}
+	}
+	f([]string{"foo", "bar"}, ".", []string{"bar", "foo"})
+	f([]string{"foo.", "bar", "aa", "ab."}, ".", []string{"ab.", "foo.", "aa", "bar"})
+	f([]string{"foo.", "bar", "aa", "ab."}, "_", []string{"aa", "ab.", "bar", "foo."})
+}
+
+func TestFilterLeaves(t *testing.T) {
+	f := func(paths []string, delimiter string, leavesExpected []string) {
+		t.Helper()
+		leaves := filterLeaves(paths, delimiter)
+		if !reflect.DeepEqual(leaves, leavesExpected) {
+			t.Fatalf("unexpected leaves; got\n%q\nwant\n%q", leaves, leavesExpected)
+		}
+	}
+	f([]string{"foo", "bar"}, ".", []string{"foo", "bar"})
+	f([]string{"a.", ".", "bc"}, ".", []string{"bc"})
+	f([]string{"a.", ".", "bc"}, "_", []string{"a.", ".", "bc"})
+	f([]string{"a_", "_", "bc"}, "_", []string{"bc"})
+	f([]string{"foo.", "bar."}, ".", []string{})
+}
+
+func TestAddAutomaticVariants(t *testing.T) {
+	f := func(query, delimiter, resultExpected string) {
+		t.Helper()
+		result := addAutomaticVariants(query, delimiter)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for addAutomaticVariants(%q, delimiter=%q); got %q; want %q", query, delimiter, result, resultExpected)
+		}
+	}
+	f("", ".", "")
+	f("foobar", ".", "foobar")
+	f("foo,bar.baz", ".", "{foo,bar}.baz")
+	f("foo,bar.baz", "_", "{foo,bar.baz}")
+	f("foo,bar_baz*", "_", "{foo,bar}_baz*")
+	f("foo.bar,baz,aa.bb,cc", ".", "foo.{bar,baz,aa}.{bb,cc}")
+}

app/vmselect/graphite/metrics_expand_response.qtpl

@@ -0,0 +1,38 @@
+{% stripspace %}
+
+MetricsExpandResponseByQuery generates response for /metrics/expand?groupByExpr=1 .
+See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
+{% func MetricsExpandResponseByQuery(m map[string][]string, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	{
+		"results":{
+			{% code i := 0 %}
+			{% for query, paths := range m %}
+				{%q= query %}:{%= metricPaths(paths) %}
+				{% code i++ %}
+				{% if i < len(m) %},{% endif %}
+			{% endfor %}
+		}
+	}
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+
+MetricsExpandResponseFlat generates response for /metrics/expand?groupByExpr=0 .
+See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
+{% func MetricsExpandResponseFlat(paths []string, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	{%= metricPaths(paths) %}
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+{% func metricPaths(paths []string) %}
+	[
+		{% for i, path := range paths %}
+			{%q= path %}
+			{% if i+1 < len(paths) %},{% endif %}
+		{% endfor %}
+	]
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/metrics_expand_response.qtpl.go

@@ -0,0 +1,187 @@
+// Code generated by qtc from "metrics_expand_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// MetricsExpandResponseByQuery generates response for /metrics/expand?groupByExpr=1 .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:5
+func StreamMetricsExpandResponseByQuery(qw422016 *qt422016.Writer, m map[string][]string, jsonp string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:6
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:6
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:6
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:6
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:6
+	qw422016.N().S(`{"results":{`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:9
+	i := 0
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:10
+	for query, paths := range m {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:11
+		qw422016.N().Q(query)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:11
+		qw422016.N().S(`:`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:11
+		streammetricPaths(qw422016, paths)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:12
+		i++
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:13
+		if i < len(m) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:13
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:13
+		}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:14
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:14
+	qw422016.N().S(`}}`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:17
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:17
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:17
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+func WriteMetricsExpandResponseByQuery(qq422016 qtio422016.Writer, m map[string][]string, jsonp string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	StreamMetricsExpandResponseByQuery(qw422016, m, jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+func MetricsExpandResponseByQuery(m map[string][]string, jsonp string) string {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	WriteMetricsExpandResponseByQuery(qb422016, m, jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+	return qs422016
+//line app/vmselect/graphite/metrics_expand_response.qtpl:18
+}
+
+// MetricsExpandResponseFlat generates response for /metrics/expand?groupByExpr=0 .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:23
+func StreamMetricsExpandResponseFlat(qw422016 *qt422016.Writer, paths []string, jsonp string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:24
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:24
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:24
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:24
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:25
+	streammetricPaths(qw422016, paths)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:26
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:26
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:26
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+func WriteMetricsExpandResponseFlat(qq422016 qtio422016.Writer, paths []string, jsonp string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	StreamMetricsExpandResponseFlat(qw422016, paths, jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+func MetricsExpandResponseFlat(paths []string, jsonp string) string {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	WriteMetricsExpandResponseFlat(qb422016, paths, jsonp)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+	return qs422016
+//line app/vmselect/graphite/metrics_expand_response.qtpl:27
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:29
+func streammetricPaths(qw422016 *qt422016.Writer, paths []string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:29
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:31
+	for i, path := range paths {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:32
+		qw422016.N().Q(path)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:33
+		if i+1 < len(paths) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:33
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:33
+		}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:34
+	}
+//line app/vmselect/graphite/metrics_expand_response.qtpl:34
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+func writemetricPaths(qq422016 qtio422016.Writer, paths []string) {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	streammetricPaths(qw422016, paths)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+}
+
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+func metricPaths(paths []string) string {
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	writemetricPaths(qb422016, paths)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+	return qs422016
+//line app/vmselect/graphite/metrics_expand_response.qtpl:36
+}

app/vmselect/graphite/metrics_find_response.qtpl

@@ -0,0 +1,138 @@
+{% import (
+	"sort"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+) %}
+
+{% stripspace %}
+
+MetricsFindResponse generates response for /metrics/find .
+See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
+{% func MetricsFindResponse(paths []string, delimiter, format string, addWildcards bool, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	{% switch format %}
+	{% case "completer" %}
+		{%= metricsFindResponseCompleter(paths, delimiter, addWildcards) %}
+	{% case "treejson" %}
+		{%= metricsFindResponseTreeJSON(paths, delimiter, addWildcards) %}
+	{% default %}
+		{% code logger.Panicf("BUG: unexpected format=%q", format) %}
+	{% endswitch %}
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+{% func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards bool) %}
+{
+	"metrics":[
+		{% for i, path := range paths %}
+		{
+			"path": {%q= path %},
+			"name": {%= metricPathName(path, delimiter) %},
+			"is_leaf": {% if strings.HasSuffix(path, delimiter) %}0{% else %}1{% endif %}
+		}
+		{% if i+1 < len(paths) %},{% endif %}
+		{% endfor %}
+		{% if addWildcards && len(paths) > 1 %}
+		,{
+			"name": "*"
+		}
+		{% endif %}
+	]
+}
+{% endfunc %}
+
+{% func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) %}
+[
+{% code
+	if len(paths) > 1 {
+		sort.Strings(paths)
+		// Substitute `path` and `path<delimiter>` with `path<delimiter><delimiter>`.
+		// Such path is treated specially during rendering - see code below for details.
+		dst := paths[:1]
+		for _, path := range paths[1:] {
+			prevPath := dst[len(dst)-1]
+			if len(path) == len(prevPath)+1 && strings.HasSuffix(path, delimiter) && strings.HasPrefix(path, prevPath) {
+				// The path is equivalent to <prevPath> + <delimiter>
+				// Overwrite the prevPath with <path> + <delimiter> as carbonapi does.
+				// I.e. the resulting path ends with double delimiter.
+				// Such path is treated specially during rendering - see metrics_find_response.qtpl for details.
+				dst[len(dst)-1] = path + delimiter
+				continue
+			}
+			dst = append(dst, path)
+		}
+		paths = dst
+	}
+%}
+	{% for i, path := range paths %}
+	{
+		{% code
+			id := path
+			allowChildren := "0"
+			isLeaf := "1"
+			if strings.HasSuffix(id, delimiter) {
+				if strings.HasSuffix(id[:len(id)-1], delimiter) {
+					// Special case when id ends with double delimiter.
+					// See the code above for details.
+					id = id[:len(id)-2]
+				}
+				allowChildren = "1"
+				isLeaf = "0"
+			}
+		%}
+		"id": {%q= id %},
+		"text": {%= metricPathName(path, delimiter) %},
+		"allowChildren": {%s= allowChildren %},
+		"expandable": {%s= allowChildren %},
+		"leaf": {%s= isLeaf %}
+	}
+	{% if i+1 < len(paths) %},{% endif %}
+	{% endfor %}
+	{% if addWildcards && len(paths) > 1 %}
+	,{
+		{% code
+			path := paths[0]
+			for strings.HasSuffix(path, delimiter) {
+				path = path[:len(path)-1]
+			}
+			id := ""
+			if n := strings.LastIndexByte(path, delimiter[0]); n >= 0 {
+				id = path[:n+1]
+			}
+			id += "*"
+
+			allowChildren := "0"
+			isLeaf := "1"
+			for _, path := range paths {
+				if strings.HasSuffix(path, delimiter) {
+					allowChildren = "1"
+					isLeaf = "0"
+					break
+				}
+			}
+		%}
+		"id": {%q= id %},
+		"text": "*",
+		"allowChildren": {%s= allowChildren %},
+		"expandable": {%s= allowChildren %},
+		"leaf": {%s= isLeaf %}
+	}
+	{% endif %}
+]
+{% endfunc %}
+
+{% func metricPathName(path, delimiter string) %}
+	{% code
+		name := path
+		for strings.HasSuffix(name, delimiter) {
+			name = name[:len(name)-1]
+		}
+		if n := strings.LastIndexByte(name, delimiter[0]); n >= 0 {
+			name = name[n+1:]
+		}
+	%}
+	{%q= name %}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/metrics_find_response.qtpl.go

@@ -0,0 +1,354 @@
+// Code generated by qtc from "metrics_find_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:1
+package graphite
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:1
+import (
+	"sort"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// MetricsFindResponse generates response for /metrics/find .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
+func StreamMetricsFindResponse(qw422016 *qt422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:14
+	switch format {
+//line app/vmselect/graphite/metrics_find_response.qtpl:15
+	case "completer":
+//line app/vmselect/graphite/metrics_find_response.qtpl:16
+		streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:17
+	case "treejson":
+//line app/vmselect/graphite/metrics_find_response.qtpl:18
+		streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:19
+	default:
+//line app/vmselect/graphite/metrics_find_response.qtpl:20
+		logger.Panicf("BUG: unexpected format=%q", format)
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:21
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:22
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_find_response.qtpl:22
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:22
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+func WriteMetricsFindResponse(qq422016 qtio422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	StreamMetricsFindResponse(qw422016, paths, delimiter, format, addWildcards, jsonp)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+func MetricsFindResponse(paths []string, delimiter, format string, addWildcards bool, jsonp string) string {
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	WriteMetricsFindResponse(qb422016, paths, delimiter, format, addWildcards, jsonp)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+	return qs422016
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:25
+func streammetricsFindResponseCompleter(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:25
+	qw422016.N().S(`{"metrics":[`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:28
+	for i, path := range paths {
+//line app/vmselect/graphite/metrics_find_response.qtpl:28
+		qw422016.N().S(`{"path":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:30
+		qw422016.N().Q(path)
+//line app/vmselect/graphite/metrics_find_response.qtpl:30
+		qw422016.N().S(`,"name":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:31
+		streammetricPathName(qw422016, path, delimiter)
+//line app/vmselect/graphite/metrics_find_response.qtpl:31
+		qw422016.N().S(`,"is_leaf":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+		if strings.HasSuffix(path, delimiter) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+			qw422016.N().S(`0`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+		} else {
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+			qw422016.N().S(`1`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+		}
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:34
+		if i+1 < len(paths) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:34
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:34
+		}
+//line app/vmselect/graphite/metrics_find_response.qtpl:35
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:36
+	if addWildcards && len(paths) > 1 {
+//line app/vmselect/graphite/metrics_find_response.qtpl:36
+		qw422016.N().S(`,{"name": "*"}`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:40
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:40
+	qw422016.N().S(`]}`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+func writemetricsFindResponseCompleter(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards bool) string {
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	writemetricsFindResponseCompleter(qb422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+	return qs422016
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:45
+func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:45
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:48
+	if len(paths) > 1 {
+		sort.Strings(paths)
+		// Substitute `path` and `path<delimiter>` with `path<delimiter><delimiter>`.
+		// Such path is treated specially during rendering - see code below for details.
+		dst := paths[:1]
+		for _, path := range paths[1:] {
+			prevPath := dst[len(dst)-1]
+			if len(path) == len(prevPath)+1 && strings.HasSuffix(path, delimiter) && strings.HasPrefix(path, prevPath) {
+				// The path is equivalent to <prevPath> + <delimiter>
+				// Overwrite the prevPath with <path> + <delimiter> as carbonapi does.
+				// I.e. the resulting path ends with double delimiter.
+				// Such path is treated specially during rendering - see metrics_find_response.qtpl for details.
+				dst[len(dst)-1] = path + delimiter
+				continue
+			}
+			dst = append(dst, path)
+		}
+		paths = dst
+	}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:68
+	for i, path := range paths {
+//line app/vmselect/graphite/metrics_find_response.qtpl:68
+		qw422016.N().S(`{`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:71
+		id := path
+		allowChildren := "0"
+		isLeaf := "1"
+		if strings.HasSuffix(id, delimiter) {
+			if strings.HasSuffix(id[:len(id)-1], delimiter) {
+				// Special case when id ends with double delimiter.
+				// See the code above for details.
+				id = id[:len(id)-2]
+			}
+			allowChildren = "1"
+			isLeaf = "0"
+		}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:83
+		qw422016.N().S(`"id":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:84
+		qw422016.N().Q(id)
+//line app/vmselect/graphite/metrics_find_response.qtpl:84
+		qw422016.N().S(`,"text":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:85
+		streammetricPathName(qw422016, path, delimiter)
+//line app/vmselect/graphite/metrics_find_response.qtpl:85
+		qw422016.N().S(`,"allowChildren":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:86
+		qw422016.N().S(allowChildren)
+//line app/vmselect/graphite/metrics_find_response.qtpl:86
+		qw422016.N().S(`,"expandable":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:87
+		qw422016.N().S(allowChildren)
+//line app/vmselect/graphite/metrics_find_response.qtpl:87
+		qw422016.N().S(`,"leaf":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:88
+		qw422016.N().S(isLeaf)
+//line app/vmselect/graphite/metrics_find_response.qtpl:88
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
+		if i+1 < len(paths) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
+		}
+//line app/vmselect/graphite/metrics_find_response.qtpl:91
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:92
+	if addWildcards && len(paths) > 1 {
+//line app/vmselect/graphite/metrics_find_response.qtpl:92
+		qw422016.N().S(`,{`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:95
+		path := paths[0]
+		for strings.HasSuffix(path, delimiter) {
+			path = path[:len(path)-1]
+		}
+		id := ""
+		if n := strings.LastIndexByte(path, delimiter[0]); n >= 0 {
+			id = path[:n+1]
+		}
+		id += "*"
+
+		allowChildren := "0"
+		isLeaf := "1"
+		for _, path := range paths {
+			if strings.HasSuffix(path, delimiter) {
+				allowChildren = "1"
+				isLeaf = "0"
+				break
+			}
+		}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:114
+		qw422016.N().S(`"id":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:115
+		qw422016.N().Q(id)
+//line app/vmselect/graphite/metrics_find_response.qtpl:115
+		qw422016.N().S(`,"text": "*","allowChildren":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:117
+		qw422016.N().S(allowChildren)
+//line app/vmselect/graphite/metrics_find_response.qtpl:117
+		qw422016.N().S(`,"expandable":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:118
+		qw422016.N().S(allowChildren)
+//line app/vmselect/graphite/metrics_find_response.qtpl:118
+		qw422016.N().S(`,"leaf":`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:119
+		qw422016.N().S(isLeaf)
+//line app/vmselect/graphite/metrics_find_response.qtpl:119
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:121
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:121
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+func writemetricsFindResponseTreeJSON(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) string {
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	writemetricsFindResponseTreeJSON(qb422016, paths, delimiter, addWildcards)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+	return qs422016
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:125
+func streammetricPathName(qw422016 *qt422016.Writer, path, delimiter string) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:127
+	name := path
+	for strings.HasSuffix(name, delimiter) {
+		name = name[:len(name)-1]
+	}
+	if n := strings.LastIndexByte(name, delimiter[0]); n >= 0 {
+		name = name[n+1:]
+	}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:135
+	qw422016.N().Q(name)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+func writemetricPathName(qq422016 qtio422016.Writer, path, delimiter string) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	streammetricPathName(qw422016, path, delimiter)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+func metricPathName(path, delimiter string) string {
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	writemetricPathName(qb422016, path, delimiter)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+	return qs422016
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
+}

app/vmselect/graphite/metrics_index_response.qtpl

@@ -0,0 +1,11 @@
+{% stripspace %}
+
+MetricsIndexResponse generates response for /metrics/index.json .
+See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
+{% func MetricsIndexResponse(metricNames []string, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	{%= metricPaths(metricNames) %}
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/metrics_index_response.qtpl.go

@@ -0,0 +1,67 @@
+// Code generated by qtc from "metrics_index_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// MetricsIndexResponse generates response for /metrics/index.json .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:5
+func StreamMetricsIndexResponse(qw422016 *qt422016.Writer, metricNames []string, jsonp string) {
+//line app/vmselect/graphite/metrics_index_response.qtpl:6
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_index_response.qtpl:6
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/metrics_index_response.qtpl:6
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/metrics_index_response.qtpl:6
+	}
+//line app/vmselect/graphite/metrics_index_response.qtpl:7
+	streammetricPaths(qw422016, metricNames)
+//line app/vmselect/graphite/metrics_index_response.qtpl:8
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_index_response.qtpl:8
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/metrics_index_response.qtpl:8
+	}
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+}
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+func WriteMetricsIndexResponse(qq422016 qtio422016.Writer, metricNames []string, jsonp string) {
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	StreamMetricsIndexResponse(qw422016, metricNames, jsonp)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+}
+
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+func MetricsIndexResponse(metricNames []string, jsonp string) string {
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	WriteMetricsIndexResponse(qb422016, metricNames, jsonp)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+	return qs422016
+//line app/vmselect/graphite/metrics_index_response.qtpl:9
+}

app/vmselect/main.go

@@ -9,8 +9,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -22,7 +24,8 @@ var (
 	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
-	maxQueueDuration  = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
+		"limit is reached; see also -search.maxQueryDuration")
 	resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 )
 
@@ -76,7 +79,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		t := timerpool.Get(*maxQueueDuration)
+		d := searchutils.GetMaxQueryDuration(r)
+		if d > *maxQueueDuration {
+			d = *maxQueueDuration
+		}
+		t := timerpool.Get(d)
 		select {
 		case concurrencyCh <- struct{}{}:
 			timerpool.Put(t)
@@ -86,8 +93,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			concurrencyLimitTimeout.Inc()
 			err := &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
-					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
-					*maxConcurrentRequests, *maxQueueDuration),
+					"increase `-search.maxQueueDuration`; increase `-search.maxQueryDuration`; increase `-search.maxConcurrentRequests`; "+
+					"increase server capacity",
+					*maxConcurrentRequests, d),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
@@ -195,6 +203,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/api/v1/export/csv":
+		exportCSVRequests.Inc()
+		if err := prometheus.ExportCSVHandler(startTime, w, r); err != nil {
+			exportCSVErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/api/v1/export/native":
+		exportNativeRequests.Inc()
+		if err := prometheus.ExportNativeHandler(startTime, w, r); err != nil {
+			exportNativeErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
 	case "/federate":
 		federateRequests.Inc()
 		if err := prometheus.FederateHandler(startTime, w, r); err != nil {
@@ -203,6 +227,33 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/metrics/find", "/metrics/find/":
+		graphiteMetricsFindRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := graphite.MetricsFindHandler(startTime, w, r); err != nil {
+			graphiteMetricsFindErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/metrics/expand", "/metrics/expand/":
+		graphiteMetricsExpandRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := graphite.MetricsExpandHandler(startTime, w, r); err != nil {
+			graphiteMetricsExpandErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/metrics/index.json", "/metrics/index.json/":
+		graphiteMetricsIndexRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := graphite.MetricsIndexHandler(startTime, w, r); err != nil {
+			graphiteMetricsIndexErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
 	case "/api/v1/rules":
 		// Return dumb placeholder
 		rulesRequests.Inc()
@@ -286,9 +337,24 @@ var (
 	exportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export"}`)
 	exportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export"}`)
 
+	exportCSVRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export/csv"}`)
+	exportCSVErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export/csv"}`)
+
+	exportNativeRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export/native"}`)
+	exportNativeErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export/native"}`)
+
 	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/federate"}`)
 	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/federate"}`)
 
+	graphiteMetricsFindRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/find"}`)
+	graphiteMetricsFindErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/find"}`)
+
+	graphiteMetricsExpandRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/expand"}`)
+	graphiteMetricsExpandErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/expand"}`)
+
+	graphiteMetricsIndexRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/index.json"}`)
+	graphiteMetricsIndexErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/index.json"}`)
+
 	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
 	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)

app/vmselect/netstorage/netstorage.go

@@ -8,11 +8,11 @@ import (
 	"runtime"
 	"sort"
 	"sync"
-	"time"
+	"sync/atomic"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -20,9 +20,10 @@ import (
 )
 
 var (
-	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search")
-	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search")
-	maxMetricsPerSearch   = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
+	maxTagKeysPerSearch          = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned from /api/v1/labels")
+	maxTagValuesPerSearch        = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned from /api/v1/label/<label_name>/values")
+	maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
+	maxMetricsPerSearch          = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
 )
 
 // Result is a single timeseries result.
@@ -52,7 +53,7 @@ func (r *Result) reset() {
 type Results struct {
 	tr        storage.TimeRange
 	fetchData bool
-	deadline  Deadline
+	deadline  searchutils.Deadline
 
 	packedTimeseries []packedTimeseries
 	sr               *storage.Search
@@ -76,10 +77,11 @@ func (rss *Results) mustClose() {
 var timeseriesWorkCh = make(chan *timeseriesWork, gomaxprocs*16)
 
 type timeseriesWork struct {
-	rss    *Results
-	pts    *packedTimeseries
-	f      func(rs *Result, workerID uint)
-	doneCh chan error
+	mustStop uint64
+	rss      *Results
+	pts      *packedTimeseries
+	f        func(rs *Result, workerID uint) error
+	doneCh   chan error
 
 	rowsProcessed int
 }
@@ -99,12 +101,19 @@ func timeseriesWorker(workerID uint) {
 			tsw.doneCh <- fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 			continue
 		}
+		if atomic.LoadUint64(&tsw.mustStop) != 0 {
+			tsw.doneCh <- nil
+			continue
+		}
 		if err := tsw.pts.Unpack(&rs, rss.tr, rss.fetchData); err != nil {
 			tsw.doneCh <- fmt.Errorf("error during time series unpacking: %w", err)
 			continue
 		}
 		if len(rs.Timestamps) > 0 || !rss.fetchData {
-			tsw.f(&rs, workerID)
+			if err := tsw.f(&rs, workerID); err != nil {
+				tsw.doneCh <- err
+				continue
+			}
 		}
 		tsw.rowsProcessed = len(rs.Values)
 		tsw.doneCh <- nil
@@ -121,9 +130,10 @@ func timeseriesWorker(workerID uint) {
 //
 // f shouldn't hold references to rs after returning.
 // workerID is the id of the worker goroutine that calls f.
+// Data processing is immediately stopped if f returns non-nil error.
 //
 // rss becomes unusable after the call to RunParallel.
-func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
+func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 	defer rss.mustClose()
 
 	// Feed workers with work.
@@ -149,6 +159,10 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			// Return just the first error, since other errors
 			// are likely duplicate the first error.
 			firstErr = err
+			// Notify all the the tsws that they shouldn't be executed.
+			for _, tsw := range tsws {
+				atomic.StoreUint64(&tsw.mustStop, 1)
+			}
 		}
 		rowsProcessedTotal += tsw.rowsProcessed
 	}
@@ -176,10 +190,9 @@ type unpackWorkItem struct {
 }
 
 type unpackWork struct {
-	ws        []unpackWorkItem
-	fetchData bool
-	sbs       []*sortBlock
-	doneCh    chan error
+	ws     []unpackWorkItem
+	sbs    []*sortBlock
+	doneCh chan error
 }
 
 func (upw *unpackWork) reset() {
@@ -190,7 +203,6 @@ func (upw *unpackWork) reset() {
 		w.tr = storage.TimeRange{}
 	}
 	upw.ws = upw.ws[:0]
-	upw.fetchData = false
 	sbs := upw.sbs
 	for i := range sbs {
 		sbs[i] = nil
@@ -201,10 +213,10 @@ func (upw *unpackWork) reset() {
 	}
 }
 
-func (upw *unpackWork) unpack() {
+func (upw *unpackWork) unpack(tmpBlock *storage.Block) {
 	for _, w := range upw.ws {
 		sb := getSortBlock()
-		if err := sb.unpackFrom(w.br, w.tr, upw.fetchData); err != nil {
+		if err := sb.unpackFrom(tmpBlock, w.br, w.tr); err != nil {
 			putSortBlock(sb)
 			upw.doneCh <- fmt.Errorf("cannot unpack block: %w", err)
 			return
@@ -238,8 +250,9 @@ func init() {
 }
 
 func unpackWorker() {
+	var tmpBlock storage.Block
 	for upw := range unpackWorkCh {
-		upw.unpack()
+		upw.unpack(&tmpBlock)
 	}
 }
 
@@ -251,21 +264,23 @@ var unpackBatchSize = 8 * runtime.GOMAXPROCS(-1)
 // Unpack unpacks pts to dst.
 func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData bool) error {
 	dst.reset()
-
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
 		return fmt.Errorf("cannot unmarshal metricName %q: %w", pts.metricName, err)
 	}
+	if !fetchData {
+		// Do not spend resources on data reading and unpacking.
+		return nil
+	}
 
 	// Feed workers with work
-	upws := make([]*unpackWork, 0, 1+len(pts.brs)/unpackBatchSize)
+	brsLen := len(pts.brs)
+	upws := make([]*unpackWork, 0, 1+brsLen/unpackBatchSize)
 	upw := getUnpackWork()
-	upw.fetchData = fetchData
 	for _, br := range pts.brs {
 		if len(upw.ws) >= unpackBatchSize {
 			unpackWorkCh <- upw
 			upws = append(upws, upw)
 			upw = getUnpackWork()
-			upw.fetchData = fetchData
 		}
 		upw.ws = append(upw.ws, unpackWorkItem{
 			br: br,
@@ -277,7 +292,7 @@ func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData
 	pts.brs = pts.brs[:0]
 
 	// Wait until work is complete
-	sbs := make([]*sortBlock, 0, len(pts.brs))
+	sbs := make([]*sortBlock, 0, brsLen)
 	var firstErr error
 	for _, upw := range upws {
 		if err := <-upw.doneCh; err != nil && firstErr == nil {
@@ -371,52 +386,26 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 var dedupsDuringSelect = metrics.NewCounter(`vm_deduplicated_samples_total{type="select"}`)
 
 type sortBlock struct {
-	// b is used as a temporary storage for unpacked rows before they
-	// go to Timestamps and Values.
-	b storage.Block
-
 	Timestamps []int64
 	Values     []float64
 	NextIdx    int
 }
 
 func (sb *sortBlock) reset() {
-	sb.b.Reset()
 	sb.Timestamps = sb.Timestamps[:0]
 	sb.Values = sb.Values[:0]
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(br storage.BlockRef, tr storage.TimeRange, fetchData bool) error {
-	br.MustReadBlock(&sb.b, fetchData)
-	if fetchData {
-		if err := sb.b.UnmarshalData(); err != nil {
-			return fmt.Errorf("cannot unmarshal block: %w", err)
-		}
+func (sb *sortBlock) unpackFrom(tmpBlock *storage.Block, br storage.BlockRef, tr storage.TimeRange) error {
+	tmpBlock.Reset()
+	br.MustReadBlock(tmpBlock, true)
+	if err := tmpBlock.UnmarshalData(); err != nil {
+		return fmt.Errorf("cannot unmarshal block: %w", err)
 	}
-	timestamps := sb.b.Timestamps()
-
-	// Skip timestamps smaller than tr.MinTimestamp.
-	i := 0
-	for i < len(timestamps) && timestamps[i] < tr.MinTimestamp {
-		i++
-	}
-
-	// Skip timestamps bigger than tr.MaxTimestamp.
-	j := len(timestamps)
-	for j > i && timestamps[j-1] > tr.MaxTimestamp {
-		j--
-	}
-	skippedRows := sb.b.RowsCount() - (j - i)
+	sb.Timestamps, sb.Values = tmpBlock.AppendRowsWithTimeRangeFilter(sb.Timestamps[:0], sb.Values[:0], tr)
+	skippedRows := tmpBlock.RowsCount() - len(sb.Timestamps)
 	metricRowsSkipped.Add(skippedRows)
-
-	// Copy the remaining values.
-	if i == j {
-		return nil
-	}
-	values := sb.b.Values()
-	sb.Timestamps = append(sb.Timestamps, timestamps[i:j]...)
-	sb.Values = decimal.AppendDecimalToFloat(sb.Values, values[i:j], sb.b.Scale())
 	return nil
 }
 
@@ -457,11 +446,11 @@ func DeleteSeries(sq *storage.SearchQuery) (int, error) {
 }
 
 // GetLabels returns labels until the given deadline.
-func GetLabels(deadline Deadline) ([]string, error) {
+func GetLabels(deadline searchutils.Deadline) ([]string, error) {
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	labels, err := vmstorage.SearchTagKeys(*maxTagKeysPerSearch, deadline.deadline)
+	labels, err := vmstorage.SearchTagKeys(*maxTagKeysPerSearch, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search: %w", err)
 	}
@@ -481,7 +470,7 @@ func GetLabels(deadline Deadline) ([]string, error) {
 
 // GetLabelValues returns label values for the given labelName
 // until the given deadline.
-func GetLabelValues(labelName string, deadline Deadline) ([]string, error) {
+func GetLabelValues(labelName string, deadline searchutils.Deadline) ([]string, error) {
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -490,7 +479,7 @@ func GetLabelValues(labelName string, deadline Deadline) ([]string, error) {
 	}
 
 	// Search for tag values
-	labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch, deadline.deadline)
+	labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search for labelName=%q: %w", labelName, err)
 	}
@@ -501,12 +490,27 @@ func GetLabelValues(labelName string, deadline Deadline) ([]string, error) {
 	return labelValues, nil
 }
 
-// GetLabelEntries returns all the label entries until the given deadline.
-func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) {
+// GetTagValueSuffixes returns tag value suffixes for the given tagKey and the given tagValuePrefix.
+//
+// It can be used for implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
+func GetTagValueSuffixes(tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, deadline searchutils.Deadline) ([]string, error) {
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch, deadline.deadline)
+	suffixes, err := vmstorage.SearchTagValueSuffixes(tr, []byte(tagKey), []byte(tagValuePrefix), delimiter, *maxTagValueSuffixesPerSearch, deadline.Deadline())
+	if err != nil {
+		return nil, fmt.Errorf("error during search for suffixes for tagKey=%q, tagValuePrefix=%q, delimiter=%c on time range %s: %w",
+			tagKey, tagValuePrefix, delimiter, tr.String(), err)
+	}
+	return suffixes, nil
+}
+
+// GetLabelEntries returns all the label entries until the given deadline.
+func GetLabelEntries(deadline searchutils.Deadline) ([]storage.TagEntry, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label entries request: %w", err)
 	}
@@ -532,11 +536,11 @@ func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) {
 }
 
 // GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
-func GetTSDBStatusForDate(deadline Deadline, date uint64, topN int) (*storage.TSDBStatus, error) {
+func GetTSDBStatusForDate(deadline searchutils.Deadline, date uint64, topN int) (*storage.TSDBStatus, error) {
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	status, err := vmstorage.GetTSDBStatusForDate(date, topN, deadline.deadline)
+	status, err := vmstorage.GetTSDBStatusForDate(date, topN, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during tsdb status request: %w", err)
 	}
@@ -544,11 +548,11 @@ func GetTSDBStatusForDate(deadline Deadline, date uint64, topN int) (*storage.TS
 }
 
 // GetSeriesCount returns the number of unique series.
-func GetSeriesCount(deadline Deadline) (uint64, error) {
+func GetSeriesCount(deadline searchutils.Deadline) (uint64, error) {
 	if deadline.Exceeded() {
 		return 0, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	n, err := vmstorage.GetSeriesCount(deadline.deadline)
+	n, err := vmstorage.GetSeriesCount(deadline.Deadline())
 	if err != nil {
 		return 0, fmt.Errorf("error during series count request: %w", err)
 	}
@@ -570,10 +574,119 @@ func putStorageSearch(sr *storage.Search) {
 
 var ssPool sync.Pool
 
-// ProcessSearchQuery performs sq on storage nodes until the given deadline.
+// ExportBlocks searches for time series matching sq and calls f for each found block.
+//
+// f is called in parallel from multiple goroutines.
+// Data processing is immediately stopped if f returns non-nil error.
+// It is the responsibility of f to call b.UnmarshalData before reading timestamps and values from the block.
+// It is the responsibility of f to filter blocks according to the given tr.
+func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) error {
+	if deadline.Exceeded() {
+		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
+	}
+	tfss, err := setupTfss(sq.TagFilterss)
+	if err != nil {
+		return err
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return err
+	}
+
+	vmstorage.WG.Add(1)
+	defer vmstorage.WG.Done()
+
+	sr := getStorageSearch()
+	defer putStorageSearch(sr)
+	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
+
+	// Start workers that call f in parallel on available CPU cores.
+	gomaxprocs := runtime.GOMAXPROCS(-1)
+	workCh := make(chan *exportWork, gomaxprocs*8)
+	var (
+		errGlobal     error
+		errGlobalLock sync.Mutex
+		mustStop      uint32
+	)
+	var wg sync.WaitGroup
+	wg.Add(gomaxprocs)
+	for i := 0; i < gomaxprocs; i++ {
+		go func() {
+			defer wg.Done()
+			for xw := range workCh {
+				if err := f(&xw.mn, &xw.b, tr); err != nil {
+					errGlobalLock.Lock()
+					if errGlobal != nil {
+						errGlobal = err
+						atomic.StoreUint32(&mustStop, 1)
+					}
+					errGlobalLock.Unlock()
+				}
+				xw.reset()
+				exportWorkPool.Put(xw)
+			}
+		}()
+	}
+
+	// Feed workers with work
+	blocksRead := 0
+	for sr.NextMetricBlock() {
+		blocksRead++
+		if deadline.Exceeded() {
+			return fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
+		}
+		if atomic.LoadUint32(&mustStop) != 0 {
+			break
+		}
+		xw := exportWorkPool.Get().(*exportWork)
+		if err := xw.mn.Unmarshal(sr.MetricBlockRef.MetricName); err != nil {
+			return fmt.Errorf("cannot unmarshal metricName for block #%d: %w", blocksRead, err)
+		}
+		sr.MetricBlockRef.BlockRef.MustReadBlock(&xw.b, true)
+		workCh <- xw
+	}
+	close(workCh)
+
+	// Wait for workers to finish.
+	wg.Wait()
+
+	// Check errors.
+	err = sr.Error()
+	if err == nil {
+		err = errGlobal
+	}
+	if err != nil {
+		if errors.Is(err, storage.ErrDeadlineExceeded) {
+			return fmt.Errorf("timeout exceeded during the query: %s", deadline.String())
+		}
+		return fmt.Errorf("search error after reading %d data blocks: %w", blocksRead, err)
+	}
+	return nil
+}
+
+type exportWork struct {
+	mn storage.MetricName
+	b  storage.Block
+}
+
+func (xw *exportWork) reset() {
+	xw.mn.Reset()
+	xw.b.Reset()
+}
+
+var exportWorkPool = &sync.Pool{
+	New: func() interface{} {
+		return &exportWork{}
+	},
+}
+
+// ProcessSearchQuery performs sq until the given deadline.
 //
 // Results.RunParallel or Results.Cancel must be called on the returned Results.
-func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, error) {
+func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, error) {
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -595,7 +708,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	maxSeriesCount := sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.deadline)
+	maxSeriesCount := sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
 
 	m := make(map[string][]storage.BlockRef, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
@@ -603,6 +716,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
+			putStorageSearch(sr)
 			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
 		metricName := sr.MetricBlockRef.MetricName
@@ -619,6 +733,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 		}
 	}
 	if err := sr.Error(); err != nil {
+		putStorageSearch(sr)
 		if errors.Is(err, storage.ErrDeadlineExceeded) {
 			return nil, fmt.Errorf("timeout exceeded during the query: %s", deadline.String())
 		}
@@ -656,33 +771,3 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 	}
 	return tfss, nil
 }
-
-// Deadline contains deadline with the corresponding timeout for pretty error messages.
-type Deadline struct {
-	deadline uint64
-
-	timeout  time.Duration
-	flagHint string
-}
-
-// NewDeadline returns deadline for the given timeout.
-//
-// flagHint must contain a hit for command-line flag, which could be used
-// in order to increase timeout.
-func NewDeadline(startTime time.Time, timeout time.Duration, flagHint string) Deadline {
-	return Deadline{
-		deadline: uint64(startTime.Add(timeout).Unix()),
-		timeout:  timeout,
-		flagHint: flagHint,
-	}
-}
-
-// Exceeded returns true if deadline is exceeded.
-func (d *Deadline) Exceeded() bool {
-	return fasttime.UnixTimestamp() > d.deadline
-}
-
-// String returns human-readable string representation for d.
-func (d *Deadline) String() string {
-	return fmt.Sprintf("%.3f seconds; the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), d.flagHint)
-}

app/vmselect/prometheus/export.qtpl

@@ -1,30 +1,102 @@
 {% import (
+	"bytes"
+	"strings"
+	"time"
+
 	"github.com/valyala/quicktemplate"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
 {% stripspace %}
 
-{% func ExportPrometheusLine(rs *netstorage.Result) %}
-	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+{% func ExportCSVLine(xb *exportBlock, fieldNames []string) %}
+	{% if len(xb.timestamps) == 0 || len(fieldNames) == 0 %}{% return %}{% endif %}
+	{% for i, timestamp := range xb.timestamps %}
+		{% code value := xb.values[i] %}
+		{%= exportCSVField(xb.mn, fieldNames[0], timestamp, value) %}
+		{% for _, fieldName := range fieldNames[1:] %}
+			,
+			{%= exportCSVField(xb.mn, fieldName, timestamp, value) %}
+		{% endfor %}
+		{% newline %}
+	{% endfor %}
+{% endfunc %}
+
+{% func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) %}
+	{% if fieldName == "__value__" %}
+		{%f= value %}
+		{% return %}
+	{% endif %}
+	{% if fieldName == "__timestamp__" %}
+		{%dl timestamp %}
+		{% return %}
+	{% endif %}
+	{% if strings.HasPrefix(fieldName, "__timestamp__:") %}
+		{% code timeFormat := fieldName[len("__timestamp__:"):] %}
+		{% switch timeFormat %}
+		{% case "unix_s" %}
+			{%dl= timestamp/1000 %}
+		{% case "unix_ms" %}
+			{%dl= timestamp %}
+		{% case "unix_ns" %}
+			{%dl= timestamp*1e6 %}
+		{% case "rfc3339" %}
+			{% code
+				bb := quicktemplate.AcquireByteBuffer()
+				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
+			%}
+			{%z= bb.B %}
+			{% code
+				quicktemplate.ReleaseByteBuffer(bb)
+			%}
+		{% default %}
+			{% if strings.HasPrefix(timeFormat, "custom:") %}
+				{% code
+					layout := timeFormat[len("custom:"):]
+					bb := quicktemplate.AcquireByteBuffer()
+					bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
+				%}
+				{% if bytes.ContainsAny(bb.B, `"`+",\n") %}
+					{%qz bb.B %}
+				{% else %}
+					{%z= bb.B %}
+				{% endif %}
+				{% code
+					quicktemplate.ReleaseByteBuffer(bb)
+				%}
+			{% else %}
+				Unsupported timeFormat={%s= timeFormat %}
+			{% endif %}
+		{% endswitch %}
+		{% return %}
+	{% endif %}
+	{% code v := mn.GetTagValue(fieldName) %}
+	{% if bytes.ContainsAny(v, `"`+",\n") %}
+		{%qz= v %}
+	{% else %}
+		{%z= v %}
+	{% endif %}
+{% endfunc %}
+
+{% func ExportPrometheusLine(xb *exportBlock) %}
+	{% if len(xb.timestamps) == 0 %}{% return %}{% endif %}
 	{% code bb := quicktemplate.AcquireByteBuffer() %}
-	{% code writeprometheusMetricName(bb, &rs.MetricName) %}
-	{% for i, ts := range rs.Timestamps %}
+	{% code writeprometheusMetricName(bb, xb.mn) %}
+	{% for i, ts := range xb.timestamps %}
 		{%z= bb.B %}{% space %}
-		{%f= rs.Values[i] %}{% space %}
+		{%f= xb.values[i] %}{% space %}
 		{%dl= ts %}{% newline %}
 	{% endfor %}
 	{% code quicktemplate.ReleaseByteBuffer(bb) %}
 {% endfunc %}
 
-{% func ExportJSONLine(rs *netstorage.Result) %}
-	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+{% func ExportJSONLine(xb *exportBlock) %}
+	{% if len(xb.timestamps) == 0 %}{% return %}{% endif %}
 	{
-		"metric":{%= metricNameObject(&rs.MetricName) %},
+		"metric":{%= metricNameObject(xb.mn) %},
 		"values":[
-			{% if len(rs.Values) > 0 %}
-				{% code values := rs.Values %}
+			{% if len(xb.values) > 0 %}
+				{% code values := xb.values %}
 				{%f= values[0] %}
 				{% code values = values[1:] %}
 				{% for _, v := range values %}
@@ -33,8 +105,8 @@
 			{% endif %}
 		],
 		"timestamps":[
-			{% if len(rs.Timestamps) > 0 %}
-				{% code timestamps := rs.Timestamps %}
+			{% if len(xb.timestamps) > 0 %}
+				{% code timestamps := xb.timestamps %}
 				{%dl= timestamps[0] %}
 				{% code timestamps = timestamps[1:] %}
 				{% for _, ts := range timestamps %}
@@ -45,10 +117,10 @@
 	}{% newline %}
 {% endfunc %}
 
-{% func ExportPromAPILine(rs *netstorage.Result) %}
+{% func ExportPromAPILine(xb *exportBlock) %}
 {
-	"metric": {%= metricNameObject(&rs.MetricName) %},
-	"values": {%= valuesWithTimestamps(rs.Values, rs.Timestamps) %}
+	"metric": {%= metricNameObject(xb.mn) %},
+	"values": {%= valuesWithTimestamps(xb.values, xb.timestamps) %}
 }
 {% endfunc %}
 

app/vmselect/prometheus/export.qtpl.go

@@ -6,380 +6,566 @@ package prometheus
 
 //line app/vmselect/prometheus/export.qtpl:1
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"bytes"
+	"strings"
+	"time"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
+//line app/vmselect/prometheus/export.qtpl:12
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
+//line app/vmselect/prometheus/export.qtpl:12
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
-func StreamExportPrometheusLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:10
-	if len(rs.Timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:10
-		return
-//line app/vmselect/prometheus/export.qtpl:10
-	}
-//line app/vmselect/prometheus/export.qtpl:11
-	bb := quicktemplate.AcquireByteBuffer()
-
 //line app/vmselect/prometheus/export.qtpl:12
-	writeprometheusMetricName(bb, &rs.MetricName)
-
+func StreamExportCSVLine(qw422016 *qt422016.Writer, xb *exportBlock, fieldNames []string) {
 //line app/vmselect/prometheus/export.qtpl:13
-	for i, ts := range rs.Timestamps {
+	if len(xb.timestamps) == 0 || len(fieldNames) == 0 {
+//line app/vmselect/prometheus/export.qtpl:13
+		return
+//line app/vmselect/prometheus/export.qtpl:13
+	}
 //line app/vmselect/prometheus/export.qtpl:14
-		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:14
-		qw422016.N().S(` `)
+	for i, timestamp := range xb.timestamps {
 //line app/vmselect/prometheus/export.qtpl:15
-		qw422016.N().F(rs.Values[i])
-//line app/vmselect/prometheus/export.qtpl:15
-		qw422016.N().S(` `)
-//line app/vmselect/prometheus/export.qtpl:16
-		qw422016.N().DL(ts)
+		value := xb.values[i]
+
 //line app/vmselect/prometheus/export.qtpl:16
+		streamexportCSVField(qw422016, xb.mn, fieldNames[0], timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:17
+		for _, fieldName := range fieldNames[1:] {
+//line app/vmselect/prometheus/export.qtpl:17
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:19
+			streamexportCSVField(qw422016, xb.mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:20
+		}
+//line app/vmselect/prometheus/export.qtpl:21
 		qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:17
+//line app/vmselect/prometheus/export.qtpl:22
 	}
-//line app/vmselect/prometheus/export.qtpl:18
-	quicktemplate.ReleaseByteBuffer(bb)
-
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:19
-func WriteExportPrometheusLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+func WriteExportCSVLine(qq422016 qtio422016.Writer, xb *exportBlock, fieldNames []string) {
+//line app/vmselect/prometheus/export.qtpl:23
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:19
-	StreamExportPrometheusLine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+	StreamExportCSVLine(qw422016, xb, fieldNames)
+//line app/vmselect/prometheus/export.qtpl:23
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:19
-func ExportPrometheusLine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+func ExportCSVLine(xb *exportBlock, fieldNames []string) string {
+//line app/vmselect/prometheus/export.qtpl:23
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:19
-	WriteExportPrometheusLine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+	WriteExportCSVLine(qb422016, xb, fieldNames)
+//line app/vmselect/prometheus/export.qtpl:23
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:21
-func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:22
-	if len(rs.Timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:22
-		return
-//line app/vmselect/prometheus/export.qtpl:22
-	}
-//line app/vmselect/prometheus/export.qtpl:22
-	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:24
-	streammetricNameObject(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/export.qtpl:24
-	qw422016.N().S(`,"values":[`)
+//line app/vmselect/prometheus/export.qtpl:25
+func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
 //line app/vmselect/prometheus/export.qtpl:26
-	if len(rs.Values) > 0 {
+	if fieldName == "__value__" {
 //line app/vmselect/prometheus/export.qtpl:27
-		values := rs.Values
-
+		qw422016.N().F(value)
 //line app/vmselect/prometheus/export.qtpl:28
-		qw422016.N().F(values[0])
+		return
 //line app/vmselect/prometheus/export.qtpl:29
-		values = values[1:]
-
+	}
 //line app/vmselect/prometheus/export.qtpl:30
-		for _, v := range values {
-//line app/vmselect/prometheus/export.qtpl:30
-			qw422016.N().S(`,`)
+	if fieldName == "__timestamp__" {
 //line app/vmselect/prometheus/export.qtpl:31
-			qw422016.N().F(v)
+		qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:32
-		}
+		return
 //line app/vmselect/prometheus/export.qtpl:33
 	}
-//line app/vmselect/prometheus/export.qtpl:33
-	qw422016.N().S(`],"timestamps":[`)
+//line app/vmselect/prometheus/export.qtpl:34
+	if strings.HasPrefix(fieldName, "__timestamp__:") {
+//line app/vmselect/prometheus/export.qtpl:35
+		timeFormat := fieldName[len("__timestamp__:"):]
+
 //line app/vmselect/prometheus/export.qtpl:36
-	if len(rs.Timestamps) > 0 {
+		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:37
-		timestamps := rs.Timestamps
-
+		case "unix_s":
 //line app/vmselect/prometheus/export.qtpl:38
-		qw422016.N().DL(timestamps[0])
+			qw422016.N().DL(timestamp / 1000)
 //line app/vmselect/prometheus/export.qtpl:39
-		timestamps = timestamps[1:]
-
+		case "unix_ms":
 //line app/vmselect/prometheus/export.qtpl:40
-		for _, ts := range timestamps {
-//line app/vmselect/prometheus/export.qtpl:40
-			qw422016.N().S(`,`)
+			qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:41
-			qw422016.N().DL(ts)
+		case "unix_ns":
 //line app/vmselect/prometheus/export.qtpl:42
-		}
+			qw422016.N().DL(timestamp * 1e6)
 //line app/vmselect/prometheus/export.qtpl:43
-	}
-//line app/vmselect/prometheus/export.qtpl:43
-	qw422016.N().S(`]}`)
+		case "rfc3339":
 //line app/vmselect/prometheus/export.qtpl:45
-	qw422016.N().S(`
-`)
-//line app/vmselect/prometheus/export.qtpl:46
-}
-
-//line app/vmselect/prometheus/export.qtpl:46
-func WriteExportJSONLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:46
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:46
-	StreamExportJSONLine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:46
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:46
-}
-
-//line app/vmselect/prometheus/export.qtpl:46
-func ExportJSONLine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:46
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:46
-	WriteExportJSONLine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:46
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:46
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:46
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:46
-}
+			bb := quicktemplate.AcquireByteBuffer()
+			bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
 
 //line app/vmselect/prometheus/export.qtpl:48
-func StreamExportPromAPILine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:48
-	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:50
-	streammetricNameObject(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/export.qtpl:50
-	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/export.qtpl:51
-	streamvaluesWithTimestamps(qw422016, rs.Values, rs.Timestamps)
-//line app/vmselect/prometheus/export.qtpl:51
-	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:53
-func WriteExportPromAPILine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:53
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:53
-	StreamExportPromAPILine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:53
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:53
-func ExportPromAPILine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:53
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:53
-	WriteExportPromAPILine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:53
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:53
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:53
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:55
-func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:55
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vmselect/prometheus/export.qtpl:61
-	bb, ok := <-resultsCh
-
-//line app/vmselect/prometheus/export.qtpl:62
-	if ok {
-//line app/vmselect/prometheus/export.qtpl:63
-		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:64
-		quicktemplate.ReleaseByteBuffer(bb)
-
-//line app/vmselect/prometheus/export.qtpl:65
-		for bb := range resultsCh {
-//line app/vmselect/prometheus/export.qtpl:65
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:66
 			qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:67
+//line app/vmselect/prometheus/export.qtpl:50
 			quicktemplate.ReleaseByteBuffer(bb)
 
+//line app/vmselect/prometheus/export.qtpl:52
+		default:
+//line app/vmselect/prometheus/export.qtpl:53
+			if strings.HasPrefix(timeFormat, "custom:") {
+//line app/vmselect/prometheus/export.qtpl:55
+				layout := timeFormat[len("custom:"):]
+				bb := quicktemplate.AcquireByteBuffer()
+				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
+
+//line app/vmselect/prometheus/export.qtpl:59
+				if bytes.ContainsAny(bb.B, `"`+",\n") {
+//line app/vmselect/prometheus/export.qtpl:60
+					qw422016.E().QZ(bb.B)
+//line app/vmselect/prometheus/export.qtpl:61
+				} else {
+//line app/vmselect/prometheus/export.qtpl:62
+					qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:63
+				}
+//line app/vmselect/prometheus/export.qtpl:65
+				quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:67
+			} else {
+//line app/vmselect/prometheus/export.qtpl:67
+				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:68
+				qw422016.N().S(timeFormat)
+//line app/vmselect/prometheus/export.qtpl:69
+			}
+//line app/vmselect/prometheus/export.qtpl:70
 		}
-//line app/vmselect/prometheus/export.qtpl:69
+//line app/vmselect/prometheus/export.qtpl:71
+		return
+//line app/vmselect/prometheus/export.qtpl:72
 	}
-//line app/vmselect/prometheus/export.qtpl:69
-	qw422016.N().S(`]}}`)
 //line app/vmselect/prometheus/export.qtpl:73
-}
-
-//line app/vmselect/prometheus/export.qtpl:73
-func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:73
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:73
-	StreamExportPromAPIResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:73
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:73
-}
-
-//line app/vmselect/prometheus/export.qtpl:73
-func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:73
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:73
-	WriteExportPromAPIResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:73
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:73
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:73
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:73
-}
+	v := mn.GetTagValue(fieldName)
 
+//line app/vmselect/prometheus/export.qtpl:74
+	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:75
-func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+		qw422016.N().QZ(v)
 //line app/vmselect/prometheus/export.qtpl:76
-	for bb := range resultsCh {
+	} else {
 //line app/vmselect/prometheus/export.qtpl:77
-		qw422016.N().Z(bb.B)
+		qw422016.N().Z(v)
 //line app/vmselect/prometheus/export.qtpl:78
-		quicktemplate.ReleaseByteBuffer(bb)
+	}
+//line app/vmselect/prometheus/export.qtpl:79
+}
 
 //line app/vmselect/prometheus/export.qtpl:79
-	}
-//line app/vmselect/prometheus/export.qtpl:80
-}
-
-//line app/vmselect/prometheus/export.qtpl:80
-func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:80
+func writeexportCSVField(qq422016 qtio422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
+//line app/vmselect/prometheus/export.qtpl:79
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:80
-	StreamExportStdResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+	streamexportCSVField(qw422016, mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:79
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 }
 
-//line app/vmselect/prometheus/export.qtpl:80
-func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) string {
+//line app/vmselect/prometheus/export.qtpl:79
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:80
-	WriteExportStdResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+	writeexportCSVField(qb422016, mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:79
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 }
 
+//line app/vmselect/prometheus/export.qtpl:81
+func StreamExportPrometheusLine(qw422016 *qt422016.Writer, xb *exportBlock) {
 //line app/vmselect/prometheus/export.qtpl:82
-func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:82
+		return
+//line app/vmselect/prometheus/export.qtpl:82
+	}
 //line app/vmselect/prometheus/export.qtpl:83
-	qw422016.N().Z(mn.MetricGroup)
+	bb := quicktemplate.AcquireByteBuffer()
+
 //line app/vmselect/prometheus/export.qtpl:84
-	if len(mn.Tags) > 0 {
-//line app/vmselect/prometheus/export.qtpl:84
-		qw422016.N().S(`{`)
+	writeprometheusMetricName(bb, xb.mn)
+
+//line app/vmselect/prometheus/export.qtpl:85
+	for i, ts := range xb.timestamps {
 //line app/vmselect/prometheus/export.qtpl:86
-		tags := mn.Tags
-
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:86
+		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().Z(tags[0].Key)
+		qw422016.N().F(xb.values[i])
 //line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().QZ(tags[0].Value)
+		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:88
-		tags = tags[1:]
-
+		qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:88
+		qw422016.N().S(`
+`)
 //line app/vmselect/prometheus/export.qtpl:89
-		for i := range tags {
+	}
 //line app/vmselect/prometheus/export.qtpl:90
-			tag := &tags[i]
+	quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:90
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().Z(tag.Key)
+}
+
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().S(`=`)
+func WriteExportPrometheusLine(qq422016 qtio422016.Writer, xb *exportBlock) {
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().QZ(tag.Value)
-//line app/vmselect/prometheus/export.qtpl:92
-		}
-//line app/vmselect/prometheus/export.qtpl:92
-		qw422016.N().S(`}`)
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:91
+	StreamExportPrometheusLine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:91
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:91
+}
+
+//line app/vmselect/prometheus/export.qtpl:91
+func ExportPrometheusLine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:91
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:91
+	WriteExportPrometheusLine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:91
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:91
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:91
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:91
+}
+
+//line app/vmselect/prometheus/export.qtpl:93
+func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:94
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:94
+		return
 //line app/vmselect/prometheus/export.qtpl:94
 	}
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:94
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:96
+	streammetricNameObject(qw422016, xb.mn)
+//line app/vmselect/prometheus/export.qtpl:96
+	qw422016.N().S(`,"values":[`)
+//line app/vmselect/prometheus/export.qtpl:98
+	if len(xb.values) > 0 {
+//line app/vmselect/prometheus/export.qtpl:99
+		values := xb.values
+
+//line app/vmselect/prometheus/export.qtpl:100
+		qw422016.N().F(values[0])
+//line app/vmselect/prometheus/export.qtpl:101
+		values = values[1:]
+
+//line app/vmselect/prometheus/export.qtpl:102
+		for _, v := range values {
+//line app/vmselect/prometheus/export.qtpl:102
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:103
+			qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:104
+		}
+//line app/vmselect/prometheus/export.qtpl:105
+	}
+//line app/vmselect/prometheus/export.qtpl:105
+	qw422016.N().S(`],"timestamps":[`)
+//line app/vmselect/prometheus/export.qtpl:108
+	if len(xb.timestamps) > 0 {
+//line app/vmselect/prometheus/export.qtpl:109
+		timestamps := xb.timestamps
+
+//line app/vmselect/prometheus/export.qtpl:110
+		qw422016.N().DL(timestamps[0])
+//line app/vmselect/prometheus/export.qtpl:111
+		timestamps = timestamps[1:]
+
+//line app/vmselect/prometheus/export.qtpl:112
+		for _, ts := range timestamps {
+//line app/vmselect/prometheus/export.qtpl:112
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:113
+			qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:114
+		}
+//line app/vmselect/prometheus/export.qtpl:115
+	}
+//line app/vmselect/prometheus/export.qtpl:115
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/export.qtpl:117
+	qw422016.N().S(`
+`)
+//line app/vmselect/prometheus/export.qtpl:118
 }
 
-//line app/vmselect/prometheus/export.qtpl:95
-func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+func WriteExportJSONLine(qq422016 qtio422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:118
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:95
-	streamprometheusMetricName(qw422016, mn)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+	StreamExportJSONLine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:118
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 }
 
-//line app/vmselect/prometheus/export.qtpl:95
-func prometheusMetricName(mn *storage.MetricName) string {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+func ExportJSONLine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:118
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:95
-	writeprometheusMetricName(qb422016, mn)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+	WriteExportJSONLine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:118
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+}
+
+//line app/vmselect/prometheus/export.qtpl:120
+func StreamExportPromAPILine(qw422016 *qt422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:120
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:122
+	streammetricNameObject(qw422016, xb.mn)
+//line app/vmselect/prometheus/export.qtpl:122
+	qw422016.N().S(`,"values":`)
+//line app/vmselect/prometheus/export.qtpl:123
+	streamvaluesWithTimestamps(qw422016, xb.values, xb.timestamps)
+//line app/vmselect/prometheus/export.qtpl:123
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:125
+func WriteExportPromAPILine(qq422016 qtio422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:125
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:125
+	StreamExportPromAPILine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:125
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:125
+func ExportPromAPILine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:125
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:125
+	WriteExportPromAPILine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:125
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:125
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:125
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:127
+func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:127
+	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/export.qtpl:133
+	bb, ok := <-resultsCh
+
+//line app/vmselect/prometheus/export.qtpl:134
+	if ok {
+//line app/vmselect/prometheus/export.qtpl:135
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:136
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:137
+		for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:137
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:138
+			qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:139
+			quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:140
+		}
+//line app/vmselect/prometheus/export.qtpl:141
+	}
+//line app/vmselect/prometheus/export.qtpl:141
+	qw422016.N().S(`]}}`)
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:145
+func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:145
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:145
+	StreamExportPromAPIResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:145
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:145
+func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:145
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:145
+	WriteExportPromAPIResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:145
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:145
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:145
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:147
+func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:148
+	for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:149
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:150
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:151
+	}
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:152
+func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:152
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:152
+	StreamExportStdResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:152
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:152
+func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:152
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:152
+	WriteExportStdResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:152
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:152
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:152
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:154
+func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:155
+	qw422016.N().Z(mn.MetricGroup)
+//line app/vmselect/prometheus/export.qtpl:156
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/export.qtpl:156
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/export.qtpl:158
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().Z(tags[0].Key)
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().QZ(tags[0].Value)
+//line app/vmselect/prometheus/export.qtpl:160
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/export.qtpl:161
+		for i := range tags {
+//line app/vmselect/prometheus/export.qtpl:162
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/export.qtpl:162
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().Z(tag.Key)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().QZ(tag.Value)
+//line app/vmselect/prometheus/export.qtpl:164
+		}
+//line app/vmselect/prometheus/export.qtpl:164
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:166
+	}
+//line app/vmselect/prometheus/export.qtpl:167
+}
+
+//line app/vmselect/prometheus/export.qtpl:167
+func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:167
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:167
+	streamprometheusMetricName(qw422016, mn)
+//line app/vmselect/prometheus/export.qtpl:167
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:167
+}
+
+//line app/vmselect/prometheus/export.qtpl:167
+func prometheusMetricName(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/export.qtpl:167
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:167
+	writeprometheusMetricName(qb422016, mn)
+//line app/vmselect/prometheus/export.qtpl:167
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:167
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:167
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:167
 }

app/vmselect/prometheus/prometheus.go

@@ -12,9 +12,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -25,18 +30,16 @@ import (
 )
 
 var (
-	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the colection. "+
+	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the collection. "+
 		"Too small value can result in incomplete last points for query results")
-	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
-	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
-	maxQueryLen       = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
-	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to -search.lookback-delta from Prometheus. "+
+	maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
+	maxLookback = flag.Duration("search.maxLookback", 0, "Synonim to -search.lookback-delta from Prometheus. "+
 		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
 		"See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons")
 	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+
 		"By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+
 		"Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+
-		"See also '-search.maxLookback' flag, which has the same meanining due to historical reasons")
+		"See also '-search.maxLookback' flag, which has the same meaning due to historical reasons")
 )
 
 // Default step used if not set.
@@ -59,15 +62,15 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 	if lookbackDelta <= 0 {
 		lookbackDelta = defaultStep
 	}
-	start, err := getTime(r, "start", ct-lookbackDelta)
+	start, err := searchutils.GetTime(r, "start", ct-lookbackDelta)
 	if err != nil {
 		return err
 	}
-	end, err := getTime(r, "end", ct)
+	end, err := searchutils.GetTime(r, "end", ct)
 	if err != nil {
 		return err
 	}
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if start >= end {
 		start = end - defaultStep
 	}
@@ -85,34 +88,203 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
-	resultsCh := make(chan *quicktemplate.ByteBuffer)
-	doneCh := make(chan error)
-	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteFederate(bb, rs)
-			resultsCh <- bb
-		})
-		close(resultsCh)
-		doneCh <- err
-	}()
-
 	w.Header().Set("Content-Type", "text/plain")
-	for bb := range resultsCh {
-		w.Write(bb.B)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+		if err := bw.Error(); err != nil {
+			return err
+		}
+		bb := quicktemplate.AcquireByteBuffer()
+		WriteFederate(bb, rs)
+		_, err := bw.Write(bb.B)
 		quicktemplate.ReleaseByteBuffer(bb)
-	}
-
-	err = <-doneCh
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("error during data fetching: %w", err)
 	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	federateDuration.UpdateDuration(startTime)
 	return nil
 }
 
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
 
+// ExportCSVHandler exports data in CSV format from /api/v1/export/csv
+func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	ct := startTime.UnixNano() / 1e6
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse request form values: %w", err)
+	}
+	format := r.FormValue("format")
+	if len(format) == 0 {
+		return fmt.Errorf("missing `format` arg; see https://victoriametrics.github.io/#how-to-export-csv-data")
+	}
+	fieldNames := strings.Split(format, ",")
+	matches := r.Form["match[]"]
+	if len(matches) == 0 {
+		// Maintain backwards compatibility
+		match := r.FormValue("match")
+		if len(match) == 0 {
+			return fmt.Errorf("missing `match[]` arg")
+		}
+		matches = []string{match}
+	}
+	start, err := searchutils.GetTime(r, "start", 0)
+	if err != nil {
+		return err
+	}
+	end, err := searchutils.GetTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	deadline := searchutils.GetDeadlineForExport(r, startTime)
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return err
+	}
+	sq := &storage.SearchQuery{
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	w.Header().Set("Content-Type", "text/csv")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+
+	resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
+	doneCh := make(chan error)
+	go func() {
+		err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+			if err := bw.Error(); err != nil {
+				return err
+			}
+			if err := b.UnmarshalData(); err != nil {
+				return fmt.Errorf("cannot unmarshal block during export: %s", err)
+			}
+			xb := exportBlockPool.Get().(*exportBlock)
+			xb.mn = mn
+			xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
+			if len(xb.timestamps) > 0 {
+				bb := quicktemplate.AcquireByteBuffer()
+				WriteExportCSVLine(bb, xb, fieldNames)
+				resultsCh <- bb
+			}
+			xb.reset()
+			exportBlockPool.Put(xb)
+			return nil
+		})
+		close(resultsCh)
+		doneCh <- err
+	}()
+	// Consume all the data from resultsCh.
+	for bb := range resultsCh {
+		// Do not check for error in bw.Write, since this error is checked inside netstorage.ExportBlocks above.
+		_, _ = bw.Write(bb.B)
+		quicktemplate.ReleaseByteBuffer(bb)
+	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	err = <-doneCh
+	if err != nil {
+		return fmt.Errorf("error during exporting data to csv: %w", err)
+	}
+	exportCSVDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/csv"}`)
+
+// ExportNativeHandler exports data in native format from /api/v1/export/native.
+func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	ct := startTime.UnixNano() / 1e6
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse request form values: %w", err)
+	}
+	matches := r.Form["match[]"]
+	if len(matches) == 0 {
+		// Maintain backwards compatibility
+		match := r.FormValue("match")
+		if len(match) == 0 {
+			return fmt.Errorf("missing `match[]` arg")
+		}
+		matches = []string{match}
+	}
+	start, err := searchutils.GetTime(r, "start", 0)
+	if err != nil {
+		return err
+	}
+	end, err := searchutils.GetTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	deadline := searchutils.GetDeadlineForExport(r, startTime)
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return err
+	}
+	sq := &storage.SearchQuery{
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	w.Header().Set("Content-Type", "VictoriaMetrics/native")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+
+	// Marshal tr
+	trBuf := make([]byte, 0, 16)
+	trBuf = encoding.MarshalInt64(trBuf, start)
+	trBuf = encoding.MarshalInt64(trBuf, end)
+	_, _ = bw.Write(trBuf)
+
+	// Marshal native blocks.
+	err = netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+		if err := bw.Error(); err != nil {
+			return err
+		}
+		dstBuf := bbPool.Get()
+		tmpBuf := bbPool.Get()
+		dst := dstBuf.B
+		tmp := tmpBuf.B
+
+		// Marshal mn
+		tmp = mn.Marshal(tmp[:0])
+		dst = encoding.MarshalUint32(dst, uint32(len(tmp)))
+		dst = append(dst, tmp...)
+
+		// Marshal b
+		tmp = b.MarshalPortable(tmp[:0])
+		dst = encoding.MarshalUint32(dst, uint32(len(tmp)))
+		dst = append(dst, tmp...)
+
+		tmpBuf.B = tmp
+		bbPool.Put(tmpBuf)
+
+		_, err := bw.Write(dst)
+
+		dstBuf.B = dst
+		bbPool.Put(dstBuf)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	exportNativeDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var exportNativeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/native"}`)
+
+var bbPool bytesutil.ByteBufferPool
+
 // ExportHandler exports data in raw format from /api/v1/export.
 func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := startTime.UnixNano() / 1e6
@@ -128,21 +300,22 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		}
 		matches = []string{match}
 	}
-	start, err := getTime(r, "start", 0)
+	start, err := searchutils.GetTime(r, "start", 0)
 	if err != nil {
 		return err
 	}
-	end, err := getTime(r, "end", ct)
+	end, err := searchutils.GetTime(r, "end", ct)
 	if err != nil {
 		return err
 	}
 	format := r.FormValue("format")
 	maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
-	deadline := getDeadlineForExport(r, startTime)
+	reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
+	deadline := searchutils.GetDeadlineForExport(r, startTime)
 	if start >= end {
 		end = start + defaultStep
 	}
-	if err := exportHandler(w, matches, start, end, format, maxRowsPerLine, deadline); err != nil {
+	if err := exportHandler(w, matches, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
 		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %w", matches, start, end, err)
 	}
 	exportDuration.UpdateDuration(startTime)
@@ -151,17 +324,34 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 
 var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
 
-func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, maxRowsPerLine int, deadline netstorage.Deadline) error {
+func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, maxRowsPerLine int, reduceMemUsage bool, deadline searchutils.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
-	writeLineFunc := func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
+	writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
 		bb := quicktemplate.AcquireByteBuffer()
-		WriteExportJSONLine(bb, rs)
+		WriteExportJSONLine(bb, xb)
 		resultsCh <- bb
 	}
+	contentType := "application/stream+json"
+	if format == "prometheus" {
+		contentType = "text/plain"
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			bb := quicktemplate.AcquireByteBuffer()
+			WriteExportPrometheusLine(bb, xb)
+			resultsCh <- bb
+		}
+	} else if format == "promapi" {
+		writeResponseFunc = WriteExportPromAPIResponse
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			bb := quicktemplate.AcquireByteBuffer()
+			WriteExportPromAPILine(bb, xb)
+			resultsCh <- bb
+		}
+	}
 	if maxRowsPerLine > 0 {
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			valuesOrig := rs.Values
-			timestampsOrig := rs.Timestamps
+		writeLineFuncOrig := writeLineFunc
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			valuesOrig := xb.values
+			timestampsOrig := xb.timestamps
 			values := valuesOrig
 			timestamps := timestampsOrig
 			for len(values) > 0 {
@@ -178,30 +368,12 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 					values = nil
 					timestamps = nil
 				}
-				rs.Values = valuesChunk
-				rs.Timestamps = timestampsChunk
-				bb := quicktemplate.AcquireByteBuffer()
-				WriteExportJSONLine(bb, rs)
-				resultsCh <- bb
+				xb.values = valuesChunk
+				xb.timestamps = timestampsChunk
+				writeLineFuncOrig(xb, resultsCh)
 			}
-			rs.Values = valuesOrig
-			rs.Timestamps = timestampsOrig
-		}
-	}
-	contentType := "application/stream+json"
-	if format == "prometheus" {
-		contentType = "text/plain"
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteExportPrometheusLine(bb, rs)
-			resultsCh <- bb
-		}
-	} else if format == "promapi" {
-		writeResponseFunc = WriteExportPromAPIResponse
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteExportPromAPILine(bb, rs)
-			resultsCh <- bb
+			xb.values = valuesOrig
+			xb.timestamps = timestampsOrig
 		}
 	}
 
@@ -214,28 +386,62 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
-	if err != nil {
-		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
-	}
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
 
 	resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
 	doneCh := make(chan error)
-	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-			writeLineFunc(rs, resultsCh)
-		})
-		close(resultsCh)
-		doneCh <- err
-	}()
+	if !reduceMemUsage {
+		rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+		}
+		go func() {
+			err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = &rs.MetricName
+				xb.timestamps = rs.Timestamps
+				xb.values = rs.Values
+				writeLineFunc(xb, resultsCh)
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	} else {
+		go func() {
+			err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				if err := b.UnmarshalData(); err != nil {
+					return fmt.Errorf("cannot unmarshal block during export: %s", err)
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = mn
+				xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
+				if len(xb.timestamps) > 0 {
+					writeLineFunc(xb, resultsCh)
+				}
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	}
 
-	w.Header().Set("Content-Type", contentType)
-	writeResponseFunc(w, resultsCh)
-
-	// Consume all the data from resultsCh in the event writeResponseFunc
-	// fails to consume all the data.
-	for bb := range resultsCh {
-		quicktemplate.ReleaseByteBuffer(bb)
+	// writeResponseFunc must consume all the data from resultsCh.
+	writeResponseFunc(bw, resultsCh)
+	if err := bw.Flush(); err != nil {
+		return err
 	}
 	err = <-doneCh
 	if err != nil {
@@ -244,6 +450,24 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 	return nil
 }
 
+type exportBlock struct {
+	mn         *storage.MetricName
+	timestamps []int64
+	values     []float64
+}
+
+func (xb *exportBlock) reset() {
+	xb.mn = nil
+	xb.timestamps = xb.timestamps[:0]
+	xb.values = xb.values[:0]
+}
+
+var exportBlockPool = &sync.Pool{
+	New: func() interface{} {
+		return &exportBlock{}
+	},
+}
+
 // DeleteHandler processes /api/v1/admin/tsdb/delete_series prometheus API request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series
@@ -282,7 +506,7 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
 func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
@@ -303,11 +527,11 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 			matches = []string{fmt.Sprintf("{%s!=''}", labelName)}
 		}
 		ct := startTime.UnixNano() / 1e6
-		end, err := getTime(r, "end", ct)
+		end, err := searchutils.GetTime(r, "end", ct)
 		if err != nil {
 			return err
 		}
-		start, err := getTime(r, "start", end-defaultStep)
+		start, err := searchutils.GetTime(r, "start", end-defaultStep)
 		if err != nil {
 			return err
 		}
@@ -318,12 +542,17 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 	}
 
 	w.Header().Set("Content-Type", "application/json")
-	WriteLabelValuesResponse(w, labelValues)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelValuesResponse(bw, labelValues)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelValuesDuration.UpdateDuration(startTime)
 	return nil
 }
 
-func labelValuesWithMatches(labelName string, matches []string, start, end int64, deadline netstorage.Deadline) ([]string, error) {
+func labelValuesWithMatches(labelName string, matches []string, start, end int64, deadline searchutils.Deadline) ([]string, error) {
 	if len(matches) == 0 {
 		logger.Panicf("BUG: matches must be non-empty")
 	}
@@ -359,14 +588,15 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
 
 	m := make(map[string]struct{})
 	var mLock sync.Mutex
-	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		labelValue := rs.MetricName.GetTagValue(labelName)
 		if len(labelValue) == 0 {
-			return
+			return nil
 		}
 		mLock.Lock()
 		m[string(labelValue)] = struct{}{}
 		mLock.Unlock()
+		return nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error when data fetching: %w", err)
@@ -384,13 +614,18 @@ var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 
 // LabelsCountHandler processes /api/v1/labels/count request.
 func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	labelEntries, err := netstorage.GetLabelEntries(deadline)
 	if err != nil {
 		return fmt.Errorf(`cannot obtain label entries: %w`, err)
 	}
 	w.Header().Set("Content-Type", "application/json")
-	WriteLabelsCountResponse(w, labelEntries)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelsCountResponse(bw, labelEntries)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelsCountDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -403,7 +638,7 @@ const secsPerDay = 3600 * 24
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
 func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
@@ -436,7 +671,12 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 		return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
 	}
 	w.Header().Set("Content-Type", "application/json")
-	WriteTSDBStatusResponse(w, status)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTSDBStatusResponse(bw, status)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	tsdbStatusDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -447,7 +687,7 @@ var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
 func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
@@ -466,11 +706,11 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 			matches = []string{"{__name__!=''}"}
 		}
 		ct := startTime.UnixNano() / 1e6
-		end, err := getTime(r, "end", ct)
+		end, err := searchutils.GetTime(r, "end", ct)
 		if err != nil {
 			return err
 		}
-		start, err := getTime(r, "start", end-defaultStep)
+		start, err := searchutils.GetTime(r, "start", end-defaultStep)
 		if err != nil {
 			return err
 		}
@@ -481,12 +721,17 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
-	WriteLabelsResponse(w, labels)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelsResponse(bw, labels)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelsDuration.UpdateDuration(startTime)
 	return nil
 }
 
-func labelsWithMatches(matches []string, start, end int64, deadline netstorage.Deadline) ([]string, error) {
+func labelsWithMatches(matches []string, start, end int64, deadline searchutils.Deadline) ([]string, error) {
 	if len(matches) == 0 {
 		logger.Panicf("BUG: matches must be non-empty")
 	}
@@ -509,7 +754,7 @@ func labelsWithMatches(matches []string, start, end int64, deadline netstorage.D
 
 	m := make(map[string]struct{})
 	var mLock sync.Mutex
-	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		mLock.Lock()
 		tags := rs.MetricName.Tags
 		for i := range tags {
@@ -518,6 +763,7 @@ func labelsWithMatches(matches []string, start, end int64, deadline netstorage.D
 		}
 		m["__name__"] = struct{}{}
 		mLock.Unlock()
+		return nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error when data fetching: %w", err)
@@ -535,13 +781,18 @@ var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 
 // SeriesCountHandler processes /api/v1/series/count request.
 func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	n, err := netstorage.GetSeriesCount(deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)
 	}
 	w.Header().Set("Content-Type", "application/json")
-	WriteSeriesCountResponse(w, n)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteSeriesCountResponse(bw, n)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	seriesCountDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -560,20 +811,20 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if len(matches) == 0 {
 		return fmt.Errorf("missing `match[]` arg")
 	}
-	end, err := getTime(r, "end", ct)
+	end, err := searchutils.GetTime(r, "end", ct)
 	if err != nil {
 		return err
 	}
-	// Do not set start to minTimeMsecs by default as Prometheus does,
+	// Do not set start to searchutils.minTimeMsecs by default as Prometheus does,
 	// since this leads to fetching and scanning all the data from the storage,
 	// which can take a lot of time for big storages.
 	// It is better setting start as end-defaultStep by default.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
-	start, err := getTime(r, "start", end-defaultStep)
+	start, err := searchutils.GetTime(r, "start", end-defaultStep)
 	if err != nil {
 		return err
 	}
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
@@ -592,25 +843,28 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
+	w.Header().Set("Content-Type", "application/json")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
 	resultsCh := make(chan *quicktemplate.ByteBuffer)
 	doneCh := make(chan error)
 	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+			if err := bw.Error(); err != nil {
+				return err
+			}
 			bb := quicktemplate.AcquireByteBuffer()
 			writemetricNameObject(bb, &rs.MetricName)
 			resultsCh <- bb
+			return nil
 		})
 		close(resultsCh)
 		doneCh <- err
 	}()
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteSeriesResponse(w, resultsCh)
-
-	// Consume all the data from resultsCh in the event WriteSeriesResponse
-	// fails to consume all the data.
-	for bb := range resultsCh {
-		quicktemplate.ReleaseByteBuffer(bb)
+	// WriteSeriesResponse must consume all the data from resultsCh.
+	WriteSeriesResponse(bw, resultsCh)
+	if err := bw.Flush(); err != nil {
+		return err
 	}
 	err = <-doneCh
 	if err != nil {
@@ -631,7 +885,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if len(query) == 0 {
 		return fmt.Errorf("missing `query` arg")
 	}
-	start, err := getTime(r, "time", ct)
+	start, err := searchutils.GetTime(r, "time", ct)
 	if err != nil {
 		return err
 	}
@@ -639,23 +893,17 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if err != nil {
 		return err
 	}
-	step, err := getDuration(r, "step", lookbackDelta)
+	step, err := searchutils.GetDuration(r, "step", lookbackDelta)
 	if err != nil {
 		return err
 	}
 	if step <= 0 {
 		step = defaultStep
 	}
-	deadline := getDeadlineForQuery(r, startTime)
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 
-	if len(query) > *maxQueryLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
-	}
-	queryOffset := getLatencyOffsetMilliseconds()
-	if !getBool(r, "nocache") && ct-start < queryOffset {
-		// Adjust start time only if `nocache` arg isn't set.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
-		start = ct - queryOffset
+	if len(query) > maxQueryLen.N {
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
 	}
 	if childQuery, windowStr, offsetStr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
 		window, err := parsePositiveDuration(windowStr, step)
@@ -669,7 +917,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		start -= offset
 		end := start
 		start = end - window
-		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", 0, deadline); err != nil {
+		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", 0, false, deadline); err != nil {
 			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
 		}
 		queryDuration.UpdateDuration(startTime)
@@ -701,6 +949,16 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		return nil
 	}
 
+	queryOffset := getLatencyOffsetMilliseconds()
+	if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
+		// Adjust start time only if `nocache` arg isn't set.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
+		startPrev := start
+		start = ct - queryOffset
+		queryOffset = startPrev - start
+	} else {
+		queryOffset = 0
+	}
 	ec := promql.EvalConfig{
 		Start:            start,
 		End:              start,
@@ -713,9 +971,22 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
 	}
+	if queryOffset > 0 {
+		for i := range result {
+			timestamps := result[i].Timestamps
+			for j := range timestamps {
+				timestamps[j] += queryOffset
+			}
+		}
+	}
 
 	w.Header().Set("Content-Type", "application/json")
-	WriteQueryResponse(w, result)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteQueryResponse(bw, result)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	queryDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -745,15 +1016,15 @@ func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	if len(query) == 0 {
 		return fmt.Errorf("missing `query` arg")
 	}
-	start, err := getTime(r, "start", ct-defaultStep)
+	start, err := searchutils.GetTime(r, "start", ct-defaultStep)
 	if err != nil {
 		return err
 	}
-	end, err := getTime(r, "end", ct)
+	end, err := searchutils.GetTime(r, "end", ct)
 	if err != nil {
 		return err
 	}
-	step, err := getDuration(r, "step", defaultStep)
+	step, err := searchutils.GetDuration(r, "step", defaultStep)
 	if err != nil {
 		return err
 	}
@@ -765,16 +1036,16 @@ func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 }
 
 func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64) error {
-	deadline := getDeadlineForQuery(r, startTime)
-	mayCache := !getBool(r, "nocache")
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	mayCache := !searchutils.GetBool(r, "nocache")
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
 	}
 
 	// Validate input args.
-	if len(query) > *maxQueryLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
+	if len(query) > maxQueryLen.N {
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
 	}
 	if start > end {
 		end = start + defaultStep
@@ -809,7 +1080,12 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	result = removeEmptyValuesAndTimeseries(result)
 
 	w.Header().Set("Content-Type", "application/json")
-	WriteQueryRangeResponse(w, result)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteQueryRangeResponse(bw, result)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -886,120 +1162,12 @@ func adjustLastPoints(tss []netstorage.Result, start, end int64) []netstorage.Re
 	return tss
 }
 
-func getTime(r *http.Request, argKey string, defaultValue int64) (int64, error) {
-	argValue := r.FormValue(argKey)
-	if len(argValue) == 0 {
-		return defaultValue, nil
-	}
-	secs, err := strconv.ParseFloat(argValue, 64)
-	if err != nil {
-		// Try parsing string format
-		t, err := time.Parse(time.RFC3339, argValue)
-		if err != nil {
-			// Handle Prometheus'-provided minTime and maxTime.
-			// See https://github.com/prometheus/client_golang/issues/614
-			switch argValue {
-			case prometheusMinTimeFormatted:
-				return minTimeMsecs, nil
-			case prometheusMaxTimeFormatted:
-				return maxTimeMsecs, nil
-			}
-			// Try parsing duration relative to the current time
-			d, err1 := metricsql.DurationValue(argValue, 0)
-			if err1 != nil {
-				return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
-			}
-			if d > 0 {
-				d = -d
-			}
-			t = time.Now().Add(time.Duration(d) * time.Millisecond)
-		}
-		secs = float64(t.UnixNano()) / 1e9
-	}
-	msecs := int64(secs * 1e3)
-	if msecs < minTimeMsecs {
-		msecs = 0
-	}
-	if msecs > maxTimeMsecs {
-		msecs = maxTimeMsecs
-	}
-	return msecs, nil
-}
-
-var (
-	// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
-	// See https://github.com/prometheus/client_golang/issues/614 for details.
-	prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano)
-	prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano)
-)
-
-const (
-	// These values prevent from overflow when storing msec-precision time in int64.
-	minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
-	maxTimeMsecs = int64(1<<63-1) / 1e6
-)
-
-func getDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) {
-	argValue := r.FormValue(argKey)
-	if len(argValue) == 0 {
-		return defaultValue, nil
-	}
-	secs, err := strconv.ParseFloat(argValue, 64)
-	if err != nil {
-		// Try parsing string format
-		d, err := metricsql.DurationValue(argValue, 0)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
-		}
-		secs = float64(d) / 1000
-	}
-	msecs := int64(secs * 1e3)
-	if msecs <= 0 || msecs > maxDurationMsecs {
-		return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs))
-	}
-	return msecs, nil
-}
-
-const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
-
 func getMaxLookback(r *http.Request) (int64, error) {
 	d := maxLookback.Milliseconds()
 	if d == 0 {
 		d = maxStalenessInterval.Milliseconds()
 	}
-	return getDuration(r, "max_lookback", d)
-}
-
-func getDeadlineForQuery(r *http.Request, startTime time.Time) netstorage.Deadline {
-	dMax := maxQueryDuration.Milliseconds()
-	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxQueryDuration")
-}
-
-func getDeadlineForExport(r *http.Request, startTime time.Time) netstorage.Deadline {
-	dMax := maxExportDuration.Milliseconds()
-	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
-}
-
-func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) netstorage.Deadline {
-	d, err := getDuration(r, "timeout", 0)
-	if err != nil {
-		d = 0
-	}
-	if d <= 0 || d > dMax {
-		d = dMax
-	}
-	timeout := time.Duration(d) * time.Millisecond
-	return netstorage.NewDeadline(startTime, timeout, flagHint)
-}
-
-func getBool(r *http.Request, argKey string) bool {
-	argValue := r.FormValue(argKey)
-	switch strings.ToLower(argValue) {
-	case "", "0", "f", "false", "no":
-		return false
-	default:
-		return true
-	}
+	return searchutils.GetDuration(r, "max_lookback", d)
 }
 
 func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) {

app/vmselect/prometheus/prometheus_test.go

@@ -1,10 +1,7 @@
 package prometheus
 
 import (
-	"fmt"
 	"math"
-	"net/http"
-	"net/url"
 	"reflect"
 	"testing"
 
@@ -50,76 +47,6 @@ func TestRemoveEmptyValuesAndTimeseries(t *testing.T) {
 	})
 }
 
-func TestGetTimeSuccess(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
-		r, err := http.NewRequest("GET", urlStr, nil)
-		if err != nil {
-			t.Fatalf("unexpected error in NewRequest: %s", err)
-		}
-
-		// Verify defaultValue
-		ts, err := getTime(r, "foo", 123)
-		if err != nil {
-			t.Fatalf("unexpected error when obtaining default time from getTime(%q): %s", s, err)
-		}
-		if ts != 123 {
-			t.Fatalf("unexpected default value for getTime(%q); got %d; want %d", s, ts, 123)
-		}
-
-		// Verify timestampExpected
-		ts, err = getTime(r, "s", 123)
-		if err != nil {
-			t.Fatalf("unexpected error in getTime(%q): %s", s, err)
-		}
-		if ts != timestampExpected {
-			t.Fatalf("unexpected timestamp for getTime(%q); got %d; want %d", s, ts, timestampExpected)
-		}
-	}
-
-	f("2019-07-07T20:01:02Z", 1562529662000)
-	f("2019-07-07T20:47:40+03:00", 1562521660000)
-	f("-292273086-05-16T16:47:06Z", minTimeMsecs)
-	f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
-	f("1562529662.324", 1562529662324)
-	f("-9223372036.854", minTimeMsecs)
-	f("-9223372036.855", minTimeMsecs)
-	f("9223372036.855", maxTimeMsecs)
-}
-
-func TestGetTimeError(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
-		r, err := http.NewRequest("GET", urlStr, nil)
-		if err != nil {
-			t.Fatalf("unexpected error in NewRequest: %s", err)
-		}
-
-		// Verify defaultValue
-		ts, err := getTime(r, "foo", 123)
-		if err != nil {
-			t.Fatalf("unexpected error when obtaining default time from getTime(%q): %s", s, err)
-		}
-		if ts != 123 {
-			t.Fatalf("unexpected default value for getTime(%q); got %d; want %d", s, ts, 123)
-		}
-
-		// Verify timestampExpected
-		_, err = getTime(r, "s", 123)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in getTime(%q)", s)
-		}
-	}
-
-	f("foo")
-	f("2019-07-07T20:01:02Zisdf")
-	f("2019-07-07T20:47:40+03:00123")
-	f("-292273086-05-16T16:47:07Z")
-	f("292277025-08-18T07:12:54.999999998Z")
-}
-
 func TestAdjustLastPoints(t *testing.T) {
 	f := func(tss []netstorage.Result, start, end int64, tssExpected []netstorage.Result) {
 		t.Helper()

app/vmselect/promql/aggr.go

@@ -494,6 +494,8 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 //
 // It is expected that a doesn't contain NaNs.
 //
+// The function modifies contents for a, so the caller must prepare it accordingly.
+//
 // See https://en.wikipedia.org/wiki/Mode_(statistics)
 func modeNoNaNs(prevValue float64, a []float64) float64 {
 	if len(a) == 0 {

app/vmselect/promql/eval.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
@@ -56,14 +57,7 @@ func AdjustStartEnd(start, end, step int64) (int64, int64) {
 
 	// Round start and end to values divisible by step in order
 	// to enable response caching (see EvalConfig.mayCache).
-
-	// Round start to the nearest smaller value divisible by step.
-	start -= start % step
-	// Round end to the nearest bigger value divisible by step.
-	adjust := end % step
-	if adjust > 0 {
-		end += step - adjust
-	}
+	start, end = alignStartEnd(start, end, step)
 
 	// Make sure that the new number of points is the same as the initial number of points.
 	newPoints := (end-start)/step + 1
@@ -75,6 +69,17 @@ func AdjustStartEnd(start, end, step int64) (int64, int64) {
 	return start, end
 }
 
+func alignStartEnd(start, end, step int64) (int64, int64) {
+	// Round start to the nearest smaller value divisible by step.
+	start -= start % step
+	// Round end to the nearest bigger value divisible by step.
+	adjust := end % step
+	if adjust > 0 {
+		end += step - adjust
+	}
+	return start, end
+}
+
 // EvalConfig is the configuration required for query evaluation via Exec
 type EvalConfig struct {
 	Start int64
@@ -84,7 +89,7 @@ type EvalConfig struct {
 	// QuotedRemoteAddr contains quoted remote address.
 	QuotedRemoteAddr string
 
-	Deadline netstorage.Deadline
+	Deadline searchutils.Deadline
 
 	MayCache bool
 
@@ -501,11 +506,13 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr
 
 	ecSQ := newEvalConfig(ec)
 	ecSQ.Start -= window + maxSilenceInterval + step
+	ecSQ.End += step
 	ecSQ.Step = step
 	if err := ValidateMaxPointsPerTimeseries(ecSQ.Start, ecSQ.End, ecSQ.Step); err != nil {
 		return nil, err
 	}
-	ecSQ.Start, ecSQ.End = AdjustStartEnd(ecSQ.Start, ecSQ.End, ecSQ.Step)
+	// unconditionally align start and end args to step for subquery as Prometheus does.
+	ecSQ.Start, ecSQ.End = alignStartEnd(ecSQ.Start, ecSQ.End, ecSQ.Step)
 	tssSQ, err := evalExpr(ecSQ, re.Expr)
 	if err != nil {
 		return nil, err
@@ -517,7 +524,6 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr
 		}
 		return nil, nil
 	}
-
 	sharedTimestamps := getTimestamps(ec.Start, ec.End, ec.Step)
 	preFunc, rcs, err := getRollupConfigs(name, rf, expr, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 	if err != nil {
@@ -735,7 +741,7 @@ func getRollupMemoryLimiter() *memoryLimiter {
 
 func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
 		defer putTimeseries(ts)
@@ -755,6 +761,7 @@ func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncCo
 			ts.Timestamps = nil
 			ts.denyReuse = false
 		}
+		return nil
 	})
 	if err != nil {
 		return nil, err
@@ -767,7 +774,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
@@ -783,6 +790,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 			tss = append(tss, &ts)
 			tssLock.Unlock()
 		}
+		return nil
 	})
 	if err != nil {
 		return nil, err
@@ -830,7 +838,7 @@ func evalTime(ec *EvalConfig) []*timeseries {
 	timestamps := rv[0].Timestamps
 	values := rv[0].Values
 	for i, ts := range timestamps {
-		values[i] = float64(ts) * 1e-3
+		values[i] = float64(ts) / 1e3
 	}
 	return rv
 }

app/vmselect/promql/exec.go

@@ -175,6 +175,7 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
 	if pcv == nil {
 		e, err := metricsql.Parse(q)
 		if err == nil {
+			e = metricsql.Optimize(e)
 			e = adjustCmpOps(e)
 		}
 		pcv = &parseCacheValue{

app/vmselect/promql/exec_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -21,7 +22,7 @@ func TestExecSuccess(t *testing.T) {
 			Start:    start,
 			End:      end,
 			Step:     step,
-			Deadline: netstorage.NewDeadline(time.Now(), time.Minute, ""),
+			Deadline: searchutils.NewDeadline(time.Now(), time.Minute, ""),
 		}
 		for i := 0; i < 5; i++ {
 			result, err := Exec(ec, q, false)
@@ -108,7 +109,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `time() offset 0s`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -119,7 +120,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort((label_set(time(), "foo", "bar"), label_set(time()+10, "foo", "baz")) offset 0s)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -128,7 +129,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{910, 1110, 1310, 1510, 1710, 1910},
+			Values:     []float64{1010, 1210, 1410, 1610, 1810, 2010},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -149,7 +150,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run("time() offset 100s", func(t *testing.T) {
+	t.Run("time() offset 1m40s0ms", func(t *testing.T) {
 		t.Parallel()
 		q := `time() offset 100s`
 		r := netstorage.Result{
@@ -209,7 +210,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{860, 1060, 1260, 1460, 1660, 1860},
+			Values:     []float64{810, 1010, 1210, 1410, 1610, 1810},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -224,7 +225,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort((label_set(time() offset 100s, "foo", "bar"), label_set(time()+10, "foo", "baz") offset 50s) offset 400s)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{300, 500, 700, 900, 1100, 1300},
+			Values:     []float64{400, 600, 800, 1000, 1200, 1400},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -233,7 +234,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{360, 560, 760, 960, 1160, 1360},
+			Values:     []float64{410, 610, 810, 1010, 1210, 1410},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -248,21 +249,21 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort((label_set(time() offset -100s, "foo", "bar"), label_set(time()+10, "foo", "baz") offset -50s) offset -400s)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1260, 1460, 1660, 1860, 2060, 2260},
+			Values:     []float64{1400, 1600, 1800, 2000, 2200, 2400},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("foo"),
-			Value: []byte("baz"),
+			Value: []byte("bar"),
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1300, 1500, 1700, 1900, 2100, 2300},
+			Values:     []float64{1410, 1610, 1810, 2010, 2210, 2410},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("foo"),
-			Value: []byte("bar"),
+			Value: []byte("baz"),
 		}}
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
@@ -305,7 +306,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `time()[300s] offset 100s`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{800, 1000, 1200, 1400, 1600, 1800},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -338,7 +339,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(123)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -349,7 +350,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(time())`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -360,7 +361,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(456/time()+123)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -371,7 +372,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(time()>=1600)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, nan, 1700, 1900},
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -581,10 +582,6 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1, 1, 1, 1, 1, 1},
 			Timestamps: timestampsExpected,
 		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("yy"),
-			Value: []byte("foo"),
-		}}
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -635,6 +632,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1000, 1200, 1400, 1400, 1400, 1400},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -646,6 +644,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1000, 1200, 1400, 1400, 1400, 1400},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -3630,9 +3629,10 @@ func TestExecSuccess(t *testing.T) {
 		q := `round(geomean_over_time(alias(time()/100, "foobar")[3i]), 0.1)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6.8, 8.8, 10.9, 12.9, 14.9, 16.9},
+			Values:     []float64{7.8, 9.9, 11.9, 13.9, 15.9, 17.9},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -3652,7 +3652,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sum2_over_time(alias(time()/100, "foobar")[3i])`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{155, 251, 371, 515, 683, 875},
+			Values:     []float64{200, 308, 440, 596, 776, 980},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -3986,6 +3986,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`count_gt_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `count_gt_over_time(rand(0)[200s:10s], 0.7)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{7, 6, 10, 6, 6, 5},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`count_le_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `count_le_over_time(rand(0)[200s:10s], 0.7)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 14, 10, 14, 14, 15},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increases_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `increases_over_time(rand(0)[200s:10s])`
@@ -4427,7 +4449,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[500s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, 2, 1},
+			Values:     []float64{3, 3, 3, 3, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -4438,7 +4460,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[2.5i])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, 2, 1},
+			Values:     []float64{3, 3, 3, 3, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -4713,10 +4735,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`ru(time() offset 1i, 2000)`, func(t *testing.T) {
 		t.Parallel()
-		q := `ru(time() offset 1i, 2000)`
+		q := `ru(time() offset 1.5i, 2000)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{65, 55.00000000000001, 45, 35, 25, 15},
+			Values:     []float64{70, 60, 50, 40, 30, 20},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -4801,10 +4823,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`integrate(time())`, func(t *testing.T) {
 		t.Parallel()
-		q := `integrate(time()*1e-3)`
+		q := `integrate(time()/1e3)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{160, 200, 240.00000000000003, 280, 320, 360},
+			Values:     []float64{160, 200, 240, 280, 320, 360},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -4903,7 +4925,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `increase(2000-time())`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1100, 900, 700, 500, 300, 100},
+			Values:     []float64{1000, 800, 600, 400, 200, 0},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5874,7 +5896,7 @@ func TestExecError(t *testing.T) {
 			Start:    1000,
 			End:      2000,
 			Step:     100,
-			Deadline: netstorage.NewDeadline(time.Now(), time.Minute, ""),
+			Deadline: searchutils.NewDeadline(time.Now(), time.Minute, ""),
 		}
 		for i := 0; i < 4; i++ {
 			rv, err := Exec(ec, q, false)
@@ -6006,6 +6028,10 @@ func TestExecError(t *testing.T) {
 	f(`prometheus_buckets()`)
 	f(`buckets_limit()`)
 	f(`buckets_limit(1)`)
+	f(`share_le_over_time()`)
+	f(`share_gt_over_time()`)
+	f(`count_le_over_time()`)
+	f(`count_gt_over_time()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)

app/vmselect/promql/rollup.go

@@ -62,6 +62,8 @@ var rollupFuncs = map[string]newRollupFunc{
 	"tmax_over_time":        newRollupFuncOneArg(rollupTmax),
 	"share_le_over_time":    newRollupShareLE,
 	"share_gt_over_time":    newRollupShareGT,
+	"count_le_over_time":    newRollupCountLE,
+	"count_gt_over_time":    newRollupCountGT,
 	"histogram_over_time":   newRollupFuncOneArg(rollupHistogram),
 	"rollup":                newRollupFuncOneArg(rollupFake),
 	"rollup_rate":           newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
@@ -167,12 +169,20 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 }
 
 var rollupFuncsKeepMetricGroup = map[string]bool{
+	"holt_winters":          true,
+	"predict_linear":        true,
 	"default_rollup":        true,
+	"avg_over_time":         true,
+	"min_over_time":         true,
+	"max_over_time":         true,
+	"quantile_over_time":    true,
 	"rollup":                true,
+	"geomean_over_time":     true,
 	"hoeffding_bound_lower": true,
 	"hoeffding_bound_upper": true,
 	"first_over_time":       true,
 	"last_over_time":        true,
+	"mode_over_time":        true,
 }
 
 func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
@@ -472,6 +482,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
+		if rc.LookbackDelta > 0 && window > rc.LookbackDelta {
+			// Implicitly set window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
+			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
+			window = rc.LookbackDelta
+		}
 	}
 	if rc.MayAdjustWindow && window < maxPrevInterval {
 		window = maxPrevInterval
@@ -485,6 +500,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	j := 0
 	ni := 0
 	nj := 0
+	stalenessInterval := int64(float64(scrapeInterval) * 0.9)
 	for _, tEnd := range rc.Timestamps {
 		tStart := tEnd - window
 		ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni)
@@ -501,9 +517,17 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 			rfa.prevValue = values[i-1]
 			rfa.prevTimestamp = timestamps[i-1]
 		}
-
 		rfa.values = values[i:j]
 		rfa.timestamps = timestamps[i:j]
+		if j == len(timestamps) && i < j && tEnd-timestamps[j-1] > stalenessInterval {
+			// Do not take into account the last data point in time series if the distance between this data point
+			// and tEnd exceeds stalenessInterval.
+			// This should prevent from double counting when a label changes in time series (for instance,
+			// during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
+			rfa.prevValue = nan
+			rfa.values = nil
+			rfa.timestamps = nil
+		}
 		rfa.currTimestamp = tEnd
 		value := rc.Func(rfa)
 		rfa.idx++
@@ -668,7 +692,7 @@ func derivValues(values []float64, timestamps []int64) {
 			values[i] = prevDeriv
 			continue
 		}
-		dt := float64(ts-prevTs) * 1e-3
+		dt := float64(ts-prevTs) / 1e3
 		prevDeriv = (v - prevValue) / dt
 		values[i] = prevDeriv
 		prevValue = v
@@ -788,7 +812,7 @@ func linearRegression(rfa *rollupFuncArg) (float64, float64) {
 		n = 0
 	}
 	for i, v := range values {
-		dt := float64(timestamps[i]-tFirst) * 1e-3
+		dt := float64(timestamps[i]-tFirst) / 1e3
 		vSum += v
 		tSum += dt
 		tvSum += dt * v
@@ -801,7 +825,7 @@ func linearRegression(rfa *rollupFuncArg) (float64, float64) {
 	k := (n*tvSum - tSum*vSum) / (n*ttSum - tSum*tSum)
 	v := (vSum - k*tSum) / n
 	// Adjust v to the last timestamp on the given time range.
-	v += k * (float64(timestamps[len(timestamps)-1]-tFirst) * 1e-3)
+	v += k * (float64(timestamps[len(timestamps)-1]-tFirst) / 1e3)
 	return v, k
 }
 
@@ -834,6 +858,25 @@ func countFilterGT(values []float64, gt float64) int {
 }
 
 func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
+	rf, err := newRollupCountFilter(args, countFilter)
+	if err != nil {
+		return nil, err
+	}
+	return func(rfa *rollupFuncArg) float64 {
+		n := rf(rfa)
+		return n / float64(len(rfa.values))
+	}, nil
+}
+
+func newRollupCountLE(args []interface{}) (rollupFunc, error) {
+	return newRollupCountFilter(args, countFilterLE)
+}
+
+func newRollupCountGT(args []interface{}) (rollupFunc, error) {
+	return newRollupCountFilter(args, countFilterGT)
+}
+
+func newRollupCountFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 2); err != nil {
 		return nil, err
 	}
@@ -849,8 +892,7 @@ func newRollupShareFilter(args []interface{}, countFilter func(values []float64,
 			return nan
 		}
 		limit := limits[rfa.idx]
-		n := countFilter(values, limit)
-		return float64(n) / float64(len(values))
+		return float64(countFilter(values, limit))
 	}
 	return rf, nil
 }
@@ -1035,7 +1077,7 @@ func rollupTmin(rfa *rollupFuncArg) float64 {
 			minTimestamp = timestamps[i]
 		}
 	}
-	return float64(minTimestamp) * 1e-3
+	return float64(minTimestamp) / 1e3
 }
 
 func rollupTmax(rfa *rollupFuncArg) float64 {
@@ -1054,7 +1096,7 @@ func rollupTmax(rfa *rollupFuncArg) float64 {
 			maxTimestamp = timestamps[i]
 		}
 	}
-	return float64(maxTimestamp) * 1e-3
+	return float64(maxTimestamp) / 1e3
 }
 
 func rollupSum(rfa *rollupFuncArg) float64 {
@@ -1283,7 +1325,7 @@ func rollupDerivFast(rfa *rollupFuncArg) float64 {
 	vEnd := values[len(values)-1]
 	tEnd := timestamps[len(timestamps)-1]
 	dv := vEnd - prevValue
-	dt := float64(tEnd-prevTimestamp) * 1e-3
+	dt := float64(tEnd-prevTimestamp) / 1e3
 	return dv / dt
 }
 
@@ -1309,7 +1351,7 @@ func rollupIderiv(rfa *rollupFuncArg) float64 {
 			// So just return nan
 			return nan
 		}
-		return (values[0] - rfa.prevValue) / (float64(timestamps[0]-rfa.prevTimestamp) * 1e-3)
+		return (values[0] - rfa.prevValue) / (float64(timestamps[0]-rfa.prevTimestamp) / 1e3)
 	}
 	vEnd := values[len(values)-1]
 	tEnd := timestamps[len(timestamps)-1]
@@ -1333,7 +1375,7 @@ func rollupIderiv(rfa *rollupFuncArg) float64 {
 	}
 	dv := vEnd - vStart
 	dt := tEnd - tStart
-	return dv / (float64(dt) * 1e-3)
+	return dv / (float64(dt) / 1e3)
 }
 
 func rollupLifetime(rfa *rollupFuncArg) float64 {
@@ -1343,12 +1385,12 @@ func rollupLifetime(rfa *rollupFuncArg) float64 {
 		if len(timestamps) < 2 {
 			return nan
 		}
-		return float64(timestamps[len(timestamps)-1]-timestamps[0]) * 1e-3
+		return float64(timestamps[len(timestamps)-1]-timestamps[0]) / 1e3
 	}
 	if len(timestamps) == 0 {
 		return nan
 	}
-	return float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) * 1e-3
+	return float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) / 1e3
 }
 
 func rollupLag(rfa *rollupFuncArg) float64 {
@@ -1358,9 +1400,9 @@ func rollupLag(rfa *rollupFuncArg) float64 {
 		if math.IsNaN(rfa.prevValue) {
 			return nan
 		}
-		return float64(rfa.currTimestamp-rfa.prevTimestamp) * 1e-3
+		return float64(rfa.currTimestamp-rfa.prevTimestamp) / 1e3
 	}
-	return float64(rfa.currTimestamp-timestamps[len(timestamps)-1]) * 1e-3
+	return float64(rfa.currTimestamp-timestamps[len(timestamps)-1]) / 1e3
 }
 
 func rollupScrapeInterval(rfa *rollupFuncArg) float64 {
@@ -1370,12 +1412,12 @@ func rollupScrapeInterval(rfa *rollupFuncArg) float64 {
 		if len(timestamps) < 2 {
 			return nan
 		}
-		return float64(timestamps[len(timestamps)-1]-timestamps[0]) * 1e-3 / float64(len(timestamps)-1)
+		return (float64(timestamps[len(timestamps)-1]-timestamps[0]) / 1e3) / float64(len(timestamps)-1)
 	}
 	if len(timestamps) == 0 {
 		return nan
 	}
-	return (float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) * 1e-3) / float64(len(timestamps))
+	return (float64(timestamps[len(timestamps)-1]-rfa.prevTimestamp) / 1e3) / float64(len(timestamps))
 }
 
 func rollupChanges(rfa *rollupFuncArg) float64 {
@@ -1554,7 +1596,23 @@ func rollupTimestamp(rfa *rollupFuncArg) float64 {
 func rollupModeOverTime(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
-	return modeNoNaNs(rfa.prevValue, rfa.values)
+
+	// Copy rfa.values to a.A, since modeNoNaNs modifies a.A contents.
+	a := float64sPool.Get().(*float64s)
+	a.A = append(a.A[:0], rfa.values...)
+	result := modeNoNaNs(rfa.prevValue, a.A)
+	float64sPool.Put(a)
+	return result
+}
+
+var float64sPool = &sync.Pool{
+	New: func() interface{} {
+		return &float64s{}
+	},
+}
+
+type float64s struct {
+	A []float64
 }
 
 func rollupAscentOverTime(rfa *rollupFuncArg) float64 {
@@ -1663,37 +1721,32 @@ func rollupDistinct(rfa *rollupFuncArg) float64 {
 }
 
 func rollupIntegrate(rfa *rollupFuncArg) float64 {
-	prevTimestamp := rfa.prevTimestamp
-
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
 	values := rfa.values
 	timestamps := rfa.timestamps
-	if len(values) == 0 {
-		if math.IsNaN(rfa.prevValue) {
+	prevValue := rfa.prevValue
+	prevTimestamp := rfa.currTimestamp - rfa.window
+	if math.IsNaN(prevValue) {
+		if len(values) == 0 {
 			return nan
 		}
-		return 0
-	}
-	prevValue := rfa.prevValue
-	if math.IsNaN(prevValue) {
 		prevValue = values[0]
 		prevTimestamp = timestamps[0]
 		values = values[1:]
 		timestamps = timestamps[1:]
 	}
-	if len(values) == 0 {
-		return 0
-	}
 
 	var sum float64
 	for i, v := range values {
 		timestamp := timestamps[i]
-		dt := float64(timestamp-prevTimestamp) * 1e-3
-		sum += 0.5 * (v + prevValue) * dt
+		dt := float64(timestamp-prevTimestamp) / 1e3
+		sum += prevValue * dt
 		prevTimestamp = timestamp
 		prevValue = v
 	}
+	dt := float64(rfa.currTimestamp-prevTimestamp) / 1e3
+	sum += prevValue * dt
 	return sum
 }
 

app/vmselect/promql/rollup_test.go

@@ -139,16 +139,16 @@ func TestDerivValues(t *testing.T) {
 
 	values = append([]float64{}, testValues...)
 	derivValues(values, testTimestamps)
-	valuesExpected = []float64{-8900, 1111.111111111111, -1916.6666666666665, 2538.461538461538, -1818.1818181818182, 3611.111111111111,
-		-43500, 1882.3529411764705, -666.6666666666666, 400, 0, 0}
+	valuesExpected = []float64{-8900, 1111.111111111111, -1916.6666666666665, 2538.4615384615386, -1818.1818181818182, 3611.1111111111113,
+		-43500, 1882.3529411764705, -666.6666666666667, 400, 0, 0}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
 
 	// remove counter resets
 	values = append([]float64{}, testValues...)
 	removeCounterResets(values)
 	derivValues(values, testTimestamps)
-	valuesExpected = []float64{3400, 1111.111111111111, 1750, 2538.461538461538, 3090.909090909091, 3611.111111111111,
-		6000, 1882.3529411764705, 1777.7777777777776, 400, 0, 0}
+	valuesExpected = []float64{3400, 1111.111111111111, 1750, 2538.4615384615386, 3090.909090909091, 3611.1111111111113,
+		6000, 1882.3529411764705, 1777.7777777777778, 400, 0, 0}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
 
 	// duplicate timestamps
@@ -239,6 +239,52 @@ func TestRollupShareGTOverTime(t *testing.T) {
 	f(1000, 0)
 }
 
+func TestRollupCountLEOverTime(t *testing.T) {
+	f := func(le, vExpected float64) {
+		t.Helper()
+		les := []*timeseries{{
+			Values:     []float64{le},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
+		testRollupFunc(t, "count_le_over_time", args, &me, vExpected)
+	}
+
+	f(-123, 0)
+	f(0, 0)
+	f(10, 0)
+	f(12, 1)
+	f(30, 2)
+	f(50, 9)
+	f(100, 11)
+	f(123, 12)
+	f(1000, 12)
+}
+
+func TestRollupCountGTOverTime(t *testing.T) {
+	f := func(gt, vExpected float64) {
+		t.Helper()
+		gts := []*timeseries{{
+			Values:     []float64{gt},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, gts}
+		testRollupFunc(t, "count_gt_over_time", args, &me, vExpected)
+	}
+
+	f(-123, 12)
+	f(0, 12)
+	f(10, 12)
+	f(12, 11)
+	f(30, 10)
+	f(50, 3)
+	f(100, 1)
+	f(123, 0)
+	f(1000, 0)
+}
+
 func TestRollupQuantileOverTime(t *testing.T) {
 	f := func(phi, vExpected float64) {
 		t.Helper()
@@ -385,7 +431,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("stdvar_over_time", 945.7430555555555)
 	f("first_over_time", 123)
 	f("last_over_time", 34)
-	f("integrate", 5.4705)
+	f("integrate", 0.817)
 	f("distinct_over_time", 8)
 	f("ideriv", 0)
 	f("decreases_over_time", 5)
@@ -537,7 +583,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 123, 34, 32}
+		valuesExpected := []float64{nan, nan, 123, 34, nan}
 		timestampsExpected := []int64{-50, 0, 50, 100, 150}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -599,7 +645,7 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
+		valuesExpected := []float64{12, nan, nan, nan, 34, 34, nan}
 		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -644,7 +690,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 54, 44, 34}
+		valuesExpected := []float64{nan, 123, 54, 44, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -658,7 +704,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 1}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -672,7 +718,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 12, 32, 34}
+		valuesExpected := []float64{nan, 21, 12, 32, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -686,7 +732,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 99, 44, 34}
+		valuesExpected := []float64{nan, 123, 99, 44, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -700,7 +746,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 222, 199, 110, 34}
+		valuesExpected := []float64{nan, 222, 199, 110, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -714,7 +760,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, -9, 22, 0}
+		valuesExpected := []float64{nan, nan, -9, 22, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -742,7 +788,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
+		valuesExpected := []float64{nan, 0.004, 0, 0, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -756,7 +802,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01}
+		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -770,7 +816,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125}
+		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -784,7 +830,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -798,7 +844,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -812,7 +858,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 0}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -840,7 +886,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2, 2, 1, 0}
+		valuesExpected := []float64{nan, 2, 2, 1, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -854,7 +900,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34}
+		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -868,7 +914,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{0, -2879.310344827587, 558.0608793686592, 422.84569138276544, 0}
+		valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -896,7 +942,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0}
+		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -910,7 +956,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 5.830951894845301}
+		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -924,7 +970,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 1.526, 2.2795, 1.325, 0.34}
+		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -938,7 +984,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 1}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -952,7 +998,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 7, 6, 3}
+		valuesExpected := []float64{nan, 4, 7, 6, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -966,7 +1012,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 34, 44, 44}
+		valuesExpected := []float64{nan, 21, 34, 34, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -980,7 +1026,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 1262.5, 3187.5, 4059.523809523809, 6200}
+		valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -994,7 +1040,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.9397878236968458, 1.1969836716333457, 2.3112921116373175, nan}
+		valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1016,7 +1062,7 @@ func TestRollupBigNumberOfValues(t *testing.T) {
 		srcTimestamps[i] = int64(i / 2)
 	}
 	values := rc.Do(nil, srcValues, srcTimestamps)
-	valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan}
+	valuesExpected := []float64{1, 4001, 8001, nan, nan, nan}
 	timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000}
 	testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 }

app/vmselect/promql/transform.go

@@ -17,6 +17,27 @@ import (
 	"github.com/valyala/histogram"
 )
 
+var transformFuncsKeepMetricGroup = map[string]bool{
+	"ceil":               true,
+	"clamp_max":          true,
+	"clamp_min":          true,
+	"floor":              true,
+	"round":              true,
+	"keep_last_value":    true,
+	"keep_next_value":    true,
+	"interpolate":        true,
+	"running_min":        true,
+	"running_max":        true,
+	"running_avg":        true,
+	"range_min":          true,
+	"range_max":          true,
+	"range_avg":          true,
+	"range_first":        true,
+	"range_last":         true,
+	"range_quantile":     true,
+	"smooth_exponential": true,
+}
+
 var transformFuncs = map[string]transformFunc{
 	// Standard promql funcs
 	// See funcs accepting instant-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
@@ -125,8 +146,12 @@ func newTransformFuncOneArg(tf func(v float64) float64) transformFunc {
 }
 
 func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) {
+	name := strings.ToLower(fe.Name)
+	keepMetricGroup := transformFuncsKeepMetricGroup[name]
 	for _, ts := range arg {
-		ts.MetricName.ResetMetricGroup()
+		if !keepMetricGroup {
+			ts.MetricName.ResetMetricGroup()
+		}
 		tf(ts.Values)
 	}
 	return arg, nil
@@ -142,23 +167,24 @@ func transformAbsent(tfa *transformFuncArg) ([]*timeseries, error) {
 	if err := expectTransformArgsNum(args, 1); err != nil {
 		return nil, err
 	}
-	arg := args[0]
-	if len(arg) == 0 {
-		rvs := getAbsentTimeseries(tfa.ec, tfa.fe.Args[0])
+	tss := args[0]
+	rvs := getAbsentTimeseries(tfa.ec, tfa.fe.Args[0])
+	if len(tss) == 0 {
 		return rvs, nil
 	}
-	for _, ts := range arg {
-		ts.MetricName.ResetMetricGroup()
-		for i, v := range ts.Values {
-			if !math.IsNaN(v) {
-				v = nan
-			} else {
-				v = 1
+	for i := range tss[0].Values {
+		isAbsent := true
+		for _, ts := range tss {
+			if !math.IsNaN(ts.Values[i]) {
+				isAbsent = false
+				break
 			}
-			ts.Values[i] = v
+		}
+		if !isAbsent {
+			rvs[0].Values[i] = nan
 		}
 	}
-	return arg, nil
+	return rvs, nil
 }
 
 func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries {
@@ -912,10 +938,9 @@ func newTransformFuncRunning(rf func(a, b float64, idx int) float64) transformFu
 			prevValue := values[0]
 			values = values[1:]
 			for i, v := range values {
-				if math.IsNaN(v) {
-					continue
+				if !math.IsNaN(v) {
+					prevValue = rf(prevValue, v, i+1)
 				}
-				prevValue = rf(prevValue, v, i+1)
 				values[i] = prevValue
 			}
 		}
@@ -1030,6 +1055,12 @@ func transformSmoothExponential(tfa *transformFuncArg) ([]*timeseries, error) {
 	rvs := args[0]
 	for _, ts := range rvs {
 		values := skipLeadingNaNs(ts.Values)
+		for i, v := range values {
+			if !math.IsInf(v, 0) {
+				values = values[i:]
+				break
+			}
+		}
 		if len(values) == 0 {
 			continue
 		}
@@ -1040,6 +1071,10 @@ func transformSmoothExponential(tfa *transformFuncArg) ([]*timeseries, error) {
 			if math.IsNaN(v) {
 				continue
 			}
+			if math.IsInf(v, 0) {
+				values[i] = avg
+				continue
+			}
 			sf := sfsX[i]
 			if math.IsNaN(sf) {
 				sf = 1
@@ -1674,15 +1709,15 @@ func newTransformFuncZeroArgs(f func(tfa *transformFuncArg) float64) transformFu
 }
 
 func transformStep(tfa *transformFuncArg) float64 {
-	return float64(tfa.ec.Step) * 1e-3
+	return float64(tfa.ec.Step) / 1e3
 }
 
 func transformStart(tfa *transformFuncArg) float64 {
-	return float64(tfa.ec.Start) * 1e-3
+	return float64(tfa.ec.Start) / 1e3
 }
 
 func transformEnd(tfa *transformFuncArg) float64 {
-	return float64(tfa.ec.End) * 1e-3
+	return float64(tfa.ec.End) / 1e3
 }
 
 // copyTimeseriesMetricNames returns a copy of tss with real copy of MetricNames,

app/vmselect/searchutils/searchutils.go

@@ -0,0 +1,188 @@
+package searchutils
+
+import (
+	"flag"
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+var (
+	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
+	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
+)
+
+func roundToSeconds(ms int64) int64 {
+	return ms - ms%1000
+}
+
+// GetTime returns time from the given argKey query arg.
+//
+// If argKey is missing in r, then defaultMs rounded to seconds is returned.
+// The rounding is needed in order to align query results in Grafana
+// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720
+func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
+	argValue := r.FormValue(argKey)
+	if len(argValue) == 0 {
+		return roundToSeconds(defaultMs), nil
+	}
+	secs, err := strconv.ParseFloat(argValue, 64)
+	if err != nil {
+		// Try parsing string format
+		t, err := time.Parse(time.RFC3339, argValue)
+		if err != nil {
+			// Handle Prometheus'-provided minTime and maxTime.
+			// See https://github.com/prometheus/client_golang/issues/614
+			switch argValue {
+			case prometheusMinTimeFormatted:
+				return minTimeMsecs, nil
+			case prometheusMaxTimeFormatted:
+				return maxTimeMsecs, nil
+			}
+			// Try parsing duration relative to the current time
+			d, err1 := metricsql.DurationValue(argValue, 0)
+			if err1 != nil {
+				return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
+			}
+			if d > 0 {
+				d = -d
+			}
+			t = time.Now().Add(time.Duration(d) * time.Millisecond)
+		}
+		secs = float64(t.UnixNano()) / 1e9
+	}
+	msecs := int64(secs * 1e3)
+	if msecs < minTimeMsecs {
+		msecs = 0
+	}
+	if msecs > maxTimeMsecs {
+		msecs = maxTimeMsecs
+	}
+	return msecs, nil
+}
+
+var (
+	// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
+	// See https://github.com/prometheus/client_golang/issues/614 for details.
+	prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano)
+	prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano)
+)
+
+const (
+	// These values prevent from overflow when storing msec-precision time in int64.
+	minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
+	maxTimeMsecs = int64(1<<63-1) / 1e6
+)
+
+// GetDuration returns duration from the given argKey query arg.
+func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) {
+	argValue := r.FormValue(argKey)
+	if len(argValue) == 0 {
+		return defaultValue, nil
+	}
+	secs, err := strconv.ParseFloat(argValue, 64)
+	if err != nil {
+		// Try parsing string format
+		d, err := metricsql.DurationValue(argValue, 0)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
+		}
+		secs = float64(d) / 1000
+	}
+	msecs := int64(secs * 1e3)
+	if msecs <= 0 || msecs > maxDurationMsecs {
+		return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs))
+	}
+	return msecs, nil
+}
+
+const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
+
+// GetMaxQueryDuration returns the maximum duration for query from r.
+func GetMaxQueryDuration(r *http.Request) time.Duration {
+	dms, err := GetDuration(r, "timeout", 0)
+	if err != nil {
+		dms = 0
+	}
+	d := time.Duration(dms) * time.Millisecond
+	if d <= 0 || d > *maxQueryDuration {
+		d = *maxQueryDuration
+	}
+	return d
+}
+
+// GetDeadlineForQuery returns deadline for the given query r.
+func GetDeadlineForQuery(r *http.Request, startTime time.Time) Deadline {
+	dMax := maxQueryDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxQueryDuration")
+}
+
+// GetDeadlineForExport returns deadline for the given request to /api/v1/export.
+func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline {
+	dMax := maxExportDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
+}
+
+func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline {
+	d, err := GetDuration(r, "timeout", 0)
+	if err != nil {
+		d = 0
+	}
+	if d <= 0 || d > dMax {
+		d = dMax
+	}
+	timeout := time.Duration(d) * time.Millisecond
+	return NewDeadline(startTime, timeout, flagHint)
+}
+
+// GetBool returns boolean value from the given argKey query arg.
+func GetBool(r *http.Request, argKey string) bool {
+	argValue := r.FormValue(argKey)
+	switch strings.ToLower(argValue) {
+	case "", "0", "f", "false", "no":
+		return false
+	default:
+		return true
+	}
+}
+
+// Deadline contains deadline with the corresponding timeout for pretty error messages.
+type Deadline struct {
+	deadline uint64
+
+	timeout  time.Duration
+	flagHint string
+}
+
+// NewDeadline returns deadline for the given timeout.
+//
+// flagHint must contain a hit for command-line flag, which could be used
+// in order to increase timeout.
+func NewDeadline(startTime time.Time, timeout time.Duration, flagHint string) Deadline {
+	return Deadline{
+		deadline: uint64(startTime.Add(timeout).Unix()),
+		timeout:  timeout,
+		flagHint: flagHint,
+	}
+}
+
+// Exceeded returns true if deadline is exceeded.
+func (d *Deadline) Exceeded() bool {
+	return fasttime.UnixTimestamp() > d.deadline
+}
+
+// Deadline returns deadline in unix timestamp seconds.
+func (d *Deadline) Deadline() uint64 {
+	return d.deadline
+}
+
+// String returns human-readable string representation for d.
+func (d *Deadline) String() string {
+	return fmt.Sprintf("%.3f seconds; the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), d.flagHint)
+}

app/vmselect/searchutils/searchutils_test.go

@@ -0,0 +1,78 @@
+package searchutils
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func TestGetTimeSuccess(t *testing.T) {
+	f := func(s string, timestampExpected int64) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		// Verify defaultValue
+		ts, err := GetTime(r, "foo", 123456)
+		if err != nil {
+			t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
+		}
+		if ts != 123000 {
+			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
+		}
+
+		// Verify timestampExpected
+		ts, err = GetTime(r, "s", 123)
+		if err != nil {
+			t.Fatalf("unexpected error in GetTime(%q): %s", s, err)
+		}
+		if ts != timestampExpected {
+			t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected)
+		}
+	}
+
+	f("2019-07-07T20:01:02Z", 1562529662000)
+	f("2019-07-07T20:47:40+03:00", 1562521660000)
+	f("-292273086-05-16T16:47:06Z", minTimeMsecs)
+	f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
+	f("1562529662.324", 1562529662324)
+	f("-9223372036.854", minTimeMsecs)
+	f("-9223372036.855", minTimeMsecs)
+	f("9223372036.855", maxTimeMsecs)
+}
+
+func TestGetTimeError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		// Verify defaultValue
+		ts, err := GetTime(r, "foo", 123456)
+		if err != nil {
+			t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
+		}
+		if ts != 123000 {
+			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
+		}
+
+		// Verify timestampExpected
+		_, err = GetTime(r, "s", 123)
+		if err == nil {
+			t.Fatalf("expecting non-nil error in GetTime(%q)", s)
+		}
+	}
+
+	f("foo")
+	f("2019-07-07T20:01:02Zisdf")
+	f("2019-07-07T20:47:40+03:00123")
+	f("-292273086-05-16T16:47:07Z")
+	f("292277025-08-18T07:12:54.999999998Z")
+}

app/vmstorage/main.go

@@ -19,14 +19,18 @@ import (
 )
 
 var (
-	retentionPeriod = flag.Int("retentionPeriod", 1, "Retention period in months")
-	snapshotAuthKey = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
+	retentionPeriod   = flag.Int("retentionPeriod", 1, "Retention period in months")
+	snapshotAuthKey   = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
+	forceMergeAuthKey = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
 
 	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
 
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
+	finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
+		"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
+		"disk IO usage and CPU usage")
 	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
 	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
 
@@ -64,6 +68,7 @@ func InitWithoutMetrics() {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
 
+	storage.SetFinalMergeDelay(*finalMergeDelay)
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
 
@@ -132,6 +137,16 @@ func SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string
 	return values, err
 }
 
+// SearchTagValueSuffixes returns all the tag value suffixes for the given tagKey and tagValuePrefix on the given tr.
+//
+// This allows implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or similar APIs.
+func SearchTagValueSuffixes(tr storage.TimeRange, tagKey, tagValuePrefix []byte, delimiter byte, maxTagValueSuffixes int, deadline uint64) ([]string, error) {
+	WG.Add(1)
+	suffixes, err := Storage.SearchTagValueSuffixes(tr, tagKey, tagValuePrefix, delimiter, maxTagValueSuffixes, deadline)
+	WG.Done()
+	return suffixes, err
+}
+
 // SearchTagEntries searches for tag entries.
 func SearchTagEntries(maxTagKeys, maxTagValues int, deadline uint64) ([]storage.TagEntry, error) {
 	WG.Add(1)
@@ -170,6 +185,26 @@ func Stop() {
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
+	if path == "/internal/force_merge" {
+		authKey := r.FormValue("authKey")
+		if authKey != *forceMergeAuthKey {
+			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceMergeAuthKey command line flag", authKey)
+			return true
+		}
+		// Run force merge in background
+		partitionNamePrefix := r.FormValue("partition_prefix")
+		go func() {
+			activeForceMerges.Inc()
+			defer activeForceMerges.Dec()
+			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+			startTime := time.Now()
+			if err := Storage.ForceMergePartitions(partitionNamePrefix); err != nil {
+				logger.Errorf("error in forced merge for partition_prefix=%q: %s", partitionNamePrefix, err)
+			}
+			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+		}()
+		return true
+	}
 	prometheusCompatibleResponse := false
 	if path == "/api/v1/admin/tsdb/snapshot" {
 		// Handle Prometheus API - https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot .
@@ -250,6 +285,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+var activeForceMerges = metrics.NewCounter("vm_active_force_merges")
+
 func registerStorageMetrics() {
 	mCache := &storage.Metrics{}
 	var mCacheLock sync.Mutex
@@ -339,12 +376,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_missing_tsids_for_metric_id_total`, func() float64 {
 		return float64(idbm().MissingTSIDsForMetricID)
 	})
-	metrics.NewGauge(`vm_recent_hour_metric_ids_search_calls_total`, func() float64 {
-		return float64(idbm().RecentHourMetricIDsSearchCalls)
-	})
-	metrics.NewGauge(`vm_recent_hour_metric_ids_search_hits_total`, func() float64 {
-		return float64(idbm().RecentHourMetricIDsSearchHits)
-	})
 	metrics.NewGauge(`vm_date_metric_ids_search_calls_total`, func() float64 {
 		return float64(idbm().DateMetricIDsSearchCalls)
 	})
@@ -365,6 +396,14 @@ func registerStorageMetrics() {
 		return float64(idbm().AssistedMerges)
 	})
 
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686
+	metrics.NewGauge(`vm_merge_need_free_disk_space{type="storage/small"}`, func() float64 {
+		return float64(tm().SmallMergeNeedFreeDiskSpace)
+	})
+	metrics.NewGauge(`vm_merge_need_free_disk_space{type="storage/big"}`, func() float64 {
+		return float64(tm().BigMergeNeedFreeDiskSpace)
+	})
+
 	metrics.NewGauge(`vm_pending_rows{type="storage"}`, func() float64 {
 		return float64(tm().PendingRows)
 	})
@@ -402,6 +441,9 @@ func registerStorageMetrics() {
 		return float64(idbm().SizeBytes)
 	})
 
+	metrics.NewGauge(`vm_rows_added_to_storage_total`, func() float64 {
+		return float64(m().RowsAddedTotal)
+	})
 	metrics.NewGauge(`vm_deduplicated_samples_total{type="merge"}`, func() float64 {
 		return float64(m().DedupsDuringMerge)
 	})
@@ -456,6 +498,13 @@ func registerStorageMetrics() {
 		return float64(m().SlowMetricNameLoads)
 	})
 
+	metrics.NewGauge(`vm_timestamps_blocks_merged_total`, func() float64 {
+		return float64(m().TimestampsBlocksMerged)
+	})
+	metrics.NewGauge(`vm_timestamps_bytes_saved_total`, func() float64 {
+		return float64(m().TimestampsBytesSaved)
+	})
+
 	metrics.NewGauge(`vm_rows{type="storage/big"}`, func() float64 {
 		return float64(tm().BigRowsCount)
 	})

dashboards/vmagent.json

@@ -5,7 +5,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "7.0.3"
+      "version": "7.1.1"
     },
     {
       "type": "panel",
@@ -51,12 +51,12 @@
       }
     ]
   },
-  "description": "Overview for VictoriaMetrics vmagent v1.38.1 or higher",
+  "description": "Overview for VictoriaMetrics vmagent v1.40.0 or higher",
   "editable": true,
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1594939790534,
+  "iteration": 1598997251171,
   "links": [
     {
       "icon": "doc",
@@ -136,9 +136,10 @@
           ],
           "fields": "",
           "values": false
-        }
+        },
+        "textMode": "auto"
       },
-      "pluginVersion": "7.0.3",
+      "pluginVersion": "7.1.1",
       "targets": [
         {
           "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"})",
@@ -199,9 +200,10 @@
           ],
           "fields": "",
           "values": false
-        }
+        },
+        "textMode": "auto"
       },
-      "pluginVersion": "7.0.3",
+      "pluginVersion": "7.1.1",
       "targets": [
         {
           "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"})",
@@ -265,9 +267,10 @@
           ],
           "fields": "",
           "values": false
-        }
+        },
+        "textMode": "auto"
       },
-      "pluginVersion": "7.0.3",
+      "pluginVersion": "7.1.1",
       "targets": [
         {
           "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[30m]))",
@@ -323,9 +326,10 @@
           ],
           "fields": "",
           "values": false
-        }
+        },
+        "textMode": "auto"
       },
-      "pluginVersion": "7.0.3",
+      "pluginVersion": "7.1.1",
       "targets": [
         {
           "expr": "sum(vm_persistentqueue_bytes_pending{job=~\"$job\", instance=~\"$instance\"})",
@@ -444,7 +448,8 @@
       "datasource": "$ds",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -477,10 +482,8 @@
       "lines": true,
       "linewidth": 1,
       "nullPointMode": "null as zero",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -550,7 +553,8 @@
       "description": "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -576,10 +580,8 @@
       "lines": true,
       "linewidth": 1,
       "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -657,7 +659,8 @@
       "description": "Shows the rate of requests served by vmagent HTTP server.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -684,10 +687,8 @@
       "linewidth": 1,
       "links": [],
       "nullPointMode": "null as zero",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -727,7 +728,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -753,7 +754,8 @@
       "description": "Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -779,10 +781,8 @@
       "lines": true,
       "linewidth": 1,
       "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -859,7 +859,8 @@
       "description": "Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -892,10 +893,8 @@
         }
       ],
       "nullPointMode": "null as zero",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -959,7 +958,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -985,7 +984,8 @@
       "description": "Shows rate of dropped samples from persistent queue. VMagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -1018,10 +1018,8 @@
         }
       ],
       "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1061,7 +1059,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -1087,7 +1085,8 @@
       "description": "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -1119,10 +1118,8 @@
         }
       ],
       "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1162,7 +1159,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -1188,7 +1185,8 @@
       "description": "Shows the rate of dropped samples due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only.",
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "custom": {},
+          "links": []
         },
         "overrides": []
       },
@@ -1221,10 +1219,8 @@
         }
       ],
       "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
       "percentage": false,
+      "pluginVersion": "7.1.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1270,7 +1266,7 @@
           "label": null,
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -1307,18 +1303,7 @@
           "fieldConfig": {
             "defaults": {
               "custom": {},
-              "decimals": 0,
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              },
-              "unit": "short"
+              "links": []
             },
             "overrides": []
           },
@@ -1344,11 +1329,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
-          "pluginVersion": "7.0.3",
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1390,7 +1372,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -1416,18 +1398,7 @@
           "fieldConfig": {
             "defaults": {
               "custom": {},
-              "decimals": 0,
-              "mappings": [],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  }
-                ]
-              },
-              "unit": "short"
+              "links": []
             },
             "overrides": []
           },
@@ -1453,11 +1424,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
-          "pluginVersion": "7.0.3",
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1499,7 +1467,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -1524,7 +1492,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -1550,10 +1519,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1629,7 +1596,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -1655,10 +1623,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1716,7 +1682,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -1741,7 +1707,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -1767,10 +1734,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1817,7 +1782,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -1862,7 +1827,7 @@
             "y": 17
           },
           "heatmap": {},
-          "hideZeroBuckets": true,
+          "hideZeroBuckets": false,
           "highlightCards": true,
           "id": 33,
           "legend": {
@@ -1871,7 +1836,7 @@
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "prometheus_buckets(sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange))",
+              "expr": "buckets_limit(12, prometheus_buckets(sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange)))",
               "format": "heatmap",
               "interval": "",
               "intervalFactor": 10,
@@ -1929,7 +1894,8 @@
           "description": "Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -1956,10 +1922,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2005,7 +1969,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2031,7 +1995,8 @@
           "description": "Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2058,10 +2023,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2107,7 +2070,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2133,7 +2096,8 @@
           "description": "Shows the rate of parsed rows from write or scrape requests.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2160,10 +2124,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null as zero",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2203,7 +2165,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2229,7 +2191,8 @@
           "description": "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2255,10 +2218,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2298,7 +2259,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2339,7 +2300,8 @@
           "description": "Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2365,10 +2327,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2409,7 +2369,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2435,7 +2395,8 @@
           "description": "Shows the global rate for number of written bytes via remote write connections.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2461,10 +2422,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2504,7 +2463,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2530,7 +2489,8 @@
           "description": "Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2556,10 +2516,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2625,7 +2583,8 @@
           "description": "Shows current number of established connections to remote write endpoints.\n\n",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2651,10 +2610,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2694,7 +2651,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -2748,7 +2705,7 @@
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "prometheus_buckets(sum(rate(vmagent_remotewrite_block_size_rows_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange))  ",
+              "expr": "buckets_limit(12, prometheus_buckets(sum(rate(vmagent_remotewrite_block_size_rows_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange)))",
               "format": "heatmap",
               "interval": "",
               "intervalFactor": 10,
@@ -2819,7 +2776,7 @@
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "prometheus_buckets(sum(rate(vmagent_remotewrite_block_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange))  ",
+              "expr": "buckets_limit(12, prometheus_buckets(sum(rate(vmagent_remotewrite_block_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange)))",
               "format": "heatmap",
               "interval": "",
               "intervalFactor": 10,
@@ -2890,7 +2847,7 @@
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "prometheus_buckets(sum(rate(vmagent_remotewrite_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange))  ",
+              "expr": "buckets_limit(12, prometheus_buckets(sum(rate(vmagent_remotewrite_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(vmrange)))",
               "format": "heatmap",
               "interval": "",
               "intervalFactor": 10,
@@ -2948,7 +2905,8 @@
           "description": "Shows the CPU usage per vmagent instance. \nIf you think that usage is abnormal or unexpected pls file an issue and attach CPU profile if possible.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -2958,7 +2916,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 5
           },
           "hiddenSeries": false,
           "id": 35,
@@ -2981,10 +2939,8 @@
             }
           ],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3025,7 +2981,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -3051,7 +3007,8 @@
           "description": "Amount of used memory (resident)\n\nIf you think that usage is abnormal or unexpected pls file an issue and attach memory profile if possible.",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -3061,7 +3018,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 5
           },
           "hiddenSeries": false,
           "id": 37,
@@ -3084,10 +3041,8 @@
             }
           ],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3152,7 +3107,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -3162,7 +3118,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 13
           },
           "hiddenSeries": false,
           "id": 41,
@@ -3179,10 +3135,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3249,7 +3203,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -3259,7 +3214,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 13
           },
           "hiddenSeries": false,
           "id": 39,
@@ -3276,10 +3231,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3346,7 +3299,8 @@
           "datasource": "$ds",
           "fieldConfig": {
             "defaults": {
-              "custom": {}
+              "custom": {},
+              "links": []
             },
             "overrides": []
           },
@@ -3356,7 +3310,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 60
+            "y": 21
           },
           "hiddenSeries": false,
           "id": 43,
@@ -3373,10 +3327,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3417,7 +3369,7 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -3440,7 +3392,7 @@
     }
   ],
   "refresh": false,
-  "schemaVersion": 25,
+  "schemaVersion": 26,
   "style": "dark",
   "tags": [
     "vmagent",

deployment/docker/Makefile

@@ -4,7 +4,7 @@ DOCKER_NAMESPACE := victoriametrics
 
 ROOT_IMAGE ?= alpine:3.12
 CERTS_IMAGE := alpine:3.12
-GO_BUILDER_IMAGE := golang:1.15.0
+GO_BUILDER_IMAGE := golang:1.15.2
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _)
 BASE_IMAGE := local/base:1.1.1-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _)
 

docs/Articles.md

@@ -25,13 +25,17 @@
 * [Prometheus storage: tech terms for humans](https://medium.com/@valyala/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
 * [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://medium.com/@valyala/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)
 * [How to migrate data from Prometheus to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
+* [Filtering and modifying time series during import to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
+* [Anomaly Detection in VictoriaMetrics](https://medium.com/@VictoriaMetrics/anomaly-detection-in-victoriametrics-9528538786a7)
 
 
 ## Third-party articles and slides
 
 * [Better Prometheus rate() function with VictoriaMetrics](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/)
+* [Making peace with Prometheus rate()](https://blog.doit-intl.com/making-peace-with-prometheus-rate-43a3ea75c4cf)
 * [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
 * [Sismology: Iguana Solutions’ Monitoring System](https://medium.com/@IG1.com/sismology-iguana-solutions-monitoring-system-f46e4170447f)
+* [Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
 * [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit)
 * [CMS monitoring R&D: Real-time monitoring and alerts](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
 * [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
@@ -40,3 +44,6 @@
 * [What are Open Source Time Series Databases?](https://www.iunera.com/kraken/fabric/time-series-database/)
 * [Evaluating performance and correctness](https://www.robustperception.io/evaluating-performance-and-correctness)
 * [Running VictoriaMetrics on Raspberry PI](https://stas.starikevich.com/posts/raspberry-pi-4-prometheus/)
+* [Calculating the Error of Quantile Estimation with Histograms](https://linuxczar.net/blog/2020/08/13/histogram-error/)
+* [Monitoring private clouds with VictoriaMetrics at LeroyMerlin](https://www.youtube.com/watch?v=74swsWqf0Uc)
+* [Monitoring Kubernetes with VictoriaMetrics+Prometheus](https://speakerdeck.com/bo0km4n/victoriametrics-plus-prometheusdegou-zhu-surufu-shu-kubernetesfalsejian-shi-ji-pan)

docs/CaseStudies.md

@@ -73,13 +73,14 @@ See [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d
 
 Numbers:
 
-* The number of active time series per VictoriaMetrics instance is 20M.
+* The number of active time series per VictoriaMetrics instance is 40M.
 * The total number of time series per VictoriaMetrics instance is 400M+.
-* Ingestion rate per VictoriaMetrics instance is 800K data points per second.
+* Ingestion rate per VictoriaMetrics instance is 1M data points per second.
+* The total number of datapoints per VictoriaMetrics instance is 8 trillions.
 * The average time series churn rate is ~3M per day.
-* The average query rate is ~1K per minute (mostly alert queries).
-* Query duration: median is ~70ms, 99th percentile is ~2sec.
-* Retention: 6 months.
+* The average query rate is ~100 per second (mostly alert queries).
+* Query duration: median is ~70ms, 99th percentile is ~1.5sec.
+* Retention: 3 months.
 
 > Alternatives that we’ve played with before choosing VictoriaMetrics are: federated Prometheus, Cortex, IronDB and Thanos.
 > Points that were critical to us when we were choosing a central tsdb, in order of importance:

docs/Cluster-VictoriaMetrics.md

@@ -177,10 +177,11 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
        This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag.
        See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#sending-opentsdb-data-via-http-apiput-requests) for details.
      - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
+     - `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below).
      - `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data) for details.
      - `prometheus/api/v1/import/prometheus` - for importing data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
 
-* URLs for querying: `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
+* URLs for [Prmetheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
   - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
   - `<suffix>` may have the following values:
     - `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries).
@@ -189,16 +190,26 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
     - `api/v1/labels` - returns a [list of label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names).
     - `api/v1/label/<label_name>/values` - returns values for the given `<label_name>` according [to API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values).
     - `federate` - returns [federated metrics](https://prometheus.io/docs/prometheus/latest/federation/).
-    - `api/v1/export` - exports raw data. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details.
+    - `api/v1/export` - exports raw data in JSON line format. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details.
+    - `api/v1/export/native` - exports raw data in native binary format. It may be imported into another VictoriaMetrics via `api/v1/import/native` (see above).
+    - `api/v1/export/csv` - exports data in CSV. It may be imported into another VictoriaMetrics via `api/v1/import/csv` (see above).
     - `api/v1/status/tsdb` - for time series stats. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
     - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries,
       which is returned in the response.
 
+* URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://<vmselect>:8481/select/<accountID>/graphite/<suffix>`, where:
+    - `<accountID>` is an arbitrary number identifying data namespace for query (aka tenant)
+    - `<suffix>` may have the following values:
+      - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find).
+      - `metrics/expand` - expands Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand).
+      - `metrics/index.json` - returns all the metric names. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json).
+
 * URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
   Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
   be used on a regular basis, since it carries non-zero overhead.
 
 * `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
+  - `/internal/force_merge` - initiate [forced compactions](https://victoriametrics.github.io/#forced-merge) on the given `vmstorage` node.
   - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
     which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
     command-line flag value.
@@ -269,6 +280,12 @@ Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the mos
 * The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
 * The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
   Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
+  The required RAM per each `vmstorage` should be multiplied by `-replicationFactor` if [replication](#replication-and-data-safety) is enabled.
+  Additional RAM can be required for query processing.
+  Calculated RAM requrements may differ from actual RAM requirements due to various factors:
+  * The average number of labels per time series. More labels require more RAM.
+  * The average length of label names and label values. Longer labels require more RAM.
+  * The type of queries. Heavy queries that scan big number of time series over long time ranges require more RAM.
 * The recommended total amount of storage space for all the `vmstorage` instances can be calculated
   from the ingestion rate and retention: `storage_space = ingestion_rate * retention_seconds`.
 

docs/FAQ.md

@@ -91,7 +91,7 @@ The main differences between Cortex and VictoriaMetrics:
   VictoriaMetrics may lose only a few seconds of recent data, which isn't synced to persistent storage yet.
   See [this article for details](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
 - Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) and [other case studies](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies).
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV.
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 
@@ -112,7 +112,7 @@ The main differences between Cortex and VictoriaMetrics:
 - Thanos may be harder to setup and operate comparing to VictoriaMetrics, since it has more moving parts, which can be connected with less reliable networks.
   See [this article for details](https://medium.com/faun/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
 - Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV.
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 
@@ -120,7 +120,7 @@ The main differences between Cortex and VictoriaMetrics:
 
 - VictoriaMetrics requires [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and it [works faster](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 - VictoriaMetrics provides [better query language](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085) than InfluxQL or Flux.
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to InfluxDB - Prometheus remote_write, OpenTSDB, Graphite, CSV.
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to InfluxDB - Prometheus remote_write, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 

docs/MetricsQL.md

@@ -15,6 +15,8 @@ The following functionality is implemented differently in MetricsQL comparing to
 * MetricsQL removes all the `NaN` values from the output, so some queries like `(-1)^0.5` return empty results in VictoriaMetrics, while returning
   a series of `NaN` values in Prometheus. Note that Grafana doesn't draw any lines or dots for `NaN` values, so usually the end result looks the same for both
   VictoriaMetrics and Prometheus.
+* MetricsQL keeps metric names after applying functions, which don't change the meaining of the original time series. For example, `min_over_time(foo)` or `round(foo)`
+  leave `foo` metric name in the result. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674) for details.
 
 Other PromQL functionality should work the same in MetricsQL. [File an issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues)
 if you notice discrepancies between PromQL and MetricsQL results other than mentioned above.
@@ -115,6 +117,8 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
   Example: `share_le_over_time(memory_usage_bytes[24h], 100*1024*1024)` returns the share of time series values for the last 24 hours when memory usage was below or equal to 100MB.
 - `share_gt_over_time(m[d], gt)` - returns share (in the range 0..1) of values in `m` over `d`, which are bigger than `gt`. Useful for calculating SLI and SLO.
   Example: `share_gt_over_time(up[24h], 0)` - returns service availability for the last 24 hours.
+- `count_le_over_time(m[d], le)` - returns the number of raw samples for `m` over `d`, which don't exceed `le`.
+- `count_gt_over_time(m[d], gt)` - returns the number of raw samples for `m` over `d`, which are bigger than `gt`.
 - `tmin_over_time(m[d])` - returns timestamp for the minimum value for `m` over `d` time range.
 - `tmax_over_time(m[d])` - returns timestamp for the maximum value for `m` over `d` time range.
 - `aggr_over_time(("aggr_func1", "aggr_func2", ...), m[d])` - simultaneously calculates all the listed `aggr_func*` for `m` over `d` time range.
@@ -124,7 +128,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
   for the given `phi` in the range `[0..1]`.
 - `last_over_time(m[d])` - returns the last value for `m` on the time range `d`.
 - `first_over_time(m[d])` - returns the first value for `m` on the time range `d`.
-- `outliersk(N, q) by (group)` - returns up to `N` outlier time series for `q` in every `group`. Outlier time series have the highest deviation from the `median(m)`.
+- `outliersk(N, q) by (group)` - returns up to `N` outlier time series for `q` in every `group`. Outlier time series have the highest deviation from the `median(q)`.
   This aggregate function is useful to detect anomalies across groups of similar time series.
 - `ascent_over_time(m[d])` - returns the sum of positive deltas between adjancent data points in `m` over `d`. Useful for tracking height gains in GPS track.
 - `descent_over_time(m[d])` - returns the absolute sum of negative deltas between adjancent data points in `m` over `d`. Useful for tracking height loss in GPS track.

docs/Release-Guide.md

@@ -2,6 +2,7 @@ Release process guidance
 
 ## Release version and Docker images
 
+0. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/CHANGELOG.md).
 1. Create release tag with `git tag v1.xx.y`.
 2. Run `make release` for creating `*.tar.gz` release archive with the corresponding `_checksums.txt` inside `bin` directory.
 3. Run `make publish` for creating and publishing Docker images.

docs/Single-server-VictoriaMetrics.md

@@ -77,7 +77,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
     if `-graphiteListenAddr` is set.
   * [OpenTSDB put message](#sending-data-via-telnet-put-protocol) if `-opentsdbListenAddr` is set.
   * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests) if `-opentsdbHTTPListenAddr` is set.
-  * [/api/v1/import](#how-to-import-time-series-data).
+  * [JSON line format](#how-to-import-data-in-json-line-format).
+  * [Native binary format](#how-to-import-data-in-native-format).
+  * [Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
   * [Arbitrary CSV data](#how-to-import-csv-data).
 * Supports metrics' relabeling. See [these docs](#relabeling) for details.
 * Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various Enterprise workloads.
@@ -99,9 +101,9 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
 * [How to send data from Graphite-compatible agents such as StatsD](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
 * [Querying Graphite data](#querying-graphite-data)
 * [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
-* [How to import data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format)
-* [How to import CSV data](#how-to-import-csv-data)
 * [Prometheus querying API usage](#prometheus-querying-api-usage)
+  * [Prometheus querying API enhancements](#prometheus-querying-api-enhancements)
+* [Graphite Metrics API usage](#graphite-metrics-api-usage)
 * [How to build from sources](#how-to-build-from-sources)
   * [Development build](#development-build)
   * [Production build](#production-build)
@@ -112,8 +114,16 @@ See [features available for enterprise customers](https://github.com/VictoriaMet
 * [Setting up service](#setting-up-service)
 * [How to work with snapshots](#how-to-work-with-snapshots)
 * [How to delete time series](#how-to-delete-time-series)
+* [Forced merge](#forced-merge)
 * [How to export time series](#how-to-export-time-series)
+  * [How to export data in native format](#how-to-export-data-in-native-format)
+  * [How to export data in JSON line format](#how-to-export-data-in-json-line-format)
+  * [How to export CSV data](#how-to-export-csv-data)
 * [How to import time series data](#how-to-import-time-series-data)
+  * [How to import data in native format](#how-to-import-data-in-native-format)
+  * [How to import data in json line format](#how-to-import-data-in-json-line-format)
+  * [How to import CSV data](#how-to-import-csv-data)
+  * [How to import data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format)
 * [Relabeling](#relabeling)
 * [Federation](#federation)
 * [Capacity planning](#capacity-planning)
@@ -285,6 +295,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
 * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
 * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
+* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config)
+* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
 
 In the future other `*_sd_config` types will be supported.
 
@@ -341,7 +353,7 @@ curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'ht
 ```
 
 An arbitrary number of lines delimited by '\n' (aka newline char) may be sent in a single request.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"measurement_.*"}'
@@ -377,7 +389,7 @@ echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N localhost 2003
 
 VictoriaMetrics sets the current time if the timestamp is omitted.
 An arbitrary number of lines delimited by `\n` (aka newline char) may be sent in one go.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
@@ -391,9 +403,11 @@ The `/api/v1/export` endpoint should return the following response:
 
 ### Querying Graphite data
 
-Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read either via
-[Prometheus querying API](#prometheus-querying-api-usage)
-or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml).
+Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
+
+* [Prometheus querying API](#prometheus-querying-api-usage)
+* Metric names can be explored via [Graphite metrics API](#graphite-metrics-api-usage)
+* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/master/cmd/carbonapi/carbonapi.example.prometheus.yaml)
 
 ### How to send data from OpenTSDB-compatible agents
 
@@ -419,7 +433,7 @@ echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N localhost
 ```
 
 An arbitrary number of lines delimited by `\n` (aka newline char) may be sent in one go.
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
@@ -454,7 +468,7 @@ Example for writing multiple data points in a single request:
 curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
 ```
 
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
 ```bash
 curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar'
@@ -469,82 +483,6 @@ The `/api/v1/export` endpoint should return the following response:
 ```
 
 
-### How to import CSV data
-
-Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
-The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
-
-```
-<column_pos>:<type>:<context>
-```
-
-* `<column_pos>` is the position of the CSV column (field). Column numbering starts from 1. The order of parsing rules may be arbitrary.
-* `<type>` describes the column type. Supported types are:
-  * `metric` - the corresponding CSV column at `<column_pos>` contains metric value, which must be integer or floating-point number.
-    The metric name is read from the `<context>`. CSV line must have at least a single metric field. Multiple metric fields per CSV line is OK.
-  * `label` - the corresponding CSV column at `<column_pos>` contains label value. The label name is read from the `<context>`.
-    CSV line may have arbitrary number of label fields. All these labels are attached to all the configured metrics.
-  * `time` - the corresponding CSV column at `<column_pos>` contains metric time. CSV line may contain either one or zero columns with time.
-    If CSV line has no time, then the current time is used. The time is applied to all the configured metrics.
-    The format of the time is configured via `<context>`. Supported time formats are:
-    * `unix_s` - unix timestamp in seconds.
-    * `unix_ms` - unix timestamp in milliseconds.
-    * `unix_ns` - unix timestamp in nanoseconds. Note that VictoriaMetrics rounds the timestamp to milliseconds.
-    * `rfc3339` - timestamp in [RFC3339](https://tools.ietf.org/html/rfc3339) format, i.e. `2006-01-02T15:04:05Z`.
-    * `custom:<layout>` - custom layout for the timestamp. The `<layout>` may contain arbitrary time layout according to [time.Parse rules in Go](https://golang.org/pkg/time/#Parse).
-
-Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
-
-Example for importing CSV data via `/api/v1/import/csv`:
-
-```bash
-curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
-curl -d "MSFT,3.21,1.67,NASDAQ" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
-```
-
-After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint:
-
-```bash
-curl -G 'http://localhost:8428/api/v1/export' -d 'match[]={ticker!=""}'
-```
-
-The following response should be returned:
-```bash
-{"metric":{"__name__":"bid","market":"NASDAQ","ticker":"MSFT"},"values":[1.67],"timestamps":[1583865146520]}
-{"metric":{"__name__":"bid","market":"NYSE","ticker":"GOOG"},"values":[4.56],"timestamps":[1583865146495]}
-{"metric":{"__name__":"ask","market":"NASDAQ","ticker":"MSFT"},"values":[3.21],"timestamps":[1583865146520]}
-{"metric":{"__name__":"ask","market":"NYSE","ticker":"GOOG"},"values":[1.23],"timestamps":[1583865146495]}
-```
-
-Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
-
-
-### How to import data in Prometheus exposition format
-
-VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
-via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
-
-```bash
-curl -d 'foo{bar="baz"} 123' -X POST 'http://localhost:8428/api/v1/import/prometheus'
-```
-
-The following command may be used for verifying the imported data:
-
-```bash
-curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"foo"}'
-```
-
-It should return somethins like the following:
-
-```
-{"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
-```
-
-VictoriaMetrics accepts arbitrary number of lines in a single request to `/api/v1/import/prometheus`, i.e. it supports data streaming.
-
-VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
-
-
 ### Prometheus querying API usage
 
 VictoriaMetrics supports the following handlers from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/):
@@ -558,6 +496,13 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
 
 These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
 
+#### Prometheus querying API enhancements
+
+Additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) VictoriaMetrics accepts relative times in `time`, `start` and `end` query args.
+For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
+
+By default, VictoriaMetrics returns time series for the last 5 minutes from /api/v1/series, while the Prometheus API defaults to all time.  Use `start` and `end` to select a different time range.
+
 VictoriaMetrics accepts additional args for `/api/v1/labels` and `/api/v1/label/.../values` handlers.
 See [this feature request](https://github.com/prometheus/prometheus/issues/6178) for details:
 
@@ -566,11 +511,27 @@ See [this feature request](https://github.com/prometheus/prometheus/issues/6178)
 
 Additionally VictoriaMetrics provides the following handlers:
 
-* `/api/v1/series/count` - it returns the total number of time series in the database. Note that this handler scans all the inverted index,
-  so it can be slow if the database contains tens of millions of time series.
+* `/api/v1/series/count` - it returns the total number of time series in the database. Some notes:
+  * the handler scans all the inverted index, so it can be slow if the database contains tens of millions of time series;
+  * the handler may count [deleted time series](#how-to-delete-time-series) additionally to normal time series due to internal implementation restrictions;
 * `/api/v1/labels/count` - it returns a list of `label: values_count` entries. It can be used for determining labels with the maximum number of values.
 * `/api/v1/status/active_queries` - it returns a list of currently running queries.
 
+
+### Graphite Metrics API usage
+
+VictoriaMetrics supports the following handlers from [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api):
+
+* [/metrics/find](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find)
+* [/metrics/expand](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand)
+* [/metrics/index.json](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json)
+
+VictoriaMetrics accepts the following additional query args at `/metrics/find` and `/metrics/expand`:
+  * `label` - for selecting arbitrary label values. By default `label=__name__`, i.e. metric names are selected.
+  * `delimiter` - for using different delimiters in metric name hierachy. For example, `/metrics/find?delimiter=_&query=node_*` would return all the metric name prefixes
+    that start with `node_`. By default `delimiter=.`.
+
+
 ### How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -622,8 +583,8 @@ Run `make package-victoria-metrics`. It builds `victoriametrics/victoria-metrics
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-victoria-metrics`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image for improved debuggability.
-It is possible to build the package on top of any other base image by setting it via `<ROOT_IMAGE>` environment variable.
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable.
 For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
@@ -675,9 +636,12 @@ Send a request to `http://<victoriametrics-addr>:8428/api/v1/admin/tsdb/delete_s
 where `<timeseries_selector_for_delete>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to delete. After that all the time series matching the given selector are deleted. Storage space for
 the deleted time series isn't freed instantly - it is freed during subsequent [background merges of data files](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+Note that background merges may never occur for data from previous months, so storage space won't be freed for historical data.
+In this case [forced merge](#forced-merge) may help freeing up storage space.
 
 It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
-before actually deleting the metrics.
+before actually deleting the metrics.  By default this query will only scan active series in the past 5 minutes, so you may need to
+adjust `start` and `end` to a suitable range to achieve match hits.
 
 The `/api/v1/admin/tsdb/delete_series` handler may be protected with `authKey` if `-deleteAuthKey` command-line flag is set.
 
@@ -693,16 +657,58 @@ It isn't recommended using delete API for the following cases, since it brings n
   See [this article](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts) for details.
 * Reducing disk space usage by deleting unneeded time series. This doesn't work as expected, since the deleted
   time series occupy disk space until the next merge operation, which can never occur when deleting too old data.
+  [Forced merge](#forced-merge) may be used for freeing up disk space occupied by old data.
 
 It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
 
+
+### Forced merge
+
+VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
+This means that compactions are stopped for per-month partitions if no new data is ingested into these partitions.
+Sometimes it is necessary to trigger compactions for old partitions. For instance, in order to free up disk space occupied by [deleted time series](#how-to-delete-time-series).
+In this case forced compaction may be initiated on the specified per-month partition by sending request to `/internal/force_merge?partition_prefix=YYYY_MM`,
+where `YYYY_MM` is per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate forced
+merge for August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in background.
+
+Forced merges may require additional CPU, disk IO and storage space resources. It is unnecessary to run forced merge under normal conditions,
+since VictoriaMetrics automatically performs [optimal merges in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+when new data is ingested into it.
+
+
 ### How to export time series
 
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
+VictoriaMetrics provides the following handlers for exporting data:
+
+* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
+  See [these docs](#how-to-export-data-in-native-format) for details.
+* `/api/v1/export` for exporing data in JSON line format. See [these docs](#how-to-export-data-in-json-line-format) for details.
+* `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
+
+
+#### How to export data in native format
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
+where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
+
+
+#### How to export data in JSON line format
+
+Consider [exporting data in native format](#how-to-export-data-in-native-format) if big amounts of data must be migrated between VictoriaMetrics instances,
+since exporting in native format usually consumes lower amounts of CPU and memory resources, while the resulting exported data occupies lower amounts of disk space.
+
+In order to export data in JSON line format, send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
 where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
 The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
-Each JSON line would contain data for a single time series. An example output:
+Each JSON line contains samples for a single time series. An example output:
 
 ```jsonl
 {"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
@@ -712,8 +718,9 @@ Each JSON line would contain data for a single time series. An example output:
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
 
-Optional `max_rows_per_line` arg may be added to the request in order to limit the maximum number of rows exported per each JSON line.
-By default each JSON line contains all the rows for a single time series.
+Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
+Optional `reduce_mem_usage=1` arg may be added to the request for reducing memory usage when exporting big number of time series.
+In this case the output may contain multiple lines with distinct samples for the same time series.
 
 Pass `Accept-Encoding: gzip` HTTP header in the request to `/api/v1/export` in order to reduce network bandwidth during exporing big amounts
 of time series data. This enables gzip compression for the exported data. Example for exporting gzipped data:
@@ -724,22 +731,82 @@ curl -H 'Accept-Encoding: gzip' http://localhost:8428/api/v1/export -d 'match[]=
 
 The maximum duration for each request to `/api/v1/export` is limited by `-search.maxExportDuration` command-line flag.
 
-Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-time-series-data).
+Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
+
+
+#### How to export CSV data
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/csv?format=<format>&match=<timeseries_selector_for_export>`,
+where:
+
+* `<format>` must contain comma-delimited label names for the exported CSV. The following special label names are supported:
+  * `__name__` - metric name
+  * `__value__` - sample value
+  * `__timestamp__:<ts_format>` - sample timestamp. `<ts_format>` can have the following values:
+    * `unix_s` - unix seconds
+    * `unix_ms` - unix milliseconds
+    * `unix_ns` - unix nanoseconds
+    * `rfc3339` - [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) time
+    * `custom:<layout>` - custom layout for time that is supported by [time.Format](https://golang.org/pkg/time/#Time.Format) function from Go.
+
+* `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export.
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
+
 
 ### How to import time series data
 
 Time series data can be imported via any supported ingestion protocol:
 
-* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-* [Influx line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
-* [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
-* [OpenTSDB telnet put protocol](#sending-data-via-telnet-put-protocol)
-* [OpenTSDB http /api/put](#sending-opentsdb-data-via-http-apiput-requests)
-* `/api/v1/import` http POST handler, which accepts data from [/api/v1/export](#how-to-export-time-series).
-* `/api/v1/import/csv` http POST handler, which accepts CSV data. See [these docs](#how-to-import-csv-data) for details.
-* `/api/v1/import/prometheus` http POST handler, which accepts data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
+* [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
+* Influx line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
+* Graphite plaintext protocol. See [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
+* OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
+* OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
+* `/api/v1/import` for importing data obtained from [/api/v1/export](#how-to-export-data-in-json-line-format).
+  See [these docs](##how-to-import-data-in-json-line-format) for details.
+* `/api/v1/import/native` for importing data obtained from [/api/v1/export/native](#how-to-export-data-in-native-format).
+  See [these docs](#how-to-import-data-in-native-format) for details.
+* `/api/v1/import/csv` for importing arbitrary CSV data. See [these docs](#how-to-import-csv-data) for details.
+* `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
 
-The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import`. Example for importing data obtained via `/api/v1/export`:
+
+#### How to import data in native format
+
+The most efficient protocol for importing data into VictoriaMetrics is `/api/v1/import/native`.
+Example for importing data obtained via [/api/v1/export/native](#how-to-export-data-in-native-format):
+
+```bash
+# Export the data from <source-victoriametrics>:
+curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
+```
+
+Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/native` for importing gzipped data:
+
+```bash
+# Export gzipped data from <source-victoriametrics>:
+curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin.gz
+
+# Import gzipped data to <destination-victoriametrics>:
+curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin.gz
+```
+
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
+#### How to import data in JSON line format
+
+Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
 
 ```bash
 # Export the data from <source-victoriametrics>:
@@ -759,16 +826,113 @@ curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export
 curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import -T exported_data.jsonl.gz
 ```
 
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
 Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
 
-Each request to `/api/v1/import` can load up to a single vCPU core on VictoriaMetrics. Import speed can be improved by splitting the original file into smaller parts
-and importing them concurrently. Note that the original file must be split on newlines.
+
+#### How to import CSV data
+
+Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
+The `format` query arg must contain comma-separated list of parsing rules for CSV fields. Each rule consists of three parts delimited by a colon:
+
+```
+<column_pos>:<type>:<context>
+```
+
+* `<column_pos>` is the position of the CSV column (field). Column numbering starts from 1. The order of parsing rules may be arbitrary.
+* `<type>` describes the column type. Supported types are:
+  * `metric` - the corresponding CSV column at `<column_pos>` contains metric value, which must be integer or floating-point number.
+    The metric name is read from the `<context>`. CSV line must have at least a single metric field. Multiple metric fields per CSV line is OK.
+  * `label` - the corresponding CSV column at `<column_pos>` contains label value. The label name is read from the `<context>`.
+    CSV line may have arbitrary number of label fields. All these labels are attached to all the configured metrics.
+  * `time` - the corresponding CSV column at `<column_pos>` contains metric time. CSV line may contain either one or zero columns with time.
+    If CSV line has no time, then the current time is used. The time is applied to all the configured metrics.
+    The format of the time is configured via `<context>`. Supported time formats are:
+    * `unix_s` - unix timestamp in seconds.
+    * `unix_ms` - unix timestamp in milliseconds.
+    * `unix_ns` - unix timestamp in nanoseconds. Note that VictoriaMetrics rounds the timestamp to milliseconds.
+    * `rfc3339` - timestamp in [RFC3339](https://tools.ietf.org/html/rfc3339) format, i.e. `2006-01-02T15:04:05Z`.
+    * `custom:<layout>` - custom layout for the timestamp. The `<layout>` may contain arbitrary time layout according to [time.Parse rules in Go](https://golang.org/pkg/time/#Parse).
+
+Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
+
+Example for importing CSV data via `/api/v1/import/csv`:
+
+```bash
+curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+curl -d "MSFT,3.21,1.67,NASDAQ" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
+```
+
+After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
+
+```bash
+curl -G 'http://localhost:8428/api/v1/export' -d 'match[]={ticker!=""}'
+```
+
+The following response should be returned:
+```bash
+{"metric":{"__name__":"bid","market":"NASDAQ","ticker":"MSFT"},"values":[1.67],"timestamps":[1583865146520]}
+{"metric":{"__name__":"bid","market":"NYSE","ticker":"GOOG"},"values":[4.56],"timestamps":[1583865146495]}
+{"metric":{"__name__":"ask","market":"NASDAQ","ticker":"MSFT"},"values":[3.21],"timestamps":[1583865146520]}
+{"metric":{"__name__":"ask","market":"NYSE","ticker":"GOOG"},"values":[1.23],"timestamps":[1583865146495]}
+```
+
+Extra labels may be added to all the imported lines by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/csv?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported lines.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
+#### How to import data in Prometheus exposition format
+
+VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
+via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
+
+```bash
+curl -d 'foo{bar="baz"} 123' -X POST 'http://localhost:8428/api/v1/import/prometheus'
+```
+
+The following command may be used for verifying the imported data:
+
+```bash
+curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"foo"}'
+```
+
+It should return something like the following:
+
+```
+{"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
+```
+
+Extra labels may be added to all the imported metrics by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/prometheus?extra_label=foo=bar` would add `{foo="bar"}` label to all the imported metrics.
+
+If timestamp is missing in `<metric> <value> <timestamp>` Prometheus exposition format line, then the current timestamp is used during data ingestion.
+It can be overriden by passing unix timestamp in *milliseconds* via `timestamp` query arg. For example, `/api/v1/import/prometheus?timestamp=1594370496905`.
+
+VictoriaMetrics accepts arbitrary number of lines in a single request to `/api/v1/import/prometheus`, i.e. it supports data streaming.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+
 
 
 ### Relabeling
 
 VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
 to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
+Example contents for `-relabelConfig` file:
+```yml
+# relabel_config.yml
+- target_label: cluster
+  replacement: dev
+- action: drop
+  source_labels: [__meta_kubernetes_pod_container_init]
+  regex: true
+```
 
 VictoriaMetrics provides the following extra actions for relabeling rules:
 
@@ -876,7 +1040,8 @@ with the enabled de-duplication. See [this section](#deduplication) for details.
 
 VictoriaMetrics de-duplicates data points if `-dedup.minScrapeInterval` command-line flag
 is set to positive duration. For example, `-dedup.minScrapeInterval=60s` would de-duplicate data points
-on the same time series if they are located closer than 60s to each other.
+on the same time series if they fall within the same discrete 60s bucket.  The earliest data point will be kept.  In the case of equal timestamps, an arbitrary data point will be kept.
+
 The de-duplication reduces disk space usage if multiple identically configured Prometheus instances in HA pair
 write data to the same VictoriaMetrics instance. Note that these Prometheus instances must have identical
 `external_labels` section in their configs, so they write data to the same time series.
@@ -956,6 +1121,7 @@ Consider setting the following command-line flags:
   with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
 * `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
 * `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
+* `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
 * `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
 
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
@@ -1037,6 +1203,9 @@ VictoriaMetrics also exposes currently running queries with their execution time
   has at least 20% of free space comparing to disk size. The remaining amount of free space
   can be [monitored](#monitoring) via `vm_free_disk_space_bytes` metric. The total size of data
   stored on the disk can be monitored via sum of `vm_data_size_bytes` metrics.
+  See also `vm_merge_need_free_disk_space` metrics, which are set to values higher than 0
+  if background merge cannot be initiated due to free disk space shortage. The value shows the number of per-month partitions,
+  which would start background merge if they had more free disk space.
 
 * If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
   then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
@@ -1063,6 +1232,8 @@ VictoriaMetrics also exposes currently running queries with their execution time
   This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
   metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
 
+* VictoriaMetrics ignores `NaN` values during data ingestion.
+
 
 ### Backfilling
 
@@ -1082,7 +1253,7 @@ for data with timestamps close to the current time.
 
 ### Data updates
 
-VictoriaMetrics doesn't support updating already exiting sample values to new ones. It stores all the ingested data points
+VictoriaMetrics doesn't support updating already existing sample values to new ones. It stores all the ingested data points
 for the same time series with identical timestamps. While is possible substituting old time series with new time series via
 [removal of old time series](#how-to-delete-timeseries) and then [writing new time series](#backfilling), this approach
 should be used only for one-off updates. It shouldn't be used for frequent updates because of non-zero overhead related to data removal.

docs/vmagent.md

@@ -25,7 +25,8 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-native-format).
   * Data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
@@ -41,7 +42,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
 
@@ -135,7 +136,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `ec2_sd_configs` - for scraping targets in Amazon EC2.
   See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
-  `vmagent` doesn't support `role_arn` config param yet.
+  `vmagent` doesn't support `profile` config param and aws credentials file yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
   `vmagent` provides the following additional functionality for `gce_sd_config`:
@@ -147,6 +148,11 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
 * `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
   See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+* `openstack_sd_configs` - for scraping OpenStack targets.
+  See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
+  [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
+  See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
 
 File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -206,6 +212,8 @@ Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) f
 If you have suggestions, improvements or found a bug - feel free to open an issue on github or add review to the dashboard.
 
 `vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`/targets` handler accepts optional `show_original_labels=1` query arg, which shows the original labels per each target
+before applying relabeling. This information may be useful for debugging target relabeling.
 
 
 ### Troubleshooting
@@ -218,8 +226,7 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
 
-* It is recommended to increase `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
-  and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+* It is recommended to increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
 
 * If you see gaps on the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, then try increasing `-remoteWrite.queues`.
   Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage, so it starts dropping the buffered data
@@ -229,8 +236,13 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* By default `vmagent` masks `-remoteWrite.url` with `secret-url` values in logs and at `/metrics` page because
+  the url may contain sensitive information such as auth tokens or passwords.
+  Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
-  or they use init container.
+  or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
+  See available options below if you prefer fixing the root cause of the error:
 
   The following `relabel_configs` section may help determining `__meta_*` labels resulting in duplicate targets:
   ```yml
@@ -275,7 +287,7 @@ Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

docs/vmalert.md

@@ -166,6 +166,10 @@ The shortlist of configuration flags is the following:
     	Optional basic auth password for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.lookback duration
+        Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+  -datasource.maxIdleConnections int
+        Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
   -datasource.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
   -datasource.tlsCertFile string
@@ -194,8 +198,12 @@ The shortlist of configuration flags is the following:
     	Supports array of values separated by comma or specified via multiple flags.
   -external.url string
     	External URL is used as alert's source for sent alerts to the notifier
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
   -http.disableResponseCompression
     	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
     	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
   -http.pathPrefix string
@@ -216,8 +224,9 @@ The shortlist of configuration flags is the following:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -memory.allowedBytes int
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
@@ -293,8 +302,8 @@ The shortlist of configuration flags is the following:
     	Path to the file with alert rules. 
     	Supports patterns. Flag can be specified multiple times. 
     	Examples:
-    	 -rule /path/to/file. Path to a single file with alerting rules
-    	 -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+    	 -rule="/path/to/file". Path to a single file with alerting rules
+    	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder, 
     	absolute path to all .yaml files in root.
     	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
     	Supports array of values separated by comma or specified via multiple flags.

docs/vmauth.md

@@ -113,7 +113,7 @@ Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker i
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash
@@ -140,3 +140,68 @@ curl -s http://<vmauth-host>:8427/debug/pprof/profile > cpu.pprof
 The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
+
+
+### Advanced usage
+
+Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
+
+```
+./vmauth -help
+
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md .
+
+  -auth.config string
+    	Path to auth config. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/README.md for details on the format of this auth config
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help spreading incoming load among a cluster of services behind load balancer. Note that the real timeout may be bigger by up to 10% as a protection from Thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses for saving CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for graceful shutdown of HTTP server. Highly loaded server may require increased value for graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this dealy the servier returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address to listen for http connections (default ":8427")
+  -loggerErrorsPerSecondLimit int
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, then the remaining errors are suppressed. Zero value disables the rate limit (default 10)
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
+  -memory.allowedBytes value
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It overrides httpAuth settings
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It overrides httpAuth settings
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs, since RSA certs are slow
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
+  -version
+    	Show VictoriaMetrics version
+```

docs/vmbackup.md

@@ -6,12 +6,12 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. This saves time and costs on data transfer.
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
 
@@ -35,8 +35,8 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
 
@@ -49,13 +49,13 @@ with the following command:
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
 ```
 
-This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
 
 #### Incremental backups
 
-Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
-This saves time and network bandwidth costs when working with big backups:
+Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+It saves time and network bandwidth costs when working with big backups:
 
 ```
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
@@ -100,16 +100,16 @@ The backup algorithm is the following:
 2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
 3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
 4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
-   This are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+   These are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
+- Old files periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -129,7 +129,45 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ### Advanced usage
 
-Run `vmbackup -help` in order to see all the available options:
+
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+  You have to add a custom url endpoint via flag:
+```
+  # for minio
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+* Run `vmbackup -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -138,7 +176,7 @@ Run `vmbackup -help` in order to see all the available options:
     	Path to file with S3 configs. Configs are loaded from default location if not set.
     	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs (default "default")
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
     	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
@@ -161,18 +199,20 @@ Run `vmbackup -help` in order to see all the available options:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -maxBytesPerSecond int
+  -maxBytesPerSecond value
     	The maximum upload speed. There is no limit if it is set to 0
-  -memory.allowedBytes int
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -origin string
     	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
   -snapshot.createURL string
-    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snaphsot/create
   -snapshot.deleteURL string
-    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snaphosts will be automatically deleted. Example: http://victoriametrics:8428/snaphsot/delete
   -snapshotName string
     	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
   -storageDataPath string
@@ -205,7 +245,7 @@ Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` dock
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

docs/vmrestore.md

@@ -21,7 +21,7 @@ vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/r
 * `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
   to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
 
-The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup,
+The original `-storageDataPath` directory may contain old files. They will be substituted by the files from backup,
 i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/questions/476041/how-do-i-make-rsync-delete-files-that-have-been-deleted-from-the-source-folder).
 
 
@@ -33,7 +33,44 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 ### Advanced usage
 
-Run `vmrestore -help` in order to see all the available options:
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint.  It is possible to use `vmrestore` with s3 api compatible storages, like  minio, cloudian and other.
+  You have to add custom url endpoint with a flag:
+```
+  # for minio:
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+*  Run `vmrestore -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -42,7 +79,7 @@ Run `vmrestore -help` in order to see all the available options:
     	Path to file with S3 configs. Configs are loaded from default location if not set.
     	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -configProfile string
-    	Profile name for S3 configs (default "default")
+    	Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used
   -credsFilePath string
     	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
@@ -62,10 +99,12 @@ Run `vmrestore -help` in order to see all the available options:
     	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
     	Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -maxBytesPerSecond int
+  -maxBytesPerSecond value
     	The maximum download speed. There is no limit if it is set to 0
-  -memory.allowedBytes int
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -memory.allowedBytes value
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to non-zero value. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage
+    	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -skipBackupCompleteCheck
@@ -102,7 +141,7 @@ Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` do
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

go.mod

@@ -1,31 +1,33 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
 require (
-	cloud.google.com/go v0.63.0 // indirect
-	cloud.google.com/go/storage v1.10.0
+	cloud.google.com/go v0.68.0 // indirect
+	cloud.google.com/go/storage v1.12.0
 	github.com/VictoriaMetrics/fastcache v1.5.7
 
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
-	github.com/VictoriaMetrics/fasthttp v1.0.5
+	github.com/VictoriaMetrics/fasthttp v1.0.7
 	github.com/VictoriaMetrics/metrics v1.12.3
-	github.com/VictoriaMetrics/metricsql v0.4.1
-	github.com/aws/aws-sdk-go v1.34.4
+	github.com/VictoriaMetrics/metricsql v0.7.1
+	github.com/aws/aws-sdk-go v1.35.5
 	github.com/cespare/xxhash/v2 v2.1.1
-	github.com/golang/snappy v0.0.1
-	github.com/klauspost/compress v1.10.11
-	github.com/valyala/fastjson v1.5.4
+	github.com/golang/snappy v0.0.2
+	github.com/klauspost/compress v1.11.1
+	github.com/stretchr/testify v1.5.1 // indirect
+	github.com/valyala/fastjson v1.6.1
 	github.com/valyala/fastrand v1.0.0
 	github.com/valyala/fasttemplate v1.2.1
 	github.com/valyala/gozstd v1.8.3
 	github.com/valyala/histogram v1.1.2
-	github.com/valyala/quicktemplate v1.6.2
-	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d
-	golang.org/x/tools v0.0.0-20200813203630-136574234359 // indirect
-	google.golang.org/api v0.30.0
-	google.golang.org/genproto v0.0.0-20200813001606-1ccf2a5ae4fd // indirect
+	github.com/valyala/quicktemplate v1.6.3
+	go.opencensus.io v0.22.5 // indirect
+	golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 // indirect
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	golang.org/x/sys v0.0.0-20201008064518-c1f3e3309c71
+	golang.org/x/tools v0.0.0-20201008025239-9df69603baec // indirect
+	google.golang.org/api v0.32.0
+	google.golang.org/genproto v0.0.0-20201007142714-5c0e72c5e71e // indirect
 	gopkg.in/yaml.v2 v2.3.0
 )
 

go.sum

@@ -15,8 +15,12 @@ cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0 h1:RmDygqvj27Zf3fCQjQRtLyC7KwFcHkeJitcO0OoGOcA=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.63.0 h1:A+DfAZQ/eWca7gvu42CS6FNSDX4R8cghF+XfWLn4R6g=
-cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
+cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.66.0 h1:DZeAkuQGQqnm9Xv36SbMJEU8aFBz4wL04UpMWPWwjzg=
+cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko=
+cloud.google.com/go v0.68.0 h1:AnVkaPGAuWaIY/8a75HlNzZNrHDee6YL4rWkwS+CeyE=
+cloud.google.com/go v0.68.0/go.mod h1:91NO4SCDjUfe1zeC0f4/dpckkUNpuNEyqm4X2KLrzNQ=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
@@ -43,25 +47,27 @@ cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4=
+cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
 github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
-github.com/VictoriaMetrics/fasthttp v1.0.5 h1:DvNn5bStvgb7cZbWUkvd7psaPY9ul+7b2gA6uoiCmHc=
-github.com/VictoriaMetrics/fasthttp v1.0.5/go.mod h1:88NVQ+fmidWtXEFT153t7lAEZNu6Q3DgK1g0l1ZyQtw=
+github.com/VictoriaMetrics/fasthttp v1.0.7 h1:9RntF8jE9z+ooyXy0tb3Pl76jan4DLaCQoCQFp/cIUE=
+github.com/VictoriaMetrics/fasthttp v1.0.7/go.mod h1:eaGv8oDDOWE8JQa6GgBgpCwjJhRxavmprtf2oWw8b3o=
 github.com/VictoriaMetrics/metrics v1.12.2 h1:SG8iAmqavDNuh7GIdHPoGHUhDL23KeKfvSZSozucNeA=
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
 github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.4.1 h1:WbVIfRNCK7HjrzayrpAl07mkh4kiDFZuECsh57rly2Q=
-github.com/VictoriaMetrics/metricsql v0.4.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.7.1 h1:2V7EbbfKkU2pDzs+D/S0IKYvNSQzDYBlbS8afXD7ntE=
+github.com/VictoriaMetrics/metricsql v0.7.1/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/aws/aws-sdk-go v1.34.4 h1:Yx49/+ZMCD9YqIVsO3CsiMs4hnUnokd9otKvWYFjnYw=
-github.com/aws/aws-sdk-go v1.34.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.35.5 h1:doSEOxC0UkirPcle20Rc+1kAhJ4Ip+GSEeZ3nKl7Qlk=
+github.com/aws/aws-sdk-go v1.35.5/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -80,7 +86,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -109,6 +114,8 @@ github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -119,6 +126,8 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
@@ -130,6 +139,7 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
@@ -137,17 +147,19 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
-github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
-github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg=
+github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.1 h1:bPb7nMRdOZYDrpPMTA3EInUQrdgoBinqUuSwlGdKDdE=
+github.com/klauspost/compress v1.11.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -165,9 +177,9 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fastjson v1.5.4 h1:r8gpiVwdzDU09NrlN38OyL5dUFpdwGQR5RQEBqY+hLg=
-github.com/valyala/fastjson v1.5.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
+github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
+github.com/valyala/fastjson v1.6.1 h1:qJs/Kz/HebWzk8LmhOrSm7kdOyJBr1XB+zSkYtEEfQE=
+github.com/valyala/fastjson v1.6.1/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
@@ -176,18 +188,21 @@ github.com/valyala/gozstd v1.8.3 h1:nHlS+sCFoNLsZpRPKDviXkhHybaRSUjH2w0P/myYo0I=
 github.com/valyala/gozstd v1.8.3/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8=
 github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg=
-github.com/valyala/quicktemplate v1.6.2 h1:k0vgK7zlmFzqAoIBIOrhrfmZ6JoTGJlLRPLbkPGr2/M=
-github.com/valyala/quicktemplate v1.6.2/go.mod h1:mtEJpQtUiBV0SHhMX6RtiJtqxncgrfmjcUy5T68X8TM=
+github.com/valyala/quicktemplate v1.6.3 h1:O7EuMwuH7Q94U2CXD6sOX8AYHqQqWtmIk690IhmpkKA=
+github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -252,14 +267,22 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
+golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -294,8 +317,11 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d h1:QQrM/CCYEzTs91GZylDCQjGHudbPTxF/1fvXdVh5lMo=
-golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201008064518-c1f3e3309c71 h1:ZPX6UakxrJCxWiyGWpXtFY+fp86Esy7xJT/jJCG8bgU=
+golang.org/x/sys v0.0.0-20201008064518-c1f3e3309c71/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -344,9 +370,14 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200813203630-136574234359 h1:eZBCv5amWGFL95UzDqwTieckVDxgaiUg+KkNnNc62hQ=
-golang.org/x/tools v0.0.0-20200813203630-136574234359/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201008025239-9df69603baec h1:RY2OghEV/7X1MLaecgm1mwFd3sGvUddm5pGVSxQvX0c=
+golang.org/x/tools v0.0.0-20201008025239-9df69603baec/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -369,6 +400,10 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.31.0 h1:1w5Sz/puhxFo9lTtip2n47k7toB/U2nCqOKNHd3Yrbo=
+google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo=
+google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts=
+google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -404,9 +439,16 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200813001606-1ccf2a5ae4fd h1:pCOIJgz7MD1XjLsF1K0X2xI97dR8sEXS34ZcYl7fcNE=
-google.golang.org/genproto v0.0.0-20200813001606-1ccf2a5ae4fd/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201002142447-3860012362da h1:DTQYk4u7nICKkkVZsBv0/0po0ChISxAJ5CTAfUhO0PQ=
+google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201007142714-5c0e72c5e71e h1:zYWvTjcAbEy4pVj21IfgFga4rAW107tKspqOi853hjI=
+google.golang.org/genproto v0.0.0-20201007142714-5c0e72c5e71e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -420,6 +462,10 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
+google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -436,6 +482,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

lib/backup/actions/backup.go

@@ -71,45 +71,47 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 
 	logger.Infof("starting backup from %s to %s using origin %s", src, dst, origin)
 
-	logger.Infof("obtaining list of parts at %s", src)
 	srcParts, err := src.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list src parts: %w", err)
 	}
-	logger.Infof("obtaining list of parts at %s", dst)
+	logger.Infof("obtained %d parts from src %s", len(srcParts), src)
+
 	dstParts, err := dst.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list dst parts: %w", err)
 	}
-	logger.Infof("obtaining list of parts at %s", origin)
+	logger.Infof("obtained %d parts from dst %s", len(dstParts), dst)
+
 	originParts, err := origin.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list origin parts: %w", err)
 	}
+	logger.Infof("obtained %d parts from origin %s", len(originParts), origin)
 
 	backupSize := getPartsSize(srcParts)
 
 	partsToDelete := common.PartsDifference(dstParts, srcParts)
 	deleteSize := getPartsSize(partsToDelete)
 	if len(partsToDelete) > 0 {
-		logger.Infof("deleting %d parts from %s", len(partsToDelete), dst)
+		logger.Infof("deleting %d parts from dst %s", len(partsToDelete), dst)
 		deletedParts := uint64(0)
 		err = runParallel(concurrency, partsToDelete, func(p common.Part) error {
-			logger.Infof("deleting %s from %s", &p, dst)
+			logger.Infof("deleting %s from dst %s", &p, dst)
 			if err := dst.DeletePart(p); err != nil {
-				return fmt.Errorf("cannot delete %s from %s: %w", &p, dst, err)
+				return fmt.Errorf("cannot delete %s from dst %s: %w", &p, dst, err)
 			}
 			atomic.AddUint64(&deletedParts, 1)
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&deletedParts)
-			logger.Infof("deleted %d out of %d parts from %s in %s", n, len(partsToDelete), dst, elapsed)
+			logger.Infof("deleted %d out of %d parts from dst %s in %s", n, len(partsToDelete), dst, elapsed)
 		})
 		if err != nil {
 			return err
 		}
 		if err := dst.RemoveEmptyDirs(); err != nil {
-			return fmt.Errorf("cannot remove empty directories at %s: %w", dst, err)
+			return fmt.Errorf("cannot remove empty directories at dst %s: %w", dst, err)
 		}
 	}
 
@@ -117,18 +119,18 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 	originCopyParts := common.PartsIntersect(originParts, partsToCopy)
 	copySize := getPartsSize(originCopyParts)
 	if len(originCopyParts) > 0 {
-		logger.Infof("server-side copying %d parts from %s to %s", len(originCopyParts), origin, dst)
+		logger.Infof("server-side copying %d parts from origin %s to dst %s", len(originCopyParts), origin, dst)
 		copiedParts := uint64(0)
 		err = runParallel(concurrency, originCopyParts, func(p common.Part) error {
-			logger.Infof("server-side copying %s from %s to %s", &p, origin, dst)
+			logger.Infof("server-side copying %s from origin %s to dst %s", &p, origin, dst)
 			if err := dst.CopyPart(origin, p); err != nil {
-				return fmt.Errorf("cannot copy %s from %s to %s: %w", &p, origin, dst, err)
+				return fmt.Errorf("cannot copy %s from origin %s to dst %s: %w", &p, origin, dst, err)
 			}
 			atomic.AddUint64(&copiedParts, 1)
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&copiedParts)
-			logger.Infof("server-side copied %d out of %d parts from %s to %s in %s", n, len(originCopyParts), origin, dst, elapsed)
+			logger.Infof("server-side copied %d out of %d parts from origin %s to dst %s in %s", n, len(originCopyParts), origin, dst, elapsed)
 		})
 		if err != nil {
 			return err
@@ -138,36 +140,36 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 	srcCopyParts := common.PartsDifference(partsToCopy, originParts)
 	uploadSize := getPartsSize(srcCopyParts)
 	if len(srcCopyParts) > 0 {
-		logger.Infof("uploading %d parts from %s to %s", len(srcCopyParts), src, dst)
+		logger.Infof("uploading %d parts from src %s to dst %s", len(srcCopyParts), src, dst)
 		bytesUploaded := uint64(0)
 		err = runParallel(concurrency, srcCopyParts, func(p common.Part) error {
-			logger.Infof("uploading %s from %s to %s", &p, src, dst)
+			logger.Infof("uploading %s from src %s to dst %s", &p, src, dst)
 			rc, err := src.NewReadCloser(p)
 			if err != nil {
-				return fmt.Errorf("cannot create reader for %s from %s: %w", &p, src, err)
+				return fmt.Errorf("cannot create reader for %s from src %s: %w", &p, src, err)
 			}
 			sr := &statReader{
 				r:         rc,
 				bytesRead: &bytesUploaded,
 			}
 			if err := dst.UploadPart(p, sr); err != nil {
-				return fmt.Errorf("cannot upload %s to %s: %w", &p, dst, err)
+				return fmt.Errorf("cannot upload %s to dst %s: %w", &p, dst, err)
 			}
 			if err = rc.Close(); err != nil {
-				return fmt.Errorf("cannot close reader for %s from %s: %w", &p, src, err)
+				return fmt.Errorf("cannot close reader for %s from src %s: %w", &p, src, err)
 			}
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&bytesUploaded)
-			logger.Infof("uploaded %d out of %d bytes from %s to %s in %s", n, uploadSize, src, dst, elapsed)
+			logger.Infof("uploaded %d out of %d bytes from src %s to dst %s in %s", n, uploadSize, src, dst, elapsed)
 		})
 		if err != nil {
 			return err
 		}
 	}
 
-	logger.Infof("backed up %d bytes in %.3f seconds; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
-		backupSize, time.Since(startTime).Seconds(), deleteSize, copySize, uploadSize)
+	logger.Infof("backup from src %s to dst %s with origin %s is complete; backed up %d bytes in %.3f seconds; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
+		src, dst, origin, backupSize, time.Since(startTime).Seconds(), deleteSize, copySize, uploadSize)
 
 	return nil
 }

lib/backup/actions/util.go

@@ -18,7 +18,8 @@ var (
 		"See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
 	configFilePath = flag.String("configFilePath", "", "Path to file with S3 configs. Configs are loaded from default location if not set.\n"+
 		"See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html")
-	configProfile    = flag.String("configProfile", "default", "Profile name for S3 configs")
+	configProfile = flag.String("configProfile", "", "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), "+
+		"or if both not set, DefaultSharedConfigProfile is used")
 	customS3Endpoint = flag.String("customS3Endpoint", "", "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set")
 )
 

lib/backup/common/fs.go

@@ -9,6 +9,9 @@ import (
 // This filesystem is used for performing server-side file copies
 // instead of uploading data from local filesystem.
 type OriginFS interface {
+	// MustStop must be called when the RemoteFS is no longer needed.
+	MustStop()
+
 	// String must return human-readable representation of OriginFS.
 	String() string
 
@@ -18,6 +21,9 @@ type OriginFS interface {
 
 // RemoteFS is a filesystem where backups are stored.
 type RemoteFS interface {
+	// MustStop must be called when the RemoteFS is no longer needed.
+	MustStop()
+
 	// String must return human-readable representation of RemoteFS.
 	String() string
 

lib/backup/fslocal/bandwidth_limiter.go

@@ -15,6 +15,9 @@ type bandwidthLimiter struct {
 
 	// quota for the current second
 	quota int
+
+	stopCh chan struct{}
+	wg     sync.WaitGroup
 }
 
 func newBandwidthLimiter(perSecondLimit int) *bandwidthLimiter {
@@ -25,10 +28,20 @@ func newBandwidthLimiter(perSecondLimit int) *bandwidthLimiter {
 	bl.perSecondLimit = perSecondLimit
 	var mu sync.Mutex
 	bl.c = sync.NewCond(&mu)
-	go bl.perSecondUpdater()
+	bl.stopCh = make(chan struct{})
+	bl.wg.Add(1)
+	go func() {
+		defer bl.wg.Done()
+		bl.perSecondUpdater()
+	}()
 	return &bl
 }
 
+func (bl *bandwidthLimiter) MustStop() {
+	close(bl.stopCh)
+	bl.wg.Wait()
+}
+
 func (bl *bandwidthLimiter) NewReadCloser(rc io.ReadCloser) *bandwidthLimitedReader {
 	return &bandwidthLimitedReader{
 		rc: rc,
@@ -83,7 +96,12 @@ func (blw *bandwidthLimitedWriter) Close() error {
 func (bl *bandwidthLimiter) perSecondUpdater() {
 	tc := time.NewTicker(time.Second)
 	c := bl.c
-	for range tc.C {
+	for {
+		select {
+		case <-tc.C:
+		case <-bl.stopCh:
+			return
+		}
 		c.L.Lock()
 		bl.quota = bl.perSecondLimit
 		c.Signal()

lib/backup/fslocal/fslocal.go

@@ -27,7 +27,9 @@ type FS struct {
 	bl *bandwidthLimiter
 }
 
-// Init initializes fs
+// Init initializes fs.
+//
+// The returned fs must be stopped when no long needed with MustStop call.
 func (fs *FS) Init() error {
 	if fs.MaxBytesPerSecond > 0 {
 		fs.bl = newBandwidthLimiter(fs.MaxBytesPerSecond)
@@ -35,6 +37,15 @@ func (fs *FS) Init() error {
 	return nil
 }
 
+// MustStop stops fs.
+func (fs *FS) MustStop() {
+	if fs.bl == nil {
+		return
+	}
+	fs.bl.MustStop()
+	fs.bl = nil
+}
+
 // String returns user-readable representation for the fs.
 func (fs *FS) String() string {
 	return fmt.Sprintf("fslocal %q", fs.Dir)

lib/backup/fsnil/fsnil.go

@@ -7,6 +7,11 @@ import (
 // FS represents nil remote filesystem.
 type FS struct{}
 
+// MustStop stops fs.
+func (fs *FS) MustStop() {
+	// Nothing to do
+}
+
 // String returns human-readable string representation for fs.
 func (fs *FS) String() string {
 	return "fsnil"

lib/backup/fsremote/fsremote.go

@@ -22,6 +22,11 @@ type FS struct {
 	Dir string
 }
 
+// MustStop stops fs.
+func (fs *FS) MustStop() {
+	// Nothing to do
+}
+
 // String returns human-readable string representation for fs.
 func (fs *FS) String() string {
 	return fmt.Sprintf("fsremote %q", fs.Dir)

lib/backup/gcsremote/gcs.go

@@ -33,6 +33,8 @@ type FS struct {
 }
 
 // Init initializes fs.
+//
+// The returned fs must be stopped when no long needed with MustStop call.
 func (fs *FS) Init() error {
 	if fs.bkt != nil {
 		logger.Panicf("BUG: fs.Init has been already called")
@@ -63,6 +65,11 @@ func (fs *FS) Init() error {
 	return nil
 }
 
+// MustStop stops fs.
+func (fs *FS) MustStop() {
+	fs.bkt = nil
+}
+
 // String returns human-readable description for fs.
 func (fs *FS) String() string {
 	return fmt.Sprintf("GCS{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)

lib/backup/s3remote/s3.go

@@ -45,6 +45,8 @@ type FS struct {
 }
 
 // Init initializes fs.
+//
+// The returned fs must be stopped when no long needed with MustStop call.
 func (fs *FS) Init() error {
 	if fs.s3 != nil {
 		logger.Panicf("BUG: Init is already called")
@@ -96,6 +98,12 @@ func (fs *FS) Init() error {
 	return nil
 }
 
+// MustStop stops fs.
+func (fs *FS) MustStop() {
+	fs.s3 = nil
+	fs.uploader = nil
+}
+
 // String returns human-readable description for fs.
 func (fs *FS) String() string {
 	return fmt.Sprintf("S3{bucket: %q, dir: %q}", fs.Bucket, fs.Dir)

lib/cgroup/cpu.go

@@ -1,8 +1,11 @@
 package cgroup
 
 import (
+	"io/ioutil"
 	"os"
 	"runtime"
+	"strconv"
+	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
@@ -12,6 +15,8 @@ import (
 // This function must be called after logger.Init().
 func UpdateGOMAXPROCSToCPUQuota() {
 	if v := os.Getenv("GOMAXPROCS"); v != "" {
+		// Do not override explicitly set GOMAXPROCS.
+		logger.Infof("using GOMAXPROCS=%q set via environment variable", v)
 		return
 	}
 	q := getCPUQuota()
@@ -20,6 +25,12 @@ func UpdateGOMAXPROCSToCPUQuota() {
 		return
 	}
 	gomaxprocs := int(q + 0.5)
+	numCPU := runtime.NumCPU()
+	if gomaxprocs > numCPU {
+		// There is no sense in setting more GOMAXPROCS than the number of available CPU cores.
+		logger.Infof("cgroup CPU quota=%d exceeds NumCPU=%d; using GOMAXPROCS=NumCPU", gomaxprocs, numCPU)
+		return
+	}
 	if gomaxprocs <= 0 {
 		gomaxprocs = 1
 	}
@@ -32,9 +43,55 @@ func getCPUQuota() float64 {
 	if err != nil {
 		return 0
 	}
+	if quotaUS <= 0 {
+		// The quota isn't set. This may be the case in multilevel containers.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728
+		return getOnlineCPUCount()
+	}
 	periodUS, err := readInt64("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "cat /sys/fs/cgroup/cpu$(cat /proc/self/cgroup | grep cpu, | cut -d: -f3)/cpu.cfs_period_us")
 	if err != nil {
 		return 0
 	}
 	return float64(quotaUS) / float64(periodUS)
 }
+
+func getOnlineCPUCount() float64 {
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728
+	data, err := ioutil.ReadFile("/sys/devices/system/cpu/online")
+	if err != nil {
+		return -1
+	}
+	n := float64(countCPUs(string(data)))
+	if n <= 0 {
+		return -1
+	}
+	return n
+}
+
+func countCPUs(data string) int {
+	data = strings.TrimSpace(data)
+	n := 0
+	for _, s := range strings.Split(data, ",") {
+		n++
+		if !strings.Contains(s, "-") {
+			if _, err := strconv.Atoi(s); err != nil {
+				return -1
+			}
+			continue
+		}
+		bounds := strings.Split(s, "-")
+		if len(bounds) != 2 {
+			return -1
+		}
+		start, err := strconv.Atoi(bounds[0])
+		if err != nil {
+			return -1
+		}
+		end, err := strconv.Atoi(bounds[1])
+		if err != nil {
+			return -1
+		}
+		n += end - start
+	}
+	return n
+}

lib/cgroup/cpu_test.go

@@ -0,0 +1,24 @@
+package cgroup
+
+import (
+	"testing"
+)
+
+func TestCountCPUs(t *testing.T) {
+	f := func(s string, nExpected int) {
+		t.Helper()
+		n := countCPUs(s)
+		if n != nExpected {
+			t.Fatalf("unexpected result from countCPUs(%q); got %d; want %d", s, n, nExpected)
+		}
+	}
+	f("", -1)
+	f("1", 1)
+	f("234", 1)
+	f("1,2", 2)
+	f("0-1", 2)
+	f("0-0", 1)
+	f("1-2,3,5-9,200-210", 19)
+	f("0-3", 4)
+	f("0-6", 7)
+}

lib/cgroup/mem.go

@@ -14,3 +14,18 @@ func GetMemoryLimit() int64 {
 	}
 	return n
 }
+
+// GetHierarchicalMemoryLimit returns hierarchical memory limit
+func GetHierarchicalMemoryLimit() int64 {
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/699
+	n, err := readInt64FromCommand("cat /sys/fs/cgroup/memory/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2")
+	if err == nil {
+		return n
+	}
+	n, err = readInt64FromCommand(
+		"cat /sys/fs/cgroup/memory$(cat /proc/self/cgroup | grep memory | cut -d: -f3)/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2")
+	if err != nil {
+		return 0
+	}
+	return n
+}

lib/cgroup/util.go

@@ -9,15 +9,18 @@ import (
 
 func readInt64(path, altCommand string) (int64, error) {
 	data, err := ioutil.ReadFile(path)
+	if err == nil {
+		data = bytes.TrimSpace(data)
+		return strconv.ParseInt(string(data), 10, 64)
+	}
+	return readInt64FromCommand(altCommand)
+}
+
+func readInt64FromCommand(command string) (int64, error) {
+	cmd := exec.Command("/bin/sh", "-c", command)
+	data, err := cmd.Output()
 	if err != nil {
-		// Read data according to https://unix.stackexchange.com/questions/242718/how-to-find-out-how-much-memory-lxc-container-is-allowed-to-consume
-		// This should properly determine the data location inside lxc container.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/84
-		cmd := exec.Command("/bin/sh", "-c", altCommand)
-		data, err = cmd.Output()
-		if err != nil {
-			return 0, err
-		}
+		return 0, err
 	}
 	data = bytes.TrimSpace(data)
 	return strconv.ParseInt(string(data), 10, 64)

lib/decimal/decimal.go

@@ -36,6 +36,9 @@ func CalibrateScale(a []int64, ae int16, b []int64, be int16) (e int16) {
 	}
 	upExp -= downExp
 	for i, v := range a {
+		if v == vInfPos || v == vInfNeg {
+			continue
+		}
 		adjExp := upExp
 		for adjExp > 0 {
 			v *= 10
@@ -45,6 +48,9 @@ func CalibrateScale(a []int64, ae int16, b []int64, be int16) (e int16) {
 	}
 	if downExp > 0 {
 		for i, v := range b {
+			if v == vInfPos || v == vInfNeg {
+				continue
+			}
 			adjExp := downExp
 			for adjExp > 0 {
 				v /= 10
@@ -76,11 +82,20 @@ func ExtendInt64sCapacity(dst []int64, additionalItems int) []int64 {
 	return dst[:dstLen]
 }
 
+func extendInt16sCapacity(dst []int16, additionalItems int) []int16 {
+	dstLen := len(dst)
+	if n := dstLen + additionalItems - cap(dst); n > 0 {
+		dst = append(dst[:cap(dst)], make([]int16, n)...)
+	}
+	return dst[:dstLen]
+}
+
 // AppendDecimalToFloat converts each item in va to f=v*10^e, appends it
 // to dst and returns the resulting dst.
 func AppendDecimalToFloat(dst []float64, va []int64, e int16) []float64 {
 	// Extend dst capacity in order to eliminate memory allocations below.
 	dst = ExtendFloat64sCapacity(dst, len(va))
+	a := dst[len(dst) : len(dst)+len(va)]
 
 	if fastnum.IsInt64Zeros(va) {
 		return fastnum.AppendFloat64Zeros(dst, len(va))
@@ -89,35 +104,53 @@ func AppendDecimalToFloat(dst []float64, va []int64, e int16) []float64 {
 		if fastnum.IsInt64Ones(va) {
 			return fastnum.AppendFloat64Ones(dst, len(va))
 		}
-		for _, v := range va {
+		_ = a[len(va)-1]
+		for i, v := range va {
 			f := float64(v)
-			dst = append(dst, f)
+			if v == vInfPos {
+				f = infPos
+			} else if v == vInfNeg {
+				f = infNeg
+			}
+			a[i] = f
 		}
-		return dst
+		return dst[:len(dst)+len(va)]
 	}
 
 	// increase conversion precision for negative exponents by dividing by e10
 	if e < 0 {
 		e10 := math.Pow10(int(-e))
-		for _, v := range va {
+		_ = a[len(va)-1]
+		for i, v := range va {
 			f := float64(v) / e10
-			dst = append(dst, f)
+			if v == vInfPos {
+				f = infPos
+			} else if v == vInfNeg {
+				f = infNeg
+			}
+			a[i] = f
 		}
-		return dst
+		return dst[:len(dst)+len(va)]
 	}
 	e10 := math.Pow10(int(e))
-	for _, v := range va {
+	_ = a[len(va)-1]
+	for i, v := range va {
 		f := float64(v) * e10
-		dst = append(dst, f)
+		if v == vInfPos {
+			f = infPos
+		} else if v == vInfNeg {
+			f = infNeg
+		}
+		a[i] = f
 	}
-	return dst
+	return dst[:len(dst)+len(va)]
 }
 
 // AppendFloatToDecimal converts each item in src to v*10^e and appends
 // each v to dst returning it as va.
 //
 // It tries minimizing each item in dst.
-func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
+func AppendFloatToDecimal(dst []int64, src []float64) ([]int64, int16) {
 	if len(src) == 0 {
 		return dst, 0
 	}
@@ -130,9 +163,6 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 		return dst, 0
 	}
 
-	// Extend dst capacity in order to eliminate memory allocations below.
-	dst = ExtendInt64sCapacity(dst, len(src))
-
 	vaev := vaeBufPool.Get()
 	if vaev == nil {
 		vaev = &vaeBuf{
@@ -141,19 +171,20 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 		}
 	}
 	vae := vaev.(*vaeBuf)
-	vae.va = vae.va[:0]
-	vae.ea = vae.ea[:0]
+	va := vae.va[:0]
+	ea := vae.ea[:0]
+	va = ExtendInt64sCapacity(va, len(src))
+	va = va[:len(src)]
+	ea = extendInt16sCapacity(ea, len(src))
+	ea = ea[:len(src)]
 
 	// Determine the minimum exponent across all src items.
-	v, exp := FromFloat(src[0])
-	vae.va = append(vae.va, v)
-	vae.ea = append(vae.ea, exp)
-	minExp := exp
-	for _, f := range src[1:] {
+	minExp := int16(1<<15 - 1)
+	for i, f := range src {
 		v, exp := FromFloat(f)
-		vae.va = append(vae.va, v)
-		vae.ea = append(vae.ea, exp)
-		if exp < minExp {
+		va[i] = v
+		ea[i] = exp
+		if exp < minExp && v != vInfPos && v != vInfNeg {
 			minExp = exp
 		}
 	}
@@ -161,8 +192,9 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 	// Determine whether all the src items may be upscaled to minExp.
 	// If not, adjust minExp accordingly.
 	downExp := int16(0)
-	for i, v := range vae.va {
-		exp := vae.ea[i]
+	_ = ea[len(va)-1]
+	for i, v := range va {
+		exp := ea[i]
 		upExp := exp - minExp
 		maxUpExp := maxUpExponent(v)
 		if upExp-maxUpExp > downExp {
@@ -171,9 +203,19 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 	}
 	minExp += downExp
 
+	// Extend dst capacity in order to eliminate memory allocations below.
+	dst = ExtendInt64sCapacity(dst, len(src))
+	a := dst[len(dst) : len(dst)+len(src)]
+
 	// Scale each item in src to minExp and append it to dst.
-	for i, v := range vae.va {
-		exp := vae.ea[i]
+	_ = a[len(va)-1]
+	_ = ea[len(va)-1]
+	for i, v := range va {
+		if v == vInfPos || v == vInfNeg {
+			a[i] = v
+			continue
+		}
+		exp := ea[i]
 		adjExp := exp - minExp
 		for adjExp > 0 {
 			v *= 10
@@ -183,12 +225,14 @@ func AppendFloatToDecimal(dst []int64, src []float64) (va []int64, e int16) {
 			v /= 10
 			adjExp++
 		}
-		dst = append(dst, v)
+		a[i] = v
 	}
 
+	vae.va = va
+	vae.ea = ea
 	vaeBufPool.Put(vae)
 
-	return dst, minExp
+	return dst[:len(dst)+len(va)], minExp
 }
 
 type vaeBuf struct {
@@ -198,8 +242,10 @@ type vaeBuf struct {
 
 var vaeBufPool sync.Pool
 
+const int64Max = int64(1<<63 - 1)
+
 func maxUpExponent(v int64) int16 {
-	if v == 0 {
+	if v == 0 || v == vInfPos || v == vInfNeg {
 		// Any exponent allowed.
 		return 1024
 	}
@@ -210,53 +256,49 @@ func maxUpExponent(v int64) int16 {
 		// Handle corner case for v=-1<<63
 		return 0
 	}
-
-	maxMultiplier := ((1 << 63) - 1) / uint64(v)
 	switch {
-	case maxMultiplier >= 1e19:
-		return 19
-	case maxMultiplier >= 1e18:
+	case v <= int64Max/1e18:
 		return 18
-	case maxMultiplier >= 1e17:
+	case v <= int64Max/1e17:
 		return 17
-	case maxMultiplier >= 1e16:
+	case v <= int64Max/1e16:
 		return 16
-	case maxMultiplier >= 1e15:
+	case v <= int64Max/1e15:
 		return 15
-	case maxMultiplier >= 1e14:
+	case v <= int64Max/1e14:
 		return 14
-	case maxMultiplier >= 1e13:
+	case v <= int64Max/1e13:
 		return 13
-	case maxMultiplier >= 1e12:
+	case v <= int64Max/1e12:
 		return 12
-	case maxMultiplier >= 1e11:
+	case v <= int64Max/1e11:
 		return 11
-	case maxMultiplier >= 1e10:
+	case v <= int64Max/1e10:
 		return 10
-	case maxMultiplier >= 1e9:
+	case v <= int64Max/1e9:
 		return 9
-	case maxMultiplier >= 1e8:
+	case v <= int64Max/1e8:
 		return 8
-	case maxMultiplier >= 1e7:
+	case v <= int64Max/1e7:
 		return 7
-	case maxMultiplier >= 1e6:
+	case v <= int64Max/1e6:
 		return 6
-	case maxMultiplier >= 1e5:
+	case v <= int64Max/1e5:
 		return 5
-	case maxMultiplier >= 1e4:
+	case v <= int64Max/1e4:
 		return 4
-	case maxMultiplier >= 1e3:
+	case v <= int64Max/1e3:
 		return 3
-	case maxMultiplier >= 1e2:
+	case v <= int64Max/1e2:
 		return 2
-	case maxMultiplier >= 1e1:
+	case v <= int64Max/1e1:
 		return 1
 	default:
 		return 0
 	}
 }
 
-// Round f to value with the given number of significant decimal digits.
+// Round f to value with the given number of significant figures.
 func Round(f float64, digits int) float64 {
 	if digits <= 0 || digits >= 18 {
 		return f
@@ -290,6 +332,12 @@ func Round(f float64, digits int) float64 {
 
 // ToFloat returns f=v*10^e.
 func ToFloat(v int64, e int16) float64 {
+	if v == vInfPos {
+		return infPos
+	}
+	if v == vInfNeg {
+		return infNeg
+	}
 	f := float64(v)
 	// increase conversion precision for negative exponents by dividing by e10
 	if e < 0 {
@@ -298,6 +346,11 @@ func ToFloat(v int64, e int16) float64 {
 	return f * math.Pow10(int(e))
 }
 
+var (
+	infPos = math.Inf(1)
+	infNeg = math.Inf(-1)
+)
+
 const (
 	vInfPos = 1<<63 - 1
 	vInfNeg = -1 << 63