docs/CHANGELOG.md: cut v1.89.1

The issue has been introduced in 0af9e2b693

.github/ISSUE_TEMPLATE/question.yml

@@ -0,0 +1,32 @@
+name: Question
+description: Ask a question regarding VictoriaMetrics or its components
+labels: [question]
+body:
+  - type: textarea
+    id: describe-the-component
+    attributes:
+      label: Is your question request related to a specific component?
+      placeholder: |
+        VictoriaMetrics, vmagent, vmalert, vmui, etc...
+    validations:
+      required: false
+  - type: textarea
+    id: describe-the-question
+    attributes:
+      label: Describe the question in detail
+      description: |
+        A clear and concise description of the issue and the question.
+    validations:
+      required: true
+  - type: checkboxes
+    id: troubleshooting
+    attributes:
+      label: Troubleshooting docs
+      description: I am familiar with the following troubleshooting docs
+      options:
+        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
+          required: false
+        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
+          required: false
+        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
+          required: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
           check-latest: true
           cache: true
 
@@ -56,7 +56,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
           check-latest: true
           cache: true
 
@@ -81,7 +81,7 @@ jobs:
         id: go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
           check-latest: true
           cache: true
 

.github/workflows/nightly-build.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.0
+          go-version: 1.20.2
         id: go
 
       - name: Setup docker scan

.golangci.yml

@@ -1,14 +1,18 @@
 run:
   timeout: 2m
 
-enable:
+linters:
+  enable:
   - revive
 
 issues:
   exclude-rules:
-    - linters:
-      - staticcheck
-      text: "SA(4003|1019|5011):"
+  - linters:
+    - staticcheck
+    text: "SA(4003|1019|5011):"
+  include:
+  - EXC0012
+  - EXC0014
 
 linters-settings:
   errcheck:

Makefile

@@ -380,7 +380,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -2,6 +2,7 @@
 
 [![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics.svg?maxAge=604800)](https://hub.docker.com/r/victoriametrics/victoria-metrics)
+[![victoriametrics](https://snapcraft.io/victoriametrics/badge.svg)](https://snapcraft.io/victoriametrics)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
@@ -105,6 +106,7 @@ Case studies:
 * [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Dig Security](https://docs.victoriametrics.com/CaseStudies.html#dig-security)
 * [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
 * [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
@@ -125,11 +127,22 @@ See also [articles and slides about VictoriaMetrics from our users](https://docs
 
 ## Operation
 
-### How to start VictoriaMetrics
+### Install
 
-Just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
+To quickly try VictoriaMetrics, just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
 See also [QuickStart guide](https://docs.victoriametrics.com/Quick-Start.html) for additional information.
 
+VictoriaMetrics can also be installed via these installation methods:
+
+* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
+* [Kubernetes operator for VictoriaMetrics](https://github.com/VictoriaMetrics/operator).
+* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
+* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
+* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
+* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
+
+### How to start VictoriaMetrics
+
 The following command-line flags are used the most:
 
 * `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
@@ -997,7 +1010,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
@@ -1046,7 +1059,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
@@ -1074,7 +1087,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
@@ -1313,7 +1326,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 By default, the last point on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden with `max_lookback` query arg.
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
@@ -1356,6 +1369,7 @@ By default VictoriaMetrics is tuned for an optimal resource usage under typical
 - `-search.maxSamplesPerQuery` limits the number of raw samples a single query can process. This allows limiting CPU usage for heavy queries.
 - `-search.maxPointsPerTimeseries` limits the number of calculated points, which can be returned per each matching time series from [range query](https://docs.victoriametrics.com/keyConcepts.html#range-query).
 - `-search.maxPointsSubqueryPerTimeseries` limits the number of calculated points, which can be generated per each matching time series during [subquery](https://docs.victoriametrics.com/MetricsQL.html#subqueries) evaluation.
+- `-search.maxSeriesPerAggrFunc` limits the number of time series, which can be generated by [MetricsQL aggregate functions](https://docs.victoriametrics.com/MetricsQL.html#aggregate-functions) in a single query.
 - `-search.maxSeries` limits the number of time series, which may be returned from [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers). This endpoint is used mostly by Grafana for auto-completion of metric names, label names and label values. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxSeries` to quite low value in order limit CPU and memory usage.
 - `-search.maxTagKeys` limits the number of items, which may be returned from [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names). This endpoint is used mostly by Grafana for auto-completion of label names. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagKeys` to quite low value in order to limit CPU and memory usage.
 - `-search.maxTagValues` limits the number of items, which may be returned from [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values). This endpoint is used mostly by Grafana for auto-completion of label values. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagValues` to quite low value in order to limit CPU and memory usage.
@@ -2037,17 +2051,10 @@ It is safe sharing the collected profiles from security point of view, since the
 
 ## Integrations
 
-* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
-* [Kubernetes operator for VictoriaMetrics](https://github.com/VictoriaMetrics/operator).
-* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
-  See [these docs](https://github.com/netdata/netdata#integrations).
 * [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
   See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
-* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
-* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
-* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
-
-* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
+* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
+  See [these docs](https://github.com/netdata/netdata#integrations).
 * [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/vmalert.html).
 
 ## Third-party contributions
@@ -2226,8 +2233,12 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s)
   -insert.maxQueueDuration duration
      The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringCacheExpireDuration duration
+     The expire duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -logNewSeries
      Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics
   -loggerDisableTimestamps
@@ -2263,11 +2274,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+     TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
   -opentsdbHTTPListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+     TCP and UDP address to listen for OpenTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
   -opentsdbListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
@@ -2337,6 +2348,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
@@ -2359,7 +2372,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.seriesLimitPerTarget int
      Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
-     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
@@ -2398,8 +2411,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
   -search.latencyOffset duration
      The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
+  -search.logQueryMemoryUsage size
+     Log queries, which require more memory than specified by this flag. This may help detecting and optimizing heavy queries. Query logging is disabled by default. See also -search.logSlowQueryDuration and -search.maxMemoryPerQuery
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -search.logSlowQueryDuration duration
-     Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
+     Log queries with execution time exceeding this value. Zero disables slow query logging. See also -search.logQueryMemoryUsage (default 5s)
   -search.maxConcurrentRequests int
      The maximum number of concurrent search requests. It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. See also -search.maxQueueDuration and -search.maxMemoryPerQuery (default 8)
   -search.maxExportDuration duration
@@ -2413,7 +2429,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
   -search.maxMemoryPerQuery size
-     The maximum amounts of memory a single query may consume. Queries requiring more memory are rejected. The total memory limit for concurrently executed queries can be estimated as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests
+     The maximum amounts of memory a single query may consume. Queries requiring more memory are rejected. The total memory limit for concurrently executed queries can be estimated as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests . See also -search.logQueryMemoryUsage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -search.maxPointsPerTimeseries int
      The maximum points per a single timeseries returned from /api/v1/query_range. This option doesn't limit the number of scanned raw samples in the database. The main purpose of this option is to limit the number of per-series points returned to graphing UI such as VMUI or Grafana. There is no sense in setting this limit to values bigger than the horizontal resolution of the graph (default 30000)
@@ -2432,6 +2448,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000)
   -search.maxSeries int
      The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 30000)
+  -search.maxSeriesPerAggrFunc int
+     The maximum number of time series an aggregate MetricsQL function can generate (default 1000000)
   -search.maxStalenessInterval duration
      The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag
   -search.maxStatusRequestDuration duration
@@ -2472,6 +2490,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum number of CPU cores to use for small merges. Default value is used if set to 0
   -snapshotAuthKey string
      authKey, which must be passed in query string to /snapshot* pages
+  -snapshotCreateTimeout duration
+     The timeout for creating new snapshot. If set, make sure that timeout is lower than backup period
   -snapshotsMaxAge value
      Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
      The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 0)

app/victoria-metrics/Makefile

@@ -82,6 +82,9 @@ victoria-metrics-linux-arm64:
 victoria-metrics-linux-ppc64le:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+victoria-metrics-linux-s390x:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 victoria-metrics-linux-386:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-metrics/main.go

@@ -26,7 +26,8 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
@@ -53,7 +54,7 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
 		return
 	}
 
@@ -92,7 +93,7 @@ func main() {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")

app/vmagent/Makefile

@@ -85,6 +85,9 @@ vmagent-linux-arm64:
 vmagent-linux-ppc64le:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmagent-linux-s390x:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/README.md

@@ -25,7 +25,9 @@ additionally to [discovering Prometheus-compatible targets and scraping metrics
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Can accept data via all the ingestion protocols supported by VictoriaMetrics - see [these docs](#how-to-push-data-to-vmagent).
 * Can aggregate incoming samples by time and by labels before sending them to remote storage - see [these docs](https://docs.victoriametrics.com/stream-aggregation.html).
-* Can replicate collected metrics simultaneously to multiple remote storage systems - see [these docs](#replication-and-high-availability).
+* Can replicate collected metrics simultaneously to multiple Prometheus-compatible remote storage systems - see [these docs](#replication-and-high-availability).
+* Can save egress network bandwidth usage costs when [VictoriaMetrics remote write protocol](#victoriametrics-remote-write-protocol)
+  is used for sending the data to VictoriaMetrics.
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as the connection
   to the remote storage is repaired. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
@@ -45,14 +47,15 @@ additionally to [discovering Prometheus-compatible targets and scraping metrics
 
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) (
 `vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags)),
-unpack it and pass the following flags to the `vmagent` binary in order to start scraping Prometheus-compatible targets:
+unpack it and pass the following flags to the `vmagent` binary in order to start scraping Prometheus-compatible targets
+and sending the data to the Prometheus-compatible remote storage:
 
-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`).
+* `-promscrape.config` with the path to [Prometheus config file](https://docs.victoriametrics.com/sd_configs.html) (usually located at `/etc/prometheus/prometheus.yml`).
   The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file,
   so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` command-line flag.
   In this case `vmagent` ignores unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified
-  multiple times to replicate data concurrently to an arbitrary number of remote storage systems. See [various use cases](#use-cases).
+* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified
+  multiple times to replicate data concurrently to multiple remote storage systems. See [various use cases](#use-cases).
 
 Example command line:
 
@@ -60,8 +63,6 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-Example of scrape configuration for `-promscrape.config` argument you can find [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus.yml)
-
 See [how to scrape Prometheus-compatible targets](#how-to-collect-metrics-in-prometheus-format) for more details.
 
 If you use single-node VictoriaMetrics, then you can discover and scrape Prometheus-compatible targets directly from VictoriaMetrics
@@ -75,6 +76,8 @@ and sending it to the provided `-remoteWrite.url`:
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
+`vmagent` can save network bandwidth usage costs under high load when [VictoriaMetrics remote write protocol is enabled](#victoriametrics-remote-write-protocol).
+
 See [troubleshooting docs](#troubleshooting) if you encounter common issues with `vmagent`.
 
 Pass `-help` to `vmagent` in order to see [the full list of supported command-line flags with their descriptions](#advanced-usage).
@@ -120,7 +123,8 @@ data to the remote storage. It re-tries sending the data to remote storage until
 The maximum on-disk size for the buffered metrics can be limited with `-remoteWrite.maxDiskUsagePerURL`.
 
 `vmagent` works on various architectures from the IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
-See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+The `vmagent` can save network bandwidth usage costs by using [VictoriaMetrics remote write protocol](#victoriametrics-remote-write-protocol).
 
 ### Drop-in replacement for Prometheus
 
@@ -175,6 +179,32 @@ the `-remoteWrite.url` command-line flag should be configured as `<schema>://<vm
 according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format).
 There is also support for multitenant writes. See [these docs](#multitenancy).
 
+## VictoriaMetrics remote write protocol
+
+`vmagent` supports sending data to the configured `-remoteWrite.url` either via Prometheus remote write protocol
+or via VictoriaMetrics remote write protocol.
+
+VictoriaMetrics remote write protocol provides the following benefits comparing to Prometheus remote write protocol:
+
+- Reduced network bandwidth usage by 2x-5x. This allows saving network bandwidth usage costs when `vmagent` and
+  the configured remote storage systems are located in different datacenters, availability zones or regions.
+
+- Reduced disk read/write IO and disk space usage at `vmagent` when the remote storage is temporarily unavailable.
+  In this case `vmagent` buffers the incoming data to disk using the VictoriaMetrics remote write format.
+  This reduces disk read/write IO and disk space usage by 2x-5x comparing to Prometheus remote write format.
+
+`vmagent` automatically switches to VictoriaMetrics remote write protocol when it sends data to VictoriaMetrics components such as other `vmagent` instances,
+[single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html)
+or `vminsert` at [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+It is possible to force switch to VictoriaMetrics remote write protocol by specifying `-remoteWrite.forceVMProto`
+command-line flag for the corresponding `-remoteWrite.url`.
+It is possible to tune the compression level for VictoriaMetrics remote write protocol with `-remoteWrite.vmProtoCompressLevel` command-line flag.
+Bigger values reduce network usage at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of higher network usage.
+
+`vmagent` automatically switches to Prometheus remote write protocol when it sends data to old versions of VictoriaMetrics components
+or to other Prometheus-compatible remote storage systems. It is possible to force switch to Prometheus remote write protocol
+by specifying `-remoteWrite.forcePromProto` command-line flag for the corresponding `-remoteWrite.url`.
+
 ## Multitenancy
 
 By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`.
@@ -880,7 +910,7 @@ If you have suggestions for improvements or have found a bug - please open an is
   The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
 * Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling).
-  Such storage systems include Prometheus, Cortex and Thanos, which typically emit `out of order sample` errors.
+  Such storage systems include Prometheus, Mimir, Cortex and Thanos, which typically emit `out of order sample` errors.
   The best solution is to use remote storage with [backfilling support](https://docs.victoriametrics.com/#backfilling) such as VictoriaMetrics.
 
 * `vmagent` buffers scraped data at the `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
@@ -1211,8 +1241,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
      The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringCacheExpireDuration duration
+     The expire duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -kafka.consumer.topic array
      Kafka topic names for data consumption. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1268,11 +1302,11 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+     TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
   -opentsdbHTTPListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+     TCP and UDP address to listen for OpenTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
   -opentsdbListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
@@ -1340,6 +1374,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
@@ -1362,7 +1398,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.seriesLimitPerTarget int
      Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
-     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
@@ -1420,6 +1456,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.flushInterval duration
      Interval for flushing the data to remote storage. This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url (default 1s)
+  -remoteWrite.forcePromProto array
+     Whether to force Prometheus remote write protocol for sending data to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
+     Supports array of values separated by comma or specified via multiple flags.
+  -remoteWrite.forceVMProto array
+     Whether to force VictoriaMetrics remote write protocol for sending data to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.headers array
      Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1505,11 +1547,13 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.tmpDataPath string
      Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-     Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
+     Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
      Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
      Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.vmProtoCompressLevel int
+     The compression level for VictoriaMetrics remote write protocol. Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
   -sortLabels
      Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls

app/vmagent/csvimport/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,7 +26,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/datadog/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -28,7 +29,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }

app/vmagent/graphite/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,7 +20,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/influx/request_handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -36,7 +37,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(nil, db, rows, nil)
 	})
 }
@@ -54,7 +55,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/main.go

@@ -46,7 +46,8 @@ var (
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
 		"See also -influxListenAddr.useProxyProtocol")
@@ -56,12 +57,12 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
 	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
@@ -98,7 +99,7 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
 		return
 	}
 	if *dryRun {
@@ -108,7 +109,7 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exitting with 0 status code")
+		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
 	}
 
@@ -208,7 +209,7 @@ func getAuthTokenFromPath(path string) (*auth.Token, error) {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
@@ -253,6 +254,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write":
+		if common.HandleVMProtoServerHandshake(w, r) {
+			return true
+		}
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()

app/vmagent/native/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,12 +30,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmagent/opentsdb/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,7 +20,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/opentsdbhttp/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -24,7 +25,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/prometheusimport/request_handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -31,7 +32,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vmagent/prometheusimport/request_handler_test.go

@@ -23,7 +23,7 @@ var (
 func TestInsertHandler(t *testing.T) {
 	setUp()
 	defer tearDown()
-	req := httptest.NewRequest("POST", "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
+	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
 		t.Errorf("unxepected error %s", err)

app/vmagent/promremotewrite/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -27,7 +27,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/remotewrite/client.go

@@ -15,11 +15,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
+	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
@@ -69,8 +75,12 @@ var (
 type client struct {
 	sanitizedURL   string
 	remoteWriteURL string
-	fq             *persistentqueue.FastQueue
-	hc             *http.Client
+
+	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
+	useVMProto bool
+
+	fq *persistentqueue.FastQueue
+	hc *http.Client
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
@@ -122,19 +132,39 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+	hc := &http.Client{
+		Transport: tr,
+		Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
 		awsCfg:         awsCfg,
 		fq:             fq,
-		hc: &http.Client{
-			Transport: tr,
-			Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
-		},
-		stopCh: make(chan struct{}),
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
+
+	useVMProto := forceVMProto.GetOptionalArg(argIdx)
+	usePromProto := forcePromProto.GetOptionalArg(argIdx)
+	if useVMProto && usePromProto {
+		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
+	}
+	if !useVMProto && !usePromProto {
+		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
+		doRequest := func(url string) (*http.Response, error) {
+			return c.doRequest(url, nil)
+		}
+		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
+		if !useVMProto {
+			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
+				"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol", sanitizedURL)
+		}
+	}
+	c.useVMProto = useVMProto
+
 	return c
 }
 
@@ -291,38 +321,45 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlockHTTP returns false only if c.stopCh is closed.
-// Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.register(len(block), c.stopCh)
-	retryDuration := time.Second
-	retriesCount := 0
-	c.bytesSent.Add(len(block))
-	c.blocksSent.Inc()
-	sigv4Hash := ""
-	if c.awsCfg != nil {
-		sigv4Hash = awsapi.HashHex(block)
-	}
-
-again:
-	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
+func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+	reqBody := bytes.NewBuffer(body)
+	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
 	}
 	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	h.Set("Content-Encoding", "snappy")
-	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.useVMProto {
+		h.Set("Content-Encoding", "zstd")
+		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	} else {
+		h.Set("Content-Encoding", "snappy")
+		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	}
 	if c.awsCfg != nil {
+		sigv4Hash := awsapi.HashHex(body)
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
 			// there is no need in retry, request will be rejected by client.Do and retried by code below
 			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
+	return c.hc.Do(req)
+}
+
+// sendBlockHTTP sends the given block to c.remoteWriteURL.
+//
+// The function returns false only if c.stopCh is closed.
+// Otherwise it tries sending the block to remote storage indefinitely.
+func (c *client) sendBlockHTTP(block []byte) bool {
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
+	retriesCount := 0
+
+again:
 	startTime := time.Now()
-	resp, err := c.hc.Do(req)
+	resp, err := c.doRequest(c.remoteWriteURL, block)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
@@ -347,6 +384,8 @@ again:
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
 		c.requestsOKCount.Inc()
+		c.bytesSent.Add(len(block))
+		c.blocksSent.Inc()
 		return true
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()

app/vmagent/remotewrite/pendingseries.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -23,6 +24,9 @@ var (
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
+	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
+		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
+		"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -33,9 +37,10 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(pushBlock func(block []byte), significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte), isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.pushBlock = pushBlock
+	ps.wr.isVMRemoteWrite = isVMRemoteWrite
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
@@ -88,6 +93,9 @@ type writeRequest struct {
 	// pushBlock is called when whe write request is ready to be sent.
 	pushBlock func(block []byte)
 
+	// Whether to encode the write request with VictoriaMetrics remote write protocol.
+	isVMRemoteWrite bool
+
 	// How many significant figures must be left before sending the writeRequest to pushBlock.
 	significantFigures int
 
@@ -104,7 +112,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset pushBlock, significantFigures and roundDigits, since they are re-used.
+	// Do not reset lastFlushTime, pushBlock, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -126,7 +134,7 @@ func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
 	wr.adjustSampleValues()
 	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
-	pushWriteRequest(&wr.wr, wr.pushBlock)
+	pushWriteRequest(&wr.wr, wr.pushBlock, wr.isVMRemoteWrite)
 	wr.reset()
 }
 
@@ -188,7 +196,7 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte), isVMRemoteWrite bool) {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return
@@ -197,7 +205,11 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
 		zb := snappyBufPool.Get()
-		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		if isVMRemoteWrite {
+			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
+		} else {
+			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		}
 		writeRequestBufPool.Put(bb)
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
 			pushBlock(zb.B)
@@ -221,18 +233,18 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		pushWriteRequest(wr, pushBlock)
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples[n:]
-		pushWriteRequest(wr, pushBlock)
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples
 		return
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	pushWriteRequest(wr, pushBlock)
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries[n:]
-	pushWriteRequest(wr, pushBlock)
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries
 }
 

app/vmagent/remotewrite/pendingseries_test.go

@@ -2,40 +2,48 @@ package remotewrite
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/golang/snappy"
 )
 
 func TestPushWriteRequest(t *testing.T) {
-	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
+	for i, rowsCount := range rowsCounts {
+		expectedBlockLenProm := expectedBlockLensProm[i]
+		expectedBlockLenVM := expectedBlockLensVM[i]
 		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
-			testPushWriteRequest(t, rowsCount)
+			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
 		})
 	}
 }
 
-func testPushWriteRequest(t *testing.T, rowsCount int) {
-	wr := newTestWriteRequest(rowsCount, 10)
-	pushBlockLen := 0
-	pushBlock := func(block []byte) {
-		if pushBlockLen > 0 {
-			panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
+	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
+		t.Helper()
+		wr := newTestWriteRequest(rowsCount, 20)
+		pushBlockLen := 0
+		pushBlock := func(block []byte) {
+			if pushBlockLen > 0 {
+				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+			}
+			pushBlockLen = len(block)
+		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
+			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
+				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
 		}
-		pushBlockLen = len(block)
-	}
-	pushWriteRequest(wr, pushBlock)
-	b := prompbmarshal.MarshalWriteRequest(nil, wr)
-	zb := snappy.Encode(nil, b)
-	maxPushBlockLen := len(zb)
-	minPushBlockLen := maxPushBlockLen / 2
-	if pushBlockLen < minPushBlockLen {
-		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be at least %d bytes", pushBlockLen, minPushBlockLen)
-	}
-	if pushBlockLen > maxPushBlockLen {
-		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be smaller or equal to %d bytes", pushBlockLen, maxPushBlockLen)
 	}
+
+	// Check Prometheus remote write
+	f(false, expectedBlockLenProm, 0)
+
+	// Check VictoriaMetrics remote write
+	f(true, expectedBlockLenVM, 15)
 }
 
 func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {

app/vmagent/remotewrite/remotewrite.go

@@ -28,9 +28,9 @@ import (
 )
 
 var (
-	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
-		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
+		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
 	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
@@ -475,6 +475,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
+
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
@@ -495,7 +496,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq.MustWriteBlock, sf, rd)
+		pss[i] = newPendingSeries(fq.MustWriteBlock, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{

app/vmagent/vmimport/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmalert/Makefile

@@ -114,6 +114,9 @@ vmalert-linux-arm64:
 vmalert-linux-ppc64le:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmalert-linux-s390x:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmalert-linux-386:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert/README.md

@@ -28,7 +28,8 @@ Use this feature for the following cases:
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight and without extra dependencies.
-* Supports [reusable templates](#reusable-templates) for annotations.
+* Supports [reusable templates](#reusable-templates) for annotations;
+* Load of recording and alerting rules from local filesystem, GCS and S3.
 
 ## Limitations
 
@@ -404,6 +405,25 @@ The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz`
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
+### Reading rules from object storage
+
+[Enterprise version](https://docs.victoriametrics.com/enterprise.html) of `vmalert` may read alerting and recording rules
+from object storage:
+
+- `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
+- `./bin/vmalert -rule=gs://bucket/bir/alert.rules` would read rules from the given path at GCS bucket
+
+S3 and GCS paths support only matching by prefix, e.g. `s3://bucket/dir/rule_` matches
+all files with prefix `rule_` in the folder `dir`.
+
+The following [command-line flags](#flags) can be used for fine-tuning access to S3 and GCS:
+
+- `-s3.credsFilePath` - path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+- `-s3.configFilePath` - path to file with S3 configs. Configs are loaded from default location if not set.
+- `-s3.configProfile` - profile name for S3 configs. If no set, the value of the environment variable will be loaded (`AWS_PROFILE` or `AWS_DEFAULT_PROFILE`).
+- `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
+- `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
+
 ### Topology examples
 
 The following sections are showing how `vmalert` may be used and configured
@@ -601,6 +621,12 @@ can read the same rules configuration as normal, evaluate them on the given time
 results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
 data source for backfilling.
 
+Please note, that response caching may lead to unexpected results during and after backfilling process.
+In order to avoid this you need to reset cache contents or disable caching when using backfilling
+as described in [backfilling docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#backfilling).
+
+See a blogpost about [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/).
+
 ### How it works
 
 In `replay` mode vmalert works as a cli-tool and exits immediately after work is done.
@@ -705,31 +731,42 @@ a review to the dashboard.
 
 ## Troubleshooting
 
-vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed,
-the data is already present in configured `-datasource.url`:
+### Data delay
+
+Data delay is one of the most common issues with rules execution.
+vmalert executes configured rules within certain intervals at specifics timestamps. 
+It expects that the data is already present in configured `-datasource.url` at the moment of time when rule is executed:
 
 <img alt="vmalert expected evaluation" src="vmalert_ts_normal.gif">
 
 Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations
-may get empty response from datasource and produce empty recording rules or reset alerts state:
+may get empty response from the datasource and produce empty recording rules or reset alerts state:
 
 <img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
 
-By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
-This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
-Usually, this results into a 30s shift for recording rules results.
-Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
+Try the following recommendations to reduce the chance of hitting the data delay issue:
 
-Try the following recommendations in such cases:
-
-* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics
-are delivered to the datasource;
+* Always configure group's `evaluationInterval` to be bigger or at least equal to 
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution);
+* Ensure that `[duration]` value is at least twice bigger than 
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution). For example,
+if expression is `rate(my_metric[2m]) > 0` then ensure that `my_metric` resolution is at least `1m` or better `30s`. 
+If you use VictoriaMetrics as datasource, `[duration]` can be omitted and VictoriaMetrics will adjust it automatically.
 * If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback`
-command-line flag to add a time shift for evaluations;
-* If time intervals between datapoints in datasource are irregular or `>=5min` - try changing vmalert's
-`-datasource.queryStep` command-line flag to specify how far search query can lookback for the recent datapoint. 
-The recommendation is to have the step at least two times bigger than `scrape_interval`, since
-there are no guarantees that scrape will not fail.
+command-line flag to add a time shift for evaluations. Or extend `[duration]` to tolerate the delay.
+For example, `max_over_time(errors_total[10m]) > 0` will be active even if there is no data in datasource for last `9m`.
+* If [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution)
+in datasource is inconsistent or `>=5min` - try changing vmalert's `-datasource.queryStep` command-line flag to specify 
+how far search query can lookback for the recent datapoint. The recommendation is to have the step 
+at least two times bigger than the resolution.
+
+> Please note, data delay is inevitable in distributed systems. And it is better to account for it instead of ignoring.
+
+By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s
+(see `-search.latencyOffset` command-line flag at vmselect). Such delay is needed to eliminate risk of incomplete
+data on the moment of querying, since metrics collectors won't be able to deliver the data in time.
+
+### Alerts state
 
 Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that
 alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only
@@ -752,6 +789,8 @@ HTTP request sent by vmalert to the `-datasource.url` during evaluation. If spec
 no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
 moment when rule was evaluated.
 
+### Debug mode
+
 vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration
 and vmalert will start printing additional log messages:
 ```terminal
@@ -907,6 +946,10 @@ The shortlist of configuration flags is the following:
      Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880")
   -httpListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+  -insert.maxQueueDuration duration
+     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -923,6 +966,13 @@ The shortlist of configuration flags is the following:
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentInserts int
+     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
+  -memory.allowedBytes size
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
+  -memory.allowedPercent float
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
@@ -1089,8 +1139,8 @@ The shortlist of configuration flags is the following:
      Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
-  -replay.maxDatapointsPerQuery int
-     Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
+  -replay.maxDatapointsPerQuery /query_range
+     Max number of data points expected in one request. It affects the max time range for every /query_range request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
      Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration
@@ -1100,18 +1150,24 @@ The shortlist of configuration flags is the following:
   -replay.timeTo string
      The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
-     Path to the file with alert rules.
-     Supports patterns. Flag can be specified multiple times.
+     Path to the files with alerting and/or recording rules.
+     Supports hierarchical patterns and regexpes.
      Examples:
       -rule="/path/to/file". Path to a single file with alerting rules
-      -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-     absolute path to all .yaml files in root.
+      -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
      Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+     
+     Enterprise version of vmalert supports S3 and GCS paths to rules.
+     For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
+     S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
+     all files with prefix rule_ in folder dir.
+     See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
+     
      Supports an array of values separated by comma or specified via multiple flags.
   -rule.configCheckInterval duration
      Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
   -rule.maxResolveDuration duration
-     Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
+     Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group.
   -rule.resendDelay duration
      Minimum amount of time to wait before resending an alert to notifier
   -rule.templates array
@@ -1128,6 +1184,18 @@ The shortlist of configuration flags is the following:
      Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
      Whether to validate annotation and label templates (default true)
+  -s3.configFilePath string
+     Path to file with S3 configs. Configs are loaded from default location if not set.
+     See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.configProfile string
+     Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.credsFilePath string
+     Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+     See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.customEndpoint string
+     Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.forcePathStyle
+     Prefixing endpoint with bucket name when set false, true by default. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default true)
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string

app/vmalert/config/config.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
-	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 
@@ -201,21 +199,37 @@ func (r *Rule) Validate() error {
 // ValidateTplFn must validate the given annotations
 type ValidateTplFn func(annotations map[string]string) error
 
+// ParseSilent parses rule configs from given file patterns without emitting logs
+func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	files, err := readFromFS(pathPatterns, true)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
+	}
+	return parse(files, validateTplFn, validateExpressions)
+}
+
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	var fp []string
-	for _, pattern := range pathPatterns {
-		matches, err := filepath.Glob(pattern)
-		if err != nil {
-			return nil, fmt.Errorf("error reading file pattern %s: %w", pattern, err)
-		}
-		fp = append(fp, matches...)
+	files, err := readFromFS(pathPatterns, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
+	groups, err := parse(files, validateTplFn, validateExpressions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
+	}
+	if len(groups) < 1 {
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	errGroup := new(utils.ErrGroup)
 	var groups []Group
-	for _, file := range fp {
+	for file, data := range files {
 		uniqueGroups := map[string]struct{}{}
-		gr, err := parseFile(file)
+		gr, err := parseConfig(data)
 		if err != nil {
 			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
 			continue
@@ -237,20 +251,19 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
 	if err := errGroup.Err(); err != nil {
 		return nil, err
 	}
-	if len(groups) < 1 {
-		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
-	}
+	sort.SliceStable(groups, func(i, j int) bool {
+		if groups[i].File != groups[j].File {
+			return groups[i].File < groups[j].File
+		}
+		return groups[i].Name < groups[j].Name
+	})
 	return groups, nil
 }
 
-func parseFile(path string) ([]Group, error) {
-	data, err := os.ReadFile(path)
+func parseConfig(data []byte) ([]Group, error) {
+	data, err := envtemplate.ReplaceBytes(data)
 	if err != nil {
-		return nil, fmt.Errorf("error reading alert rule file %q: %w", path, err)
-	}
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
+		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
 	g := struct {
 		Groups []Group `yaml:"groups"`

app/vmalert/config/fs.go

@@ -0,0 +1,108 @@
+package config
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represent a file system abstract for reading files.
+type FS interface {
+	// Init initializes FS.
+	Init() error
+
+	// String must return human-readable representation of FS.
+	String() string
+
+	// List returns the list of file names which will be read via Read fn
+	List() ([]string, error)
+
+	// Read returns a list of read files in form of a map
+	// where key is a file name and value is a content of read file.
+	// Read must be called only after the successful Init call.
+	Read(files []string) (map[string][]byte, error)
+}
+
+var (
+	fsRegistryMu sync.Mutex
+	fsRegistry   = make(map[string]FS)
+)
+
+// readFromFS parses the given path list and inits FS for each item.
+// Once inited, readFromFS will try to read and return files from each FS.
+// readFromFS returns an error if at least one FS failed to init.
+// The function can be called multiple times but each unique path
+// will be inited only once.
+// If silent == true, readFromFS will not emit any logs.
+//
+// It is allowed to mix different FS types in path list.
+func readFromFS(paths []string, silent bool) (map[string][]byte, error) {
+	var err error
+	result := make(map[string][]byte)
+	for _, path := range paths {
+
+		fsRegistryMu.Lock()
+		fs, ok := fsRegistry[path]
+		if !ok {
+			fs, err = newFS(path)
+			if err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
+			}
+			if err := fs.Init(); err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
+			}
+			fsRegistry[path] = fs
+		}
+		fsRegistryMu.Unlock()
+
+		list, err := fs.List()
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files from %q", fs)
+		}
+
+		if !silent {
+			logger.Infof("found %d files to read from %q", len(list), fs)
+		}
+
+		if len(list) < 1 {
+			continue
+		}
+
+		files, err := fs.Read(list)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
+		}
+		for k, v := range files {
+			if _, ok := result[k]; ok {
+				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
+			}
+			result[k] = v
+		}
+	}
+	return result, nil
+}
+
+// newFS creates FS based on the give path.
+// Supported file systems are: fs
+func newFS(path string) (FS, error) {
+	scheme := "fs"
+	n := strings.Index(path, "://")
+	if n >= 0 {
+		scheme = path[:n]
+		path = path[n+len("://"):]
+	}
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	switch scheme {
+	case "fs":
+		return &fslocal.FS{Pattern: path}, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmalert/config/fs_test.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNewFS(t *testing.T) {
+	f := func(path, expStr string) {
+		t.Helper()
+		fs, err := newFS(path)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
+		if fs.String() != expStr {
+			t.Fatalf("expected FS %q; got %q", expStr, fs.String())
+		}
+	}
+
+	f("/foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+	f("fs:///foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+}
+
+func TestNewFSNegative(t *testing.T) {
+	f := func(path, expErr string) {
+		t.Helper()
+		_, err := newFS(path)
+		if err == nil {
+			t.Fatalf("expected to have err: %s", expErr)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Fatalf("expected to have err %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("", "path cannot be empty")
+	f("fs://", "path cannot be empty")
+	f("foobar://baz", `unsupported scheme "foobar"`)
+}

app/vmalert/config/fslocal/fslocal.go

@@ -0,0 +1,49 @@
+package fslocal
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// FS represents a local file system
+type FS struct {
+	// Pattern is used for matching one or multiple files.
+	// The pattern may describe hierarchical names such as
+	// /usr/*/bin/ed (assuming the Separator is '/').
+	Pattern string
+}
+
+// Init verifies that configured Pattern is correct
+func (fs *FS) Init() error {
+	_, err := filepath.Glob(fs.Pattern)
+	return err
+}
+
+// String implements Stringer interface
+func (fs *FS) String() string {
+	return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern)
+}
+
+// List returns the list of file names which will be read via Read fn
+func (fs *FS) List() ([]string, error) {
+	matches, err := filepath.Glob(fs.Pattern)
+	if err != nil {
+		return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err)
+	}
+	return matches, nil
+}
+
+// Read returns a map of read files where
+// key is the file name and value is file's content.
+func (fs *FS) Read(files []string) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	for _, path := range files {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading file %q: %w", path, err)
+		}
+		result[path] = data
+	}
+	return result, nil
+}

app/vmalert/datasource/vm.go

@@ -174,7 +174,7 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 }
 
 func (s *VMStorage) newRequestPOST() (*http.Request, error) {
-	req, err := http.NewRequest("POST", s.datasourceURL, nil)
+	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		return nil, err
 	}

app/vmalert/group.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"net/url"
@@ -44,6 +45,9 @@ type Group struct {
 	// channel accepts new Group obj
 	// which supposed to update current group
 	updateCh chan *Group
+	// evalCancel stores the cancel fn for interrupting
+	// rules evaluation. Used on groups update() and close().
+	evalCancel context.CancelFunc
 
 	metrics *groupMetrics
 }
@@ -233,11 +237,24 @@ func (g *Group) updateWith(newGroup *Group) error {
 	return nil
 }
 
+// interruptEval interrupts in-flight rules evaluations
+// within the group. It is expected that g.evalCancel
+// will be repopulated after the call.
+func (g *Group) interruptEval() {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	if g.evalCancel != nil {
+		g.evalCancel()
+	}
+}
+
 func (g *Group) close() {
 	if g.doneCh == nil {
 		return
 	}
 	close(g.doneCh)
+	g.interruptEval()
 	<-g.finishedCh
 
 	g.metrics.iterationDuration.Unregister()
@@ -254,6 +271,26 @@ var skipRandSleepOnGroupStart bool
 func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
+	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	if !skipRandSleepOnGroupStart {
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(g.Interval)
+		}
+		randSleep -= sleepOffset
+		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		select {
+		case <-ctx.Done():
+			sleepTimer.Stop()
+			return
+		case <-g.doneCh:
+			sleepTimer.Stop()
+			return
+		case <-sleepTimer.C:
+		}
+	}
+
 	e := &executor{
 		rw:                       rw,
 		notifiers:                nts,
@@ -263,7 +300,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 
-	eval := func(ts time.Time) {
+	eval := func(ctx context.Context, ts time.Time) {
 		g.metrics.iterationTotal.Inc()
 
 		start := time.Now()
@@ -285,7 +322,13 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		g.LastEvaluation = start
 	}
 
-	eval(evalTS)
+	evalCtx, cancel := context.WithCancel(ctx)
+	g.mu.Lock()
+	g.evalCancel = cancel
+	g.mu.Unlock()
+	defer g.evalCancel()
+
+	eval(evalCtx, evalTS)
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
@@ -309,6 +352,14 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			return
 		case ng := <-g.updateCh:
 			g.mu.Lock()
+
+			// it is expected that g.evalCancel will be evoked
+			// somewhere else to unblock group from the rules evaluation.
+			// we recreate the evalCtx and g.evalCancel, so it can
+			// be called again.
+			evalCtx, cancel = context.WithCancel(ctx)
+			g.evalCancel = cancel
+
 			err := g.updateWith(ng)
 			if err != nil {
 				logger.Errorf("group %q: failed to update: %s", g.Name, err)
@@ -333,7 +384,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			}
 			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
-			eval(evalTS)
+			eval(evalCtx, evalTS)
 		}
 	}
 }
@@ -407,6 +458,11 @@ func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDur
 
 	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			// the context can be cancelled on graceful shutdown
+			// or on group update. So no need to handle the error as usual.
+			return nil
+		}
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
 	}

app/vmalert/group_test.go

@@ -474,3 +474,31 @@ func TestFaultyRW(t *testing.T) {
 		t.Fatalf("expected to get an error from faulty RW client, got nil instead")
 	}
 }
+
+func TestCloseWithEvalInterruption(t *testing.T) {
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("failed to parse rules: %s", err)
+	}
+
+	const delay = time.Second * 2
+	fq := &fakeQuerierWithDelay{delay: delay}
+
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], fq, evalInterval, nil)
+
+	go g.start(context.Background(), nil, nil, nil)
+
+	time.Sleep(evalInterval * 20)
+
+	go func() {
+		g.close()
+	}()
+
+	deadline := time.Tick(delay / 2)
+	select {
+	case <-deadline:
+		t.Fatalf("deadline for close exceeded")
+	case <-g.finishedCh:
+	}
+}

app/vmalert/helpers_test.go

@@ -104,6 +104,24 @@ func (fqr *fakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time
 	return cp, req, nil
 }
 
+type fakeQuerierWithDelay struct {
+	fakeQuerier
+	delay time.Duration
+}
+
+func (fqd *fakeQuerierWithDelay) Query(ctx context.Context, expr string, ts time.Time) ([]datasource.Metric, *http.Request, error) {
+	timer := time.NewTimer(fqd.delay)
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+	return fqd.fakeQuerier.Query(ctx, expr, ts)
+}
+
+func (fqd *fakeQuerierWithDelay) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqd
+}
+
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert

app/vmalert/main.go

@@ -28,13 +28,19 @@ import (
 )
 
 var (
-	rulePath = flagutil.NewArrayString("rule", `Path to the file with alert rules.
-Supports patterns. Flag can be specified multiple times.
+	rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules.
+Supports hierarchical patterns and regexpes.
 Examples:
  -rule="/path/to/file". Path to a single file with alerting rules
- -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-absolute path to all .yaml files in root.
-Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
+ -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
+Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+
+Enterprise version of vmalert supports S3 and GCS paths to rules.
+For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
+S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
+all files with prefix rule_ in folder dir.
+See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
+`)
 
 	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
 	for rules annotations templating. Flag can be specified multiple times.
@@ -51,13 +57,14 @@ absolute path to all .tpl files in root.`)
 
 	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
 	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
-		"which is by default equal to 3 evaluation intervals of the parent group.")
+		"which by default is 4 times evaluationInterval of the parent group.")
 	resendDelay            = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param.")
@@ -338,7 +345,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
+		newGroupsCfg, err := config.ParseSilent(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)

app/vmalert/manager.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"sort"
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -87,32 +86,12 @@ func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error
 	m.wg.Add(1)
 	id := g.ID()
 	go func() {
-		// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
-		if !skipRandSleepOnGroupStart {
-			randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
-			sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
-			if randSleep < sleepOffset {
-				randSleep += uint64(g.Interval)
-			}
-			randSleep -= sleepOffset
-			sleepTimer := time.NewTimer(time.Duration(randSleep))
-			select {
-			case <-ctx.Done():
-				sleepTimer.Stop()
-				return
-			case <-g.doneCh:
-				sleepTimer.Stop()
-				return
-			case <-sleepTimer.C:
-			}
-		}
+		defer m.wg.Done()
 		if restore {
 			g.start(ctx, m.notifiers, m.rw, m.rr)
 		} else {
 			g.start(ctx, m.notifiers, m.rw, nil)
 		}
-
-		m.wg.Done()
 	}()
 	m.groups[id] = g
 	return nil
@@ -168,6 +147,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	}
 	for _, ng := range groupsRegistry {
 		if err := m.startGroup(ctx, ng, restore); err != nil {
+			m.groupsMu.Unlock()
 			return err
 		}
 	}
@@ -181,6 +161,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 				old.updateCh <- new
 				wg.Done()
 			}(item.old, item.new)
+			item.old.interruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alert.go

@@ -41,7 +41,7 @@ type Alert struct {
 	LastSent time.Time
 	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	// ID is the unique identifer for the Alert
+	// ID is the unique identifier for the Alert
 	ID uint64
 	// Restored is true if Alert was restored after restart
 	Restored bool

app/vmalert/notifier/alertmanager.go

@@ -64,7 +64,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
-	req, err := http.NewRequest("POST", am.addr, b)
+	req, err := http.NewRequest(http.MethodPost, am.addr, b)
 	if err != nil {
 		return err
 	}

app/vmalert/notifier/config.go

@@ -29,7 +29,7 @@ type Config struct {
 	// ConsulSDConfigs contains list of settings for service discovery via Consul
 	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
-	// DNSSDConfigs ontains list of settings for service discovery via DNS.
+	// DNSSDConfigs contains list of settings for service discovery via DNS.
 	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
 	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
 

app/vmalert/remotewrite/remotewrite.go

@@ -225,7 +225,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 func (c *Client) send(ctx context.Context, data []byte) error {
 	r := bytes.NewReader(data)
-	req, err := http.NewRequest("POST", c.addr, r)
+	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}

app/vmalert/templates/template.go

@@ -225,7 +225,7 @@ func templateFuncs() textTpl.FuncMap {
 		"toLower": strings.ToLower,
 
 		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
-		// This funcion is deprectated.
+		// This function is deprecated.
 		//
 		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
 		// these functions properly escape `\n` and `\r` chars according to their purpose.

app/vmalert/web.go

@@ -57,7 +57,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	switch r.URL.Path {
 	case "/", "/vmalert", "/vmalert/":
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			httpserver.Errorf(w, r, "path %q supports only GET method", r.URL.Path)
 			return false
 		}

app/vmauth/Makefile

@@ -84,6 +84,9 @@ vmauth-linux-arm64:
 vmauth-linux-ppc64le:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmauth-linux-s390x:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmauth-linux-386:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmauth/README.md

@@ -28,7 +28,36 @@ accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.co
 
 ## Load balancing
 
-Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls.
+In the latter case `vmauth` balances load among the configured urls in least-loaded round-robin manner.
+`vmauth` retries failing `GET` requests across the configured list of urls.
+This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes
+in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+
+## Concurrency limiting
+
+`vmauth` limits the number of concurrent requests it can proxy according to the following command-line flags:
+
+- `-maxConcurrentRequests` limits the global number of concurrent requests `vmauth` can serve across all the configured users.
+- `-maxConcurrentPerUserRequests` limits the number of concurrent requests `vmauth` can serve per each configured user.
+
+It is also possible to set individual limits on the number of concurrent requests per each user
+with the `max_concurrent_requests` option - see [auth config example](#auth-config).
+
+`vmauth` responds with `429 Too Many Requests` HTTP error when the number of concurrent requests exceeds the configured limits.
+
+The following [metrics](#monitoring) related to concurrency limits are exposed by `vmauth`:
+
+- `vmauth_concurrent_requests_capacity` - the global limit on the number of concurrent requests `vmauth` can serve.
+  It is set via `-maxConcurrentRequests` command-line flag.
+- `vmauth_concurrent_requests_current` - the current number of concurrent requests `vmauth` processes.
+- `vmauth_concurrent_requests_limit_reached_total` - the number of requests rejected with `429 Too Many Requests` error
+  because of the global concurrency limit has been reached.
+- `vmauth_user_concurrent_requests_capacity{username="..."}` - the limit on the number of concurrent requests for the given `username`.
+- `vmauth_user_concurrent_requests_current{username="..."}` - the current number of concurrent requests for the given `username`.
+- `vmauth_user_concurrent_requests_limit_reached_total{username="foo"}` - the number of requests rejected with `429 Too Many Requests` error
+  because of the concurrency limit has been reached for the given `username`.
+
 
 ## Auth config
 
@@ -55,26 +84,27 @@ users:
   headers:
   - "X-Scope-OrgID: foobar"
 
-  # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://localhost:8428 .
+  # are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  #
+  # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests.
+  # Excess concurrent requests are rejected with 429 HTTP status code.
+  # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags.
 - username: "local-single-node"
   password: "***"
   url_prefix: "http://localhost:8428"
+  max_concurrent_requests: 10
 
-  # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
   # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
-- username: "local-single-node"
+- username: "local-single-node2"
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
-  # The user for querying account 123 in VictoriaMetrics cluster
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+  # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
   #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
   #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
@@ -84,10 +114,8 @@ users:
   - "http://vmselect1:8481/select/123/prometheus"
   - "http://vmselect2:8481/select/123/prometheus"
 
-  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+  # are load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
   # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
   #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
   #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write
@@ -265,7 +293,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -httpListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
   -logInvalidAuthTokens
      Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
@@ -284,8 +312,10 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentPerUserRequests int
+     The maximum number of concurrent requests vmauth can process per each configured user. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option in per-user config (default 300)
   -maxConcurrentRequests int
-     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend (default 1000)
+     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options (default 1000)
   -maxIdleConnsPerBackend int
      The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
   -memory.allowedBytes size

app/vmauth/auth_config.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -32,17 +33,43 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name        string     `yaml:"name,omitempty"`
-	BearerToken string     `yaml:"bearer_token,omitempty"`
-	Username    string     `yaml:"username,omitempty"`
-	Password    string     `yaml:"password,omitempty"`
-	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
-	URLMaps     []URLMap   `yaml:"url_map,omitempty"`
-	Headers     []Header   `yaml:"headers,omitempty"`
+	Name                  string     `yaml:"name,omitempty"`
+	BearerToken           string     `yaml:"bearer_token,omitempty"`
+	Username              string     `yaml:"username,omitempty"`
+	Password              string     `yaml:"password,omitempty"`
+	URLPrefix             *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMaps               []URLMap   `yaml:"url_map,omitempty"`
+	Headers               []Header   `yaml:"headers,omitempty"`
+	MaxConcurrentRequests int        `yaml:"max_concurrent_requests,omitempty"`
+
+	concurrencyLimitCh      chan struct{}
+	concurrencyLimitReached *metrics.Counter
 
 	requests *metrics.Counter
 }
 
+func (ui *UserInfo) beginConcurrencyLimit() error {
+	select {
+	case ui.concurrencyLimitCh <- struct{}{}:
+		return nil
+	default:
+		ui.concurrencyLimitReached.Inc()
+		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+	}
+}
+
+func (ui *UserInfo) endConcurrencyLimit() {
+	<-ui.concurrencyLimitCh
+}
+
+func (ui *UserInfo) getMaxConcurrentRequests() int {
+	mcr := ui.MaxConcurrentRequests
+	if mcr <= 0 {
+		mcr = *maxConcurrentPerUserRequests
+	}
+	return mcr
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -83,16 +110,77 @@ type SrcPath struct {
 	re        *regexp.Regexp
 }
 
-// URLPrefix represents pased `url_prefix`
+// URLPrefix represents passed `url_prefix`
 type URLPrefix struct {
-	n    uint32
-	urls []*url.URL
+	n   uint32
+	bus []*backendURL
 }
 
-func (up *URLPrefix) getNextURL() *url.URL {
+type backendURL struct {
+	brokenDeadline     uint64
+	concurrentRequests int32
+	url                *url.URL
+}
+
+func (bu *backendURL) isBroken() bool {
+	ct := fasttime.UnixTimestamp()
+	return ct < atomic.LoadUint64(&bu.brokenDeadline)
+}
+
+func (bu *backendURL) setBroken() {
+	deadline := fasttime.UnixTimestamp() + 3
+	atomic.StoreUint64(&bu.brokenDeadline, deadline)
+}
+
+func (bu *backendURL) put() {
+	atomic.AddInt32(&bu.concurrentRequests, -1)
+}
+
+func (up *URLPrefix) getBackendsCount() int {
+	return len(up.bus)
+}
+
+// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+//
+// backendURL.put() must be called on the returned backendURL after the request is complete.
+func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
+	bus := up.bus
+	if len(bus) == 1 {
+		// Fast path - return the only backend url.
+		bu := bus[0]
+		atomic.AddInt32(&bu.concurrentRequests, 1)
+		return bu
+	}
+
+	// Slow path - select other backend urls.
 	n := atomic.AddUint32(&up.n, 1)
-	idx := n % uint32(len(up.urls))
-	return up.urls[idx]
+
+	for i := uint32(0); i < uint32(len(bus)); i++ {
+		idx := (n + i) % uint32(len(bus))
+		bu := bus[idx]
+		if bu.isBroken() {
+			continue
+		}
+		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
+			// Fast path - return the backend with zero concurrently executed requests.
+			return bu
+		}
+	}
+
+	// Slow path - return the backend with the minimum number of concurrently executed requests.
+	buMin := bus[n%uint32(len(bus))]
+	minRequests := atomic.LoadInt32(&buMin.concurrentRequests)
+	for _, bu := range bus {
+		if bu.isBroken() {
+			continue
+		}
+		if n := atomic.LoadInt32(&bu.concurrentRequests); n < minRequests {
+			buMin = bu
+			minRequests = n
+		}
+	}
+	atomic.AddInt32(&buMin.concurrentRequests, 1)
+	return buMin
 }
 
 // UnmarshalYAML unmarshals up from yaml.
@@ -121,31 +209,33 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 	default:
 		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
 	}
-	pus := make([]*url.URL, len(urls))
+	bus := make([]*backendURL, len(urls))
 	for i, u := range urls {
 		pu, err := url.Parse(u)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
 		}
-		pus[i] = pu
+		bus[i] = &backendURL{
+			url: pu,
+		}
 	}
-	up.urls = pus
+	up.bus = bus
 	return nil
 }
 
 // MarshalYAML marshals up to yaml.
 func (up *URLPrefix) MarshalYAML() (interface{}, error) {
 	var b []byte
-	if len(up.urls) == 1 {
-		u := up.urls[0].String()
+	if len(up.bus) == 1 {
+		u := up.bus[0].url.String()
 		b = strconv.AppendQuote(b, u)
 		return string(b), nil
 	}
 	b = append(b, '[')
-	for i, pu := range up.urls {
-		u := pu.String()
+	for i, bu := range up.bus {
+		u := bu.url.String()
 		b = strconv.AppendQuote(b, u)
-		if i+1 < len(up.urls) {
+		if i+1 < len(up.bus) {
 			b = append(b, ',')
 		}
 	}
@@ -298,29 +388,44 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
+		name := ui.name()
 		if ui.BearerToken != "" {
-			name := "bearer_token"
-			if ui.Name != "" {
-				name = ui.Name
-			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 		}
 		if ui.Username != "" {
-			name := ui.Username
-			if ui.Name != "" {
-				name = ui.Name
-			}
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 		}
+		mcr := ui.getMaxConcurrentRequests()
+		ui.concurrencyLimitCh = make(chan struct{}, mcr)
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+			return float64(cap(ui.concurrencyLimitCh))
+		})
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+			return float64(len(ui.concurrencyLimitCh))
+		})
 		byAuthToken[at1] = ui
 		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
+func (ui *UserInfo) name() string {
+	if ui.Name != "" {
+		return ui.Name
+	}
+	if ui.Username != "" {
+		return ui.Username
+	}
+	if ui.BearerToken != "" {
+		return "bearer_token"
+	}
+	return ""
+}
+
 func getAuthTokens(bearerToken, username, password string) (string, string) {
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
@@ -342,12 +447,12 @@ func getAuthToken(bearerToken, username, password string) string {
 }
 
 func (up *URLPrefix) sanitize() error {
-	for i, pu := range up.urls {
-		puNew, err := sanitizeURLPrefix(pu)
+	for _, bu := range up.bus {
+		puNew, err := sanitizeURLPrefix(bu.url)
 		if err != nil {
 			return err
 		}
-		up.urls[i] = puNew
+		bu.url = puNew
 	}
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -218,11 +218,13 @@ users:
 - username: foo
   password: bar
   url_prefix: http://aaa:343/bbb
+  max_concurrent_requests: 5
 `, map[string]*UserInfo{
 		getAuthToken("", "foo", "bar"): {
-			Username:  "foo",
-			Password:  "bar",
-			URLPrefix: mustParseURL("http://aaa:343/bbb"),
+			Username:              "foo",
+			Password:              "bar",
+			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
+			MaxConcurrentRequests: 5,
 		},
 	})
 
@@ -390,15 +392,17 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	pus := make([]*url.URL, len(us))
+	bus := make([]*backendURL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		pus[i] = pu
+		bus[i] = &backendURL{
+			url: pu,
+		}
 	}
 	return &URLPrefix{
-		urls: pus,
+		bus: bus,
 	}
 }

app/vmauth/example_config.yml

@@ -18,26 +18,27 @@ users:
   headers:
   - "X-Scope-OrgID: foobar"
 
-  # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://localhost:8428 .
+  # are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  #
+  # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests.
+  # Excess concurrent requests are rejected with 429 HTTP status code.
+  # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags.
 - username: "local-single-node"
   password: "***"
   url_prefix: "http://localhost:8428"
+  max_concurrent_requests: 10
 
-  # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
   # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
-- username: "local-single-node"
+- username: "local-single-node2"
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
-  # The user for querying account 123 in VictoriaMetrics cluster
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+  # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
   #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
   #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
@@ -47,10 +48,8 @@ users:
   - "http://vmselect1:8481/select/123/prometheus"
   - "http://vmselect2:8481/select/123/prometheus"
 
-  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+  # are load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
   # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
   #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
   #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write

app/vmauth/main.go

@@ -28,12 +28,16 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
 		"See also -maxConcurrentRequests")
 	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend")
+		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
+		"in per-user config")
 	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
@@ -107,32 +111,54 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	ui.requests.Inc()
-	targetURL, headers, err := createTargetURL(ui, r.URL)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
-		return true
-	}
 
 	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-	default:
-		concurrentRequestsLimitReachedTotal.Inc()
-		w.Header().Add("Retry-After", "10")
-		err := &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh)),
-			StatusCode: http.StatusTooManyRequests,
+		if err := ui.beginConcurrencyLimit(); err != nil {
+			handleConcurrencyLimitError(w, r, err)
+			<-concurrencyLimitCh
+			return true
 		}
-		httpserver.Errorf(w, r, "%s", err)
+	default:
+		concurrentRequestsLimitReached.Inc()
+		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
+		handleConcurrencyLimitError(w, r, err)
 		return true
 	}
-	processRequest(w, r, targetURL, headers)
+	processRequest(w, r, ui)
+	ui.endConcurrencyLimit()
 	<-concurrencyLimitCh
 	return true
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+	u := normalizeURL(r.URL)
+	up, headers, err := ui.getURLPrefixAndHeaders(u)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
+		return
+	}
+	maxAttempts := up.getBackendsCount()
+	for i := 0; i < maxAttempts; i++ {
+		bu := up.getLeastLoadedBackendURL()
+		targetURL := mergeURLs(bu.url, u)
+		ok := tryProcessingRequest(w, r, targetURL, headers)
+		bu.put()
+		if ok {
+			return
+		}
+		bu.setBroken()
+	}
+	err = &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
+		StatusCode: http.StatusServiceUnavailable,
+	}
+	httpserver.Errorf(w, r, "%s", err)
+}
+
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
@@ -142,12 +168,20 @@ func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL,
 	transportOnce.Do(transportInit)
 	res, err := transport.RoundTrip(req)
 	if err != nil {
-		err = &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("error when proxying the request to %q: %s", targetURL, err),
-			StatusCode: http.StatusBadGateway,
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		if r.Method == http.MethodPost || r.Method == http.MethodPut {
+			// It is impossible to retry POST and PUT requests,
+			// since we already proxied the request body to the backend.
+			err = &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
 		}
-		httpserver.Errorf(w, r, "%s", err)
-		return
+		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying the request to %q: %s", remoteAddr, requestURI, targetURL, err)
+		return false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
@@ -162,8 +196,9 @@ func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL,
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-		return
+		return true
 	}
+	return true
 }
 
 var copyBufPool bytesutil.ByteBufferPool
@@ -269,7 +304,7 @@ func concurrencyLimitInit() {
 	})
 }
 
-var concurrentRequestsLimitReachedTotal = metrics.NewCounter("vmauth_concurrent_requests_limit_reached_total")
+var concurrentRequestsLimitReached = metrics.NewCounter("vmauth_concurrent_requests_limit_reached_total")
 
 func usage() {
 	const s = `
@@ -279,3 +314,12 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
 `
 	flagutil.Usage(s)
 }
+
+func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	w.Header().Add("Retry-After", "10")
+	err = &httpserver.ErrorWithStatusCode{
+		Err:        err,
+		StatusCode: http.StatusTooManyRequests,
+	}
+	httpserver.Errorf(w, r, "%s", err)
+}

app/vmauth/target_url.go

@@ -7,11 +7,6 @@ import (
 	"strings"
 )
 
-func (up *URLPrefix) mergeURLs(requestURI *url.URL) *url.URL {
-	pu := up.getNextURL()
-	return mergeURLs(pu, requestURI)
-}
-
 func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	targetURL := *uiURL
 	targetURL.Path += requestURI.Path
@@ -35,12 +30,27 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header, error) {
+	for _, e := range ui.URLMaps {
+		for _, sp := range e.SrcPaths {
+			if sp.match(u.Path) {
+				return e.URLPrefix, e.Headers, nil
+			}
+		}
+	}
+	if ui.URLPrefix != nil {
+		return ui.URLPrefix, ui.Headers, nil
+	}
+	missingRouteRequests.Inc()
+	return nil, nil, fmt.Errorf("missing route for %q", u.String())
+}
+
+func normalizeURL(uOrig *url.URL) *url.URL {
 	u := *uOrig
 	// Prevent from attacks with using `..` in r.URL.Path
 	u.Path = path.Clean(u.Path)
 	if !strings.HasSuffix(u.Path, "/") && strings.HasSuffix(uOrig.Path, "/") {
-		// The path.Clean() removes traling slash.
+		// The path.Clean() removes trailing slash.
 		// Return it back if needed.
 		// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752
 		u.Path += "/"
@@ -52,16 +62,5 @@ func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1554
 		u.Path = ""
 	}
-	for _, e := range ui.URLMaps {
-		for _, sp := range e.SrcPaths {
-			if sp.match(u.Path) {
-				return e.URLPrefix.mergeURLs(&u), e.Headers, nil
-			}
-		}
-	}
-	if ui.URLPrefix != nil {
-		return ui.URLPrefix.mergeURLs(&u), ui.Headers, nil
-	}
-	missingRouteRequests.Inc()
-	return nil, nil, fmt.Errorf("missing route for %q", u.String())
+	return &u
 }

app/vmauth/target_url_test.go

@@ -13,10 +13,14 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, headers, err := createTargetURL(ui, u)
+		u = normalizeURL(u)
+		up, headers, err := ui.getURLPrefixAndHeaders(u)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
+		bu := up.getLeastLoadedBackendURL()
+		target := mergeURLs(bu.url, u)
+		bu.put()
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
@@ -119,15 +123,16 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, headers, err := createTargetURL(ui, u)
+		u = normalizeURL(u)
+		up, headers, err := ui.getURLPrefixAndHeaders(u)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if target != nil {
-			t.Fatalf("unexpected target=%q; want empty string", target)
+		if up != nil {
+			t.Fatalf("unexpected non-empty up=%#v", up)
 		}
 		if headers != nil {
-			t.Fatalf("unexpected headers=%q; want empty string", headers)
+			t.Fatalf("unexpected non-empty headers=%q", headers)
 		}
 	}
 	f(&UserInfo{}, "/foo/bar")

app/vmbackup/Makefile

@@ -75,6 +75,9 @@ vmbackup-linux-arm64:
 vmbackup-linux-ppc64le:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmbackup-linux-s390x:
+	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmbackup-linux-386:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmbackup/main.go

@@ -51,27 +51,27 @@ func main() {
 
 	if len(*snapshotCreateURL) > 0 {
 		// create net/url object
-		createUrl, err := url.Parse(*snapshotCreateURL)
+		createURL, err := url.Parse(*snapshotCreateURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotCreateURL: %s", err)
 		}
 		if len(*snapshotName) > 0 {
 			logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case")
 		}
-		logger.Infof("Snapshot create url %s", createUrl.Redacted())
+		logger.Infof("Snapshot create url %s", createURL.Redacted())
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
 			if err != nil {
 				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
 			}
 		}
-		deleteUrl, err := url.Parse(*snapshotDeleteURL)
+		deleteURL, err := url.Parse(*snapshotDeleteURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotDeleteURL: %s", err)
 		}
-		logger.Infof("Snapshot delete url %s", deleteUrl.Redacted())
+		logger.Infof("Snapshot delete url %s", deleteURL.Redacted())
 
-		name, err := snapshot.Create(createUrl.String())
+		name, err := snapshot.Create(createURL.String())
 		if err != nil {
 			logger.Fatalf("cannot create snapshot: %s", err)
 		}
@@ -81,7 +81,7 @@ func main() {
 		}
 
 		defer func() {
-			err := snapshot.Delete(deleteUrl.String(), name)
+			err := snapshot.Delete(deleteURL.String(), name)
 			if err != nil {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}

app/vmctl/Makefile

@@ -78,6 +78,9 @@ vmctl-linux-arm64:
 vmctl-linux-ppc64le:
 	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmctl-linux-s390x:
+	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmctl-linux-386:
 	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmctl/README.md

@@ -483,6 +483,10 @@ Processing ranges: 8798 / 8798 [████████████████
 2022/10/19 16:45:37 Total time: 1m19.406283424s
 ```
 
+Migrating big volumes of data may result in remote read client reaching the timeout.
+Consider increasing the value of `--remote-read-http-timeout` (default `5m`) command-line flag when seeing
+timeouts or `context canceled` errors.
+
 ### Filtering
 
 The filtering consists of two parts: by labels and time.
@@ -733,21 +737,33 @@ or higher.
 
 See `./vmctl vm-native --help` for details and full list of flags.
 
-In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`)
-and processing is done by "destination" (`dst`). Because of that, `vmctl` doesn't actually know how much data will be
-processed and can't show the progress bar. It will show the current processing speed and total number of processed bytes:
+Migration in `vm-native` mode takes two steps:
+1. Explore the list of the metrics to migrate via `/api/v1/series` API;
+2. Migrate explored metrics one-by-one.
 
 ```
-./vmctl vm-native --vm-native-src-addr=http://localhost:8528  \
-  --vm-native-dst-addr=http://localhost:8428 \
-  --vm-native-filter-match='{job="vmagent"}' \
-  --vm-native-filter-time-start='2020-01-01T20:07:00Z'
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ 
+    --vm-native-dst-addr=http://localhost:8428 \
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \
+    --vm-native-filter-match='{__name__=~"vm_cache_.*"}'    
 VictoriaMetrics Native import mode
-Initing export pipe from "http://localhost:8528" with filters:
-        filter: match[]={job="vmagent"}
-Initing import process to "http://localhost:8428":
-Total: 336.75 KiB ↖ Speed: 454.46 KiB p/s
-2020/10/13 17:04:59 Total time: 952.143376ms
+
+2023/03/02 09:22:02 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__=~"vm_cache_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:22:02 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:22:04 Requests to make: 9
+Requests to make: 9 / 9 [███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/03/02 09:22:06 Import finished!
+2023/03/02 09:22:06 VictoriaMetrics importer stats:
+  time spent while importing: 3.632638875s;
+  total bytes: 7.8 MB;
+  bytes/s: 2.1 MB;
+  requests: 9;
+  requests retries: 0;
+2023/03/02 09:22:06 Total time: 3.633127625s
 ```
 
 Importing tips:
@@ -755,6 +771,7 @@ Importing tips:
 1. Migrating big volumes of data may result in reaching the safety limits on `src` side.
 Please verify that `-search.maxExportDuration` and `-search.maxExportSeries` were set with
 proper values for `src`. If hitting the limits, follow the recommendations [here](https://docs.victoriametrics.com/#how-to-export-data-in-native-format).
+If hitting `the number of matching timeseries exceeds...` error, adjust filters to match less time series or update `-search.maxSeries` command-line flag on vmselect/vmsingle;
 2. Migrating all the metrics from one VM to another may collide with existing application metrics
 (prefixed with `vm_`) at destination and lead to confusion when using
 [official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
@@ -766,71 +783,64 @@ Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/
 5. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
 and specify `accountID` param.
 6. When migrating large volumes of data it might be useful to use `--vm-native-step-interval` flag to split single process into smaller steps.
+7. `vmctl` supports `--vm-concurrency` which controls the number of concurrent workers that process the input from source query results.
+Please note that each import request can load up to a single vCPU core on VictoriaMetrics. So try to set it according
+to allocated CPU resources of your VictoriaMetrics installation.
+8. `vmctl` supports `--vm-native-src-headers` and `--vm-native-dst-headers` which defines headers to send with each request
+to the corresponding source address.
+9. `vmctl` supports `--vm-native-disable-http-keep-alive` to allow `vmctl` to use non-persistent HTTP connections to avoid
+error `use of closed network connection` when run a longer export.
+10. Migrating data with overlapping time range for destination data can produce duplicates series at destination.
+To avoid duplicates on the destination set `-dedup.minScrapeInterval=1ms` for `vmselect` and `vmstorage`.
+This will instruct `vmselect` and `vmstorage` to ignore duplicates with match timestamps.
+
+In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`)
+and processing is done by "destination" (`dst`). So no extra memory or CPU resources required on `vmctl` side. Only
+`src` and `dst` resource matter.
 
 #### Using time-based chunking of migration
 
-It is possible split migration process into set of smaller batches based on time. This is especially useful when migrating large volumes of data as this adds indication of progress and ability to restore process from certain point in case of failure.
+It is possible split migration process into set of smaller batches based on time. This is especially useful when 
+migrating large volumes of data as this adds indication of progress and ability to restore process from certain point 
+in case of failure.
 
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`.
-Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for export process.
+To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`, `minute`.
+Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for 
+export process.
 
 Every range is being processed independently, which means that:
 - after range processing is finished all data within range is migrated
-- if process fails on one of stages it is guaranteed that data of prior stages is already written, so it is possible to restart process starting from failed range
+- if process fails on one of stages it is guaranteed that data of prior stages is already written,
+so it is possible to restart process starting from failed range.
 
-It is recommended using the `month` step when migrating the data over multiple months, since the migration with `day` and `hour` steps may take longer time to complete
-because of additional overhead.
+It is recommended using the `month` step when migrating the data over multiple months, 
+since the migration with `day` and `hour` steps may take longer time to complete because of additional overhead.
 
 Usage example:
 ```console
-./vmctl vm-native 
-    --vm-native-filter-time-start 2022-06-17T00:07:00Z \
-    --vm-native-filter-time-end 2022-10-03T00:07:00Z \
-    --vm-native-src-addr http://localhost:8428 \
-    --vm-native-dst-addr http://localhost:8528 \
-    --vm-native-step-interval=month
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ 
+    --vm-native-dst-addr=http://localhost:8428 \
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \
+    --vm-native-step-interval=month \
+    --vm-native-filter-match='{__name__=~"vm_cache_.*"}'    
 VictoriaMetrics Native import mode
-2022/08/30 19:48:24 Processing range 1/5: 2022-06-17T00:07:00Z - 2022-06-30T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-06-17T00:07:00Z
-        end: 2022-06-30T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 28.89 KiB p/s 
-2022/08/30 19:48:24 Processing range 2/5: 2022-07-01T00:00:00Z - 2022-07-31T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-07-01T00:00:00Z
-        end: 2022-07-31T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 164.35 KiB p/s 
-2022/08/30 19:48:24 Processing range 3/5: 2022-08-01T00:00:00Z - 2022-08-31T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-08-01T00:00:00Z
-        end: 2022-08-31T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 191.42 KiB p/s 
-2022/08/30 19:48:24 Processing range 4/5: 2022-09-01T00:00:00Z - 2022-09-30T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-09-01T00:00:00Z
-        end: 2022-09-30T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 141.04 KiB p/s 
-2022/08/30 19:48:24 Processing range 5/5: 2022-10-01T00:00:00Z - 2022-10-03T00:07:00Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-10-01T00:00:00Z
-        end: 2022-10-03T00:07:00Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 186.32 KiB p/s 
-2022/08/30 19:48:24 Total time: 12.680582ms
+
+2023/03/02 09:18:05 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__=~"vm_cache_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:18:05 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:18:07 Selected time range will be split into 5 ranges according to "month" step. Requests to make: 45.
+Requests to make: 45 / 45 [█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/03/02 09:18:12 Import finished!
+2023/03/02 09:18:12 VictoriaMetrics importer stats:
+  time spent while importing: 7.111870667s;
+  total bytes: 7.7 MB;
+  bytes/s: 1.1 MB;
+  requests: 45;
+  requests retries: 0;
+2023/03/02 09:18:12 Total time: 7.112405875s
 ```
 
 #### Cluster-to-cluster migration mode
@@ -842,70 +852,41 @@ Cluster-to-cluster uses `/admin/tenants` endpoint (available starting from [v1.8
 To use this mode you need to set `--vm-intercluster` flag to `true`, `--vm-native-src-addr` flag to 'http://vmselect:8481/' and `--vm-native-dst-addr` value to http://vminsert:8480/:
 
 ```console
-./bin/vmctl vm-native --vm-intercluster=true --vm-native-src-addr=http://localhost:8481/ --vm-native-dst-addr=http://172.17.0.3:8480/
+  ./vmctl vm-native --vm-native-src-addr=http://127.0.0.1:8481/ \
+  --vm-native-dst-addr=http://127.0.0.1:8480/ \
+  --vm-native-filter-match='{__name__="vm_app_uptime_seconds"}' \
+  --vm-native-filter-time-start='2023-02-01T00:00:00Z' \
+  --vm-native-step-interval=day \  
+--vm-intercluster
 VictoriaMetrics Native import mode
-2022/12/05 21:20:06 Discovered tenants: [123:1 12812919:1 1289198:1 1289:1283 12:1 1:0 1:1 1:1231231 1:1271727 1:12819 1:281 812891298:1]
-2022/12/05 21:20:06 Initing export pipe from "http://localhost:8481/select/123:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/123:1/prometheus/api/v1/import/native":
-Total: 61.13 MiB ↖ Speed: 2.05 MiB p/s 
-Total: 61.13 MiB ↗ Speed: 2.30 MiB p/s 
-2022/12/05 21:20:33 Initing export pipe from "http://localhost:8481/select/12812919:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/12812919:1/prometheus/api/v1/import/native":
-Total: 43.14 MiB ↘ Speed: 1.86 MiB p/s  
-Total: 43.14 MiB ↙ Speed: 2.36 MiB p/s 
-2022/12/05 21:20:51 Initing export pipe from "http://localhost:8481/select/1289198:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1289198:1/prometheus/api/v1/import/native":
-Total: 16.64 MiB ↗ Speed: 2.66 MiB p/s 
-Total: 16.64 MiB ↘ Speed: 2.19 MiB p/s 
-2022/12/05 21:20:59 Initing export pipe from "http://localhost:8481/select/1289:1283/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1289:1283/prometheus/api/v1/import/native":
-Total: 43.33 MiB ↙ Speed: 1.94 MiB p/s 
-Total: 43.33 MiB ↖ Speed: 2.35 MiB p/s 
-2022/12/05 21:21:18 Initing export pipe from "http://localhost:8481/select/12:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/12:1/prometheus/api/v1/import/native":
-Total: 63.78 MiB ↙ Speed: 1.96 MiB p/s 
-Total: 63.78 MiB ↖ Speed: 2.28 MiB p/s 
-2022/12/05 21:21:46 Initing export pipe from "http://localhost:8481/select/1:0/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:0/prometheus/api/v1/import/native":
-2022/12/05 21:21:46 Import finished!
-Total: 330 B ↗ Speed: 3.53 MiB p/s 
-2022/12/05 21:21:46 Initing export pipe from "http://localhost:8481/select/1:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1/prometheus/api/v1/import/native":
-Total: 63.81 MiB ↙ Speed: 1.96 MiB p/s 
-Total: 63.81 MiB ↖ Speed: 2.28 MiB p/s 
-2022/12/05 21:22:14 Initing export pipe from "http://localhost:8481/select/1:1231231/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1231231/prometheus/api/v1/import/native":
-Total: 63.84 MiB ↙ Speed: 1.93 MiB p/s 
-Total: 63.84 MiB ↖ Speed: 2.29 MiB p/s 
-2022/12/05 21:22:42 Initing export pipe from "http://localhost:8481/select/1:1271727/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1271727/prometheus/api/v1/import/native":
-Total: 54.37 MiB ↘ Speed: 1.90 MiB p/s 
-Total: 54.37 MiB ↙ Speed: 2.37 MiB p/s 
-2022/12/05 21:23:05 Initing export pipe from "http://localhost:8481/select/1:12819/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:12819/prometheus/api/v1/import/native":
-Total: 17.01 MiB ↙ Speed: 1.75 MiB p/s 
-Total: 17.01 MiB ↖ Speed: 2.15 MiB p/s 
-2022/12/05 21:23:13 Initing export pipe from "http://localhost:8481/select/1:281/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:281/prometheus/api/v1/import/native":
-Total: 63.89 MiB ↘ Speed: 1.90 MiB p/s 
-Total: 63.89 MiB ↙ Speed: 2.29 MiB p/s 
-2022/12/05 21:23:42 Initing export pipe from "http://localhost:8481/select/812891298:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/812891298:1/prometheus/api/v1/import/native":
-Total: 63.84 MiB ↖ Speed: 1.99 MiB p/s 
-Total: 63.84 MiB ↗ Speed: 2.26 MiB p/s 
-2022/12/05 21:24:10 Total time: 4m4.1466565s
+2023/02/28 10:41:42 Discovering tenants...
+2023/02/28 10:41:42 The following tenants were discovered: [0:0 1:0 2:0 3:0 4:0]
+2023/02/28 10:41:42 Initing import process from "http://127.0.0.1:8481/select/0:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/0:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 0:0 
+2023/02/28 10:41:42 Exploring metrics...
+2023/02/28 10:41:42 Found 1 metrics to import 
+2023/02/28 10:41:42 Selected time range will be split into 28 ranges according to "day" step. 
+Requests to make for tenant 0:0: 28 / 28 [███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+
+2023/02/28 10:41:45 Initing import process from "http://127.0.0.1:8481/select/1:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/1:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 1:0 
+2023/02/28 10:41:45 Exploring metrics...
+2023/02/28 10:41:45 Found 1 metrics to import 
+2023/02/28 10:41:45 Selected time range will be split into 28 ranges according to "day" step. Requests to make: 28 
+Requests to make for tenant 1:0: 28 / 28 [████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+
+...
+
+2023/02/28 10:42:49 Import finished!
+2023/02/28 10:42:49 VictoriaMetrics importer stats:
+  time spent while importing: 1m6.714210417s;
+  total bytes: 39.7 MB;
+  bytes/s: 594.4 kB;
+  requests: 140;
+  requests retries: 0;
+2023/02/28 10:42:49 Total time: 1m7.147971417s
 ```
 
 ## Verifying exported blocks from VictoriaMetrics
@@ -972,6 +953,7 @@ a sign of network issues or VM being overloaded. See the logs during import for
 By default `vmctl` waits confirmation from user before starting the import. If this is unwanted
 behavior and no user interaction required - pass `-s` flag to enable "silence" mode:
 
+See below the example of `vm-native` migration process:
 ```
     -s Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
 ```

app/vmctl/auth/auth.go

@@ -0,0 +1,222 @@
+package auth
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+)
+
+// HTTPClientConfig represents http client config.
+type HTTPClientConfig struct {
+	BasicAuth   *BasicAuthConfig
+	BearerToken string
+	Headers     string
+}
+
+// NewConfig creates auth config for the given hcc.
+func (hcc *HTTPClientConfig) NewConfig() (*Config, error) {
+	opts := &Options{
+		BasicAuth:   hcc.BasicAuth,
+		BearerToken: hcc.BearerToken,
+		Headers:     hcc.Headers,
+	}
+	return opts.NewConfig()
+}
+
+// BasicAuthConfig represents basic auth config.
+type BasicAuthConfig struct {
+	Username     string
+	Password     string
+	PasswordFile string
+}
+
+// ConfigOptions options which helps build Config
+type ConfigOptions func(config *HTTPClientConfig)
+
+// Generate returns Config based on the given params
+func Generate(filterOptions ...ConfigOptions) (*Config, error) {
+	authCfg := &HTTPClientConfig{}
+	for _, option := range filterOptions {
+		option(authCfg)
+	}
+
+	return authCfg.NewConfig()
+}
+
+// WithBasicAuth returns AuthConfigOptions and initialized BasicAuthConfig based on given params
+func WithBasicAuth(username, password string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if username != "" || password != "" {
+			config.BasicAuth = &BasicAuthConfig{
+				Username: username,
+				Password: password,
+			}
+		}
+	}
+}
+
+// WithBearer returns AuthConfigOptions and set BearerToken or BearerTokenFile based on given params
+func WithBearer(token string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if token != "" {
+			config.BearerToken = token
+		}
+	}
+}
+
+// WithHeaders returns AuthConfigOptions and set Headers based on the given params
+func WithHeaders(headers string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if headers != "" {
+			config.Headers = headers
+		}
+	}
+}
+
+// Config is auth config.
+type Config struct {
+	getAuthHeader      func() string
+	authHeaderLock     sync.Mutex
+	authHeader         string
+	authHeaderDeadline uint64
+
+	headers []keyValue
+
+	authDigest string
+}
+
+// SetHeaders sets the configured ac headers to req.
+func (ac *Config) SetHeaders(req *http.Request, setAuthHeader bool) {
+	reqHeaders := req.Header
+	for _, h := range ac.headers {
+		reqHeaders.Set(h.key, h.value)
+	}
+	if setAuthHeader {
+		if ah := ac.GetAuthHeader(); ah != "" {
+			reqHeaders.Set("Authorization", ah)
+		}
+	}
+}
+
+// GetAuthHeader returns optional `Authorization: ...` http header.
+func (ac *Config) GetAuthHeader() string {
+	f := ac.getAuthHeader
+	if f == nil {
+		return ""
+	}
+	ac.authHeaderLock.Lock()
+	defer ac.authHeaderLock.Unlock()
+	if fasttime.UnixTimestamp() > ac.authHeaderDeadline {
+		ac.authHeader = f()
+		// Cache the authHeader for a second.
+		ac.authHeaderDeadline = fasttime.UnixTimestamp() + 1
+	}
+	return ac.authHeader
+}
+
+type authContext struct {
+	// getAuthHeader must return <value> for 'Authorization: <value>' http request header
+	getAuthHeader func() string
+
+	// authDigest must contain the digest for the used authorization
+	// The digest must be changed whenever the original config changes.
+	authDigest string
+}
+
+func (ac *authContext) initFromBasicAuthConfig(ba *BasicAuthConfig) error {
+	if ba.Username == "" {
+		return fmt.Errorf("missing `username` in `basic_auth` section")
+	}
+	if ba.Password != "" {
+		ac.getAuthHeader = func() string {
+			// See https://en.wikipedia.org/wiki/Basic_access_authentication
+			token := ba.Username + ":" + ba.Password
+			token64 := base64.StdEncoding.EncodeToString([]byte(token))
+			return "Basic " + token64
+		}
+		ac.authDigest = fmt.Sprintf("basic(username=%q, password=%q)", ba.Username, ba.Password)
+		return nil
+	}
+	return nil
+}
+
+func (ac *authContext) initFromBearerToken(bearerToken string) error {
+	ac.getAuthHeader = func() string {
+		return "Bearer " + bearerToken
+	}
+	ac.authDigest = fmt.Sprintf("bearer(token=%q)", bearerToken)
+	return nil
+}
+
+// Options contain options, which must be passed to NewConfig.
+type Options struct {
+	// BasicAuth contains optional BasicAuthConfig.
+	BasicAuth *BasicAuthConfig
+
+	// BearerToken contains optional bearer token.
+	BearerToken string
+
+	// Headers contains optional http request headers in the form 'Foo: bar'.
+	Headers string
+}
+
+// NewConfig creates auth config from the given opts.
+func (opts *Options) NewConfig() (*Config, error) {
+	var ac authContext
+	if opts.BasicAuth != nil {
+		if ac.getAuthHeader != nil {
+			return nil, fmt.Errorf("cannot use both `authorization` and `basic_auth`")
+		}
+		if err := ac.initFromBasicAuthConfig(opts.BasicAuth); err != nil {
+			return nil, err
+		}
+	}
+	if opts.BearerToken != "" {
+		if ac.getAuthHeader != nil {
+			return nil, fmt.Errorf("cannot simultaneously use `authorization`, `basic_auth` and `bearer_token`")
+		}
+		if err := ac.initFromBearerToken(opts.BearerToken); err != nil {
+			return nil, err
+		}
+	}
+
+	headers, err := parseHeaders(opts.Headers)
+	if err != nil {
+		return nil, err
+	}
+	c := &Config{
+		getAuthHeader: ac.getAuthHeader,
+		headers:       headers,
+		authDigest:    ac.authDigest,
+	}
+	return c, nil
+}
+
+type keyValue struct {
+	key   string
+	value string
+}
+
+func parseHeaders(headers string) ([]keyValue, error) {
+	if len(headers) == 0 {
+		return nil, nil
+	}
+
+	var headersSplitByDelimiter = strings.Split(headers, "^^")
+
+	kvs := make([]keyValue, len(headersSplitByDelimiter))
+	for i, h := range headersSplitByDelimiter {
+		n := strings.IndexByte(h, ':')
+		if n < 0 {
+			return nil, fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, h)
+		}
+		kv := &kvs[i]
+		kv.key = strings.TrimSpace(h[:n])
+		kv.value = strings.TrimSpace(h[n+1:])
+	}
+	return kvs, nil
+}

app/vmctl/backoff/backoff.go

@@ -0,0 +1,61 @@
+package backoff
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	backoffRetries     = 5
+	backoffFactor      = 1.7
+	backoffMinDuration = time.Second
+)
+
+// retryableFunc describes call back which will repeat on errors
+type retryableFunc func() error
+
+// ErrBadRequest is an error returned on bad request
+var ErrBadRequest = errors.New("bad request")
+
+// Backoff describes object with backoff policy params
+type Backoff struct {
+	retries     int
+	factor      float64
+	minDuration time.Duration
+}
+
+// New initialize backoff object
+func New() *Backoff {
+	return &Backoff{
+		retries:     backoffRetries,
+		factor:      backoffFactor,
+		minDuration: backoffMinDuration,
+	}
+}
+
+// Retry process retries until all attempts are completed
+func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
+	var attempt uint64
+	for i := 0; i < b.retries; i++ {
+		// @TODO we should use context to cancel retries
+		err := cb()
+		if err == nil {
+			return attempt, nil
+		}
+		if errors.Is(err, ErrBadRequest) || errors.Is(err, context.Canceled) {
+			logger.Errorf("unrecoverable error: %s", err)
+			return attempt, err // fail fast if not recoverable
+		}
+		attempt++
+		backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
+		dur := time.Duration(backoff)
+		logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
+		time.Sleep(time.Duration(backoff))
+	}
+	return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
+}

app/vmctl/backoff/backoff_test.go

@@ -0,0 +1,96 @@
+package backoff
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestRetry_Do(t *testing.T) {
+	counter := 0
+	tests := []struct {
+		name               string
+		backoffRetries     int
+		backoffFactor      float64
+		backoffMinDuration time.Duration
+		retryableFunc      retryableFunc
+		ctx                context.Context
+		withCancel         bool
+		want               uint64
+		wantErr            bool
+	}{
+		{
+			name: "return bad request",
+			retryableFunc: func() error {
+				return ErrBadRequest
+			},
+			ctx:     context.Background(),
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name: "empty retries values",
+			retryableFunc: func() error {
+				time.Sleep(time.Millisecond * 100)
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:               "only one retry test",
+			backoffRetries:     5,
+			backoffFactor:      1.7,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					counter++
+					if counter%2 == 0 {
+						return fmt.Errorf("got some error")
+					}
+					if counter%3 == 0 {
+						return nil
+					}
+				}
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    1,
+			wantErr: false,
+		},
+		{
+			name:               "all retries failed test",
+			backoffRetries:     5,
+			backoffFactor:      0.1,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					return fmt.Errorf("got some error")
+				}
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    5,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := New()
+			got, err := r.Retry(tt.ctx, tt.retryableFunc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Retry() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

app/vmctl/flags.go

@@ -325,13 +325,19 @@ const (
 	vmNativeFilterTimeEnd   = "vm-native-filter-time-end"
 	vmNativeStepInterval    = "vm-native-step-interval"
 
-	vmNativeSrcAddr     = "vm-native-src-addr"
-	vmNativeSrcUser     = "vm-native-src-user"
-	vmNativeSrcPassword = "vm-native-src-password"
+	vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive"
 
-	vmNativeDstAddr     = "vm-native-dst-addr"
-	vmNativeDstUser     = "vm-native-dst-user"
-	vmNativeDstPassword = "vm-native-dst-password"
+	vmNativeSrcAddr        = "vm-native-src-addr"
+	vmNativeSrcUser        = "vm-native-src-user"
+	vmNativeSrcPassword    = "vm-native-src-password"
+	vmNativeSrcHeaders     = "vm-native-src-headers"
+	vmNativeSrcBearerToken = "vm-native-src-bearer-token"
+
+	vmNativeDstAddr        = "vm-native-dst-addr"
+	vmNativeDstUser        = "vm-native-dst-user"
+	vmNativeDstPassword    = "vm-native-dst-password"
+	vmNativeDstHeaders     = "vm-native-dst-headers"
+	vmNativeDstBearerToken = "vm-native-dst-bearer-token"
 )
 
 var (
@@ -344,8 +350,9 @@ var (
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{
-			Name:  vmNativeFilterTimeStart,
-			Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Name:     vmNativeFilterTimeStart,
+			Usage:    "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
 		},
 		&cli.StringFlag{
 			Name:  vmNativeFilterTimeEnd,
@@ -355,6 +362,11 @@ var (
 			Name:  vmNativeStepInterval,
 			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
 		},
+		&cli.BoolFlag{
+			Name:  vmNativeDisableHTTPKeepAlive,
+			Usage: "Disable HTTP persistent connections for requests made to VictoriaMetrics components during export",
+			Value: false,
+		},
 		&cli.StringFlag{
 			Name: vmNativeSrcAddr,
 			Usage: "VictoriaMetrics address to perform export from. \n" +
@@ -372,6 +384,16 @@ var (
 			Usage:   "VictoriaMetrics password for basic auth",
 			EnvVars: []string{"VM_NATIVE_SRC_PASSWORD"},
 		},
+		&cli.StringFlag{
+			Name: vmNativeSrcHeaders,
+			Usage: "Optional HTTP headers to send with each request to the corresponding source address. \n" +
+				"For example, --vm-native-src-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding source address. \n" +
+				"Multiple headers must be delimited by '^^': --vm-native-src-headers='header1:value1^^header2:value2'",
+		},
+		&cli.StringFlag{
+			Name:  vmNativeSrcBearerToken,
+			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-src-addr`",
+		},
 		&cli.StringFlag{
 			Name: vmNativeDstAddr,
 			Usage: "VictoriaMetrics address to perform import to. \n" +
@@ -389,6 +411,16 @@ var (
 			Usage:   "VictoriaMetrics password for basic auth",
 			EnvVars: []string{"VM_NATIVE_DST_PASSWORD"},
 		},
+		&cli.StringFlag{
+			Name: vmNativeDstHeaders,
+			Usage: "Optional HTTP headers to send with each request to the corresponding destination address. \n" +
+				"For example, --vm-native-dst-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding destination address. \n" +
+				"Multiple headers must be delimited by '^^': --vm-native-dst-headers='header1:value1^^header2:value2'",
+		},
+		&cli.StringFlag{
+			Name:  vmNativeDstBearerToken,
+			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-dst-addr`",
+		},
 		&cli.StringSliceFlag{
 			Name:  vmExtraLabel,
 			Value: nil,
@@ -406,6 +438,11 @@ var (
 				fmt.Sprintf(" In this mode --%s flag format is: 'http://vmselect:8481/'. --%s flag format is: http://vminsert:8480/. \n", vmNativeSrcAddr, vmNativeDstAddr) +
 				" TenantID will be appended automatically after discovering tenants from src.",
 		},
+		&cli.UintFlag{
+			Name:  vmConcurrency,
+			Usage: "Number of workers concurrently performing import requests to VM",
+			Value: 2,
+		},
 	}
 )
 
@@ -485,7 +522,7 @@ var (
 		},
 		&cli.DurationFlag{
 			Name:  remoteReadHTTPTimeout,
-			Usage: "Timeout defines timeout for HTTP write request to remote storage",
+			Usage: "Timeout defines timeout for HTTP requests made by remote read client",
 		},
 		&cli.StringFlag{
 			Name:  remoteReadHeaders,

app/vmctl/influx/influx.go

@@ -162,7 +162,8 @@ func (c *Client) Explore() ([]*Series, error) {
 	for _, s := range series {
 		fields, ok := mFields[s.Measurement]
 		if !ok {
-			return nil, fmt.Errorf("can't find field keys for measurement %q", s.Measurement)
+			log.Printf("skip measurement %q since it has no fields", s.Measurement)
+			continue
 		}
 		for _, field := range fields {
 			is := &Series{

app/vmctl/main.go

@@ -11,6 +11,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/urfave/cli/v2"
 
@@ -20,7 +23,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 )
 
 func main() {
@@ -35,9 +38,6 @@ func main() {
 		Name:    "vmctl",
 		Usage:   "VictoriaMetrics command-line tool",
 		Version: buildinfo.Version,
-		// Disable `-version` flag to avoid conflict with lib/buildinfo flags
-		// see https://github.com/urfave/cli/issues/1560
-		HideVersion: true,
 		Commands: []*cli.Command{
 			{
 				Name:  "opentsdb",
@@ -65,7 +65,7 @@ func main() {
 					// disable progress bars since openTSDB implementation
 					// does not use progress bar pool
 					vmCfg.DisableProgressBar = true
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -100,7 +100,7 @@ func main() {
 					}
 
 					vmCfg := initConfigVM(c)
-					importer, err = vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -137,7 +137,7 @@ func main() {
 
 					vmCfg := initConfigVM(c)
 
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -163,7 +163,7 @@ func main() {
 					fmt.Println("Prometheus import mode")
 
 					vmCfg := initConfigVM(c)
-					importer, err = vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -192,7 +192,7 @@ func main() {
 			{
 				Name:  "vm-native",
 				Usage: "Migrate time series between VictoriaMetrics installations via native binary format",
-				Flags: vmNativeFlags,
+				Flags: mergeFlags(globalFlags, vmNativeFlags),
 				Action: func(c *cli.Context) error {
 					fmt.Println("VictoriaMetrics Native import mode")
 
@@ -200,28 +200,51 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
+					var srcExtraLabels []string
+					srcAddr := strings.Trim(c.String(vmNativeSrcAddr), "/")
+					srcAuthConfig, err := auth.Generate(
+						auth.WithBasicAuth(c.String(vmNativeSrcUser), c.String(vmNativeSrcPassword)),
+						auth.WithBearer(c.String(vmNativeSrcBearerToken)),
+						auth.WithHeaders(c.String(vmNativeSrcHeaders)))
+					if err != nil {
+						return fmt.Errorf("error initilize auth config for source: %s", srcAddr)
+					}
+
+					dstAddr := strings.Trim(c.String(vmNativeDstAddr), "/")
+					dstExtraLabels := c.StringSlice(vmExtraLabel)
+					dstAuthConfig, err := auth.Generate(
+						auth.WithBasicAuth(c.String(vmNativeDstUser), c.String(vmNativeDstPassword)),
+						auth.WithBearer(c.String(vmNativeDstBearerToken)),
+						auth.WithHeaders(c.String(vmNativeDstHeaders)))
+					if err != nil {
+						return fmt.Errorf("error initilize auth config for destination: %s", dstAddr)
+					}
+
 					p := vmNativeProcessor{
 						rateLimit:    c.Int64(vmRateLimit),
 						interCluster: c.Bool(vmInterCluster),
-						filter: filter{
-							match:     c.String(vmNativeFilterMatch),
-							timeStart: c.String(vmNativeFilterTimeStart),
-							timeEnd:   c.String(vmNativeFilterTimeEnd),
-							chunk:     c.String(vmNativeStepInterval),
+						filter: native.Filter{
+							Match:     c.String(vmNativeFilterMatch),
+							TimeStart: c.String(vmNativeFilterTimeStart),
+							TimeEnd:   c.String(vmNativeFilterTimeEnd),
+							Chunk:     c.String(vmNativeStepInterval),
 						},
-						src: &vmNativeClient{
-							addr:     strings.Trim(c.String(vmNativeSrcAddr), "/"),
-							user:     c.String(vmNativeSrcUser),
-							password: c.String(vmNativeSrcPassword),
+						src: &native.Client{
+							AuthCfg:              srcAuthConfig,
+							Addr:                 srcAddr,
+							ExtraLabels:          srcExtraLabels,
+							DisableHTTPKeepAlive: c.Bool(vmNativeDisableHTTPKeepAlive),
 						},
-						dst: &vmNativeClient{
-							addr:        strings.Trim(c.String(vmNativeDstAddr), "/"),
-							user:        c.String(vmNativeDstUser),
-							password:    c.String(vmNativeDstPassword),
-							extraLabels: c.StringSlice(vmExtraLabel),
+						dst: &native.Client{
+							AuthCfg:              dstAuthConfig,
+							Addr:                 dstAddr,
+							ExtraLabels:          dstExtraLabels,
+							DisableHTTPKeepAlive: c.Bool(vmNativeDisableHTTPKeepAlive),
 						},
+						backoff: backoff.New(),
+						cc:      c.Int(vmConcurrency),
 					}
-					return p.run(ctx)
+					return p.run(ctx, c.Bool(globalSilent))
 				},
 			},
 			{
@@ -247,7 +270,7 @@ func main() {
 						return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1)
 					}
 					var blocksCount uint64
-					if err := parser.ParseStream(f, isBlockGzipped, func(block *parser.Block) error {
+					if err := stream.Parse(f, isBlockGzipped, func(block *stream.Block) error {
 						atomic.AddUint64(&blocksCount, 1)
 						return nil
 					}); err != nil {

app/vmctl/native/client.go

@@ -0,0 +1,186 @@
+package native
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
+)
+
+const (
+	nativeTenantsAddr = "admin/tenants"
+	nativeSeriesAddr  = "api/v1/series"
+	nameLabel         = "__name__"
+)
+
+// Client is an HTTP client for exporting and importing
+// time series via native protocol.
+type Client struct {
+	AuthCfg              *auth.Config
+	Addr                 string
+	ExtraLabels          []string
+	DisableHTTPKeepAlive bool
+}
+
+// LabelValues represents series from api/v1/series response
+type LabelValues map[string]string
+
+// Response represents response from api/v1/series
+type Response struct {
+	Status string        `json:"status"`
+	Series []LabelValues `json:"data"`
+}
+
+// Explore finds series by provided filter from api/v1/series
+func (c *Client) Explore(ctx context.Context, f Filter, tenantID string) (map[string]struct{}, error) {
+	url := fmt.Sprintf("%s/%s", c.Addr, nativeSeriesAddr)
+	if tenantID != "" {
+		url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeSeriesAddr)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
+	}
+
+	params := req.URL.Query()
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	params.Set("match[]", f.Match)
+	req.URL.RawQuery = params.Encode()
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("series request failed: %s", err)
+	}
+
+	var response Response
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return nil, fmt.Errorf("cannot decode series response: %s", err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, fmt.Errorf("cannot close series response body: %s", err)
+	}
+	names := make(map[string]struct{})
+	for _, series := range response.Series {
+		// TODO: consider tweaking /api/v1/series API to return metric names only
+		// this could make explore response much lighter.
+		for key, value := range series {
+			if key != nameLabel {
+				continue
+			}
+			if _, ok := names[value]; ok {
+				continue
+			}
+			names[value] = struct{}{}
+		}
+	}
+	return names, nil
+}
+
+// ImportPipe uses pipe reader in request to process data
+func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
+	if err != nil {
+		return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
+	}
+
+	importResp, err := c.do(req, http.StatusNoContent)
+	if err != nil {
+		return fmt.Errorf("import request failed: %s", err)
+	}
+	if err := importResp.Body.Close(); err != nil {
+		return fmt.Errorf("cannot close import response body: %s", err)
+	}
+	return nil
+}
+
+// ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
+func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
+	}
+
+	params := req.URL.Query()
+	params.Set("match[]", f.Match)
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	req.URL.RawQuery = params.Encode()
+
+	// disable compression since it is meaningless for native format
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("export request failed: %w", err)
+	}
+	return resp.Body, nil
+}
+
+// GetSourceTenants discovers tenants by provided filter
+func (c *Client) GetSourceTenants(ctx context.Context, f Filter) ([]string, error) {
+	u := fmt.Sprintf("%s/%s", c.Addr, nativeTenantsAddr)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", u, err)
+	}
+
+	params := req.URL.Query()
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	req.URL.RawQuery = params.Encode()
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("tenants request failed: %s", err)
+	}
+
+	var r struct {
+		Tenants []string `json:"data"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		return nil, fmt.Errorf("cannot decode tenants response: %s", err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, fmt.Errorf("cannot close tenants response body: %s", err)
+	}
+
+	return r.Tenants, nil
+}
+
+func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
+	if c.AuthCfg != nil {
+		c.AuthCfg.SetHeaders(req, true)
+	}
+	var httpClient = &http.Client{Transport: &http.Transport{DisableKeepAlives: c.DisableHTTPKeepAlive}}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error when performing request: %w", err)
+	}
+
+	if resp.StatusCode != expSC {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body for status code %d: %s", resp.StatusCode, err)
+		}
+		return nil, fmt.Errorf("unexpected response code %d: %s", resp.StatusCode, string(body))
+	}
+	return resp, err
+}

app/vmctl/native/filter.go

@@ -0,0 +1,22 @@
+package native
+
+import "fmt"
+
+// Filter represents request filter
+type Filter struct {
+	Match     string
+	TimeStart string
+	TimeEnd   string
+	Chunk     string
+}
+
+func (f Filter) String() string {
+	s := fmt.Sprintf("\n\tfilter: match[]=%s", f.Match)
+	if f.TimeStart != "" {
+		s += fmt.Sprintf("\n\tstart: %s", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		s += fmt.Sprintf("\n\tend: %s", f.TimeEnd)
+	}
+	return s
+}

app/vmctl/prometheus_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"testing"
 	"time"
@@ -58,7 +59,7 @@ func Test_prometheusProcessor_run(t *testing.T) {
 					return client
 				},
 				im: func(vmCfg vm.Config) *vm.Importer {
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
 					if err != nil {
 						t.Fatalf("error init importer: %s", err)
 					}
@@ -95,7 +96,7 @@ func Test_prometheusProcessor_run(t *testing.T) {
 					return client
 				},
 				im: func(vmCfg vm.Config) *vm.Importer {
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
 					if err != nil {
 						t.Fatalf("error init importer: %s", err)
 					}

app/vmctl/remote_read_test.go

@@ -110,6 +110,7 @@ func TestRemoteRead(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
 			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
 			defer remoteReadServer.Close()
 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
@@ -139,7 +140,7 @@ func TestRemoteRead(t *testing.T) {
 
 			tt.vmCfg.Addr = remoteWriteServer.URL()
 
-			importer, err := vm.NewImporter(tt.vmCfg)
+			importer, err := vm.NewImporter(ctx, tt.vmCfg)
 			if err != nil {
 				t.Fatalf("failed to create VM importer: %s", err)
 			}
@@ -156,7 +157,6 @@ func TestRemoteRead(t *testing.T) {
 				cc: 1,
 			}
 
-			ctx := context.Background()
 			err = rmp.run(ctx, true, false)
 			if err != nil {
 				t.Fatalf("failed to run remote read processor: %s", err)
@@ -263,6 +263,7 @@ func TestSteamRemoteRead(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
 			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
 			defer remoteReadServer.Close()
 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
@@ -292,7 +293,7 @@ func TestSteamRemoteRead(t *testing.T) {
 
 			tt.vmCfg.Addr = remoteWriteServer.URL()
 
-			importer, err := vm.NewImporter(tt.vmCfg)
+			importer, err := vm.NewImporter(ctx, tt.vmCfg)
 			if err != nil {
 				t.Fatalf("failed to create VM importer: %s", err)
 			}
@@ -309,7 +310,6 @@ func TestSteamRemoteRead(t *testing.T) {
 				cc: 1,
 			}
 
-			ctx := context.Background()
 			err = rmp.run(ctx, true, false)
 			if err != nil {
 				t.Fatalf("failed to run remote read processor: %s", err)

app/vmctl/remoteread/remoteread.go

@@ -21,7 +21,7 @@ import (
 )
 
 const (
-	defaultReadTimeout = 30 * time.Second
+	defaultReadTimeout = 5 * time.Minute
 	remoteReadPath     = "/api/v1/read"
 	healthPath         = "/-/healthy"
 )
@@ -157,7 +157,7 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 // Ping checks the health of the read source
 func (c *Client) Ping() error {
 	url := c.addr + healthPath
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", url, err)
 	}
@@ -174,7 +174,7 @@ func (c *Client) Ping() error {
 func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback) error {
 	r := bytes.NewReader(data)
 	url := c.addr + remoteReadPath
-	req, err := http.NewRequest("POST", url, r)
+	req, err := http.NewRequest(http.MethodPost, url, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}

app/vmctl/vm/vm.go

@@ -3,15 +3,16 @@ package vm
 import (
 	"bufio"
 	"compress/gzip"
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
@@ -75,7 +76,8 @@ type Importer struct {
 	wg   sync.WaitGroup
 	once sync.Once
 
-	s *stats
+	s       *stats
+	backoff *backoff.Backoff
 }
 
 // ResetStats resets im stats.
@@ -107,7 +109,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 }
 
 // NewImporter creates new Importer for the given cfg.
-func NewImporter(cfg Config) (*Importer, error) {
+func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 	if cfg.Concurrency < 1 {
 		return nil, fmt.Errorf("concurrency can't be lower than 1")
 	}
@@ -136,6 +138,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
+		backoff:    backoff.New(),
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
@@ -154,7 +157,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		}
 		go func(bar *pb.ProgressBar) {
 			defer im.wg.Done()
-			im.startWorker(bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
+			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
 		}(bar)
 	}
 	im.ResetStats()
@@ -205,7 +208,7 @@ func (im *Importer) Close() {
 	})
 }
 
-func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigures, roundDigits int) {
+func (im *Importer) startWorker(ctx context.Context, bar *pb.ProgressBar, batchSize, significantFigures, roundDigits int) {
 	var batch []*TimeSeries
 	var dataPoints int
 	var waitForBatch time.Time
@@ -219,7 +222,9 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 			exitErr := &ImportError{
 				Batch: batch,
 			}
-			if err := im.Import(batch); err != nil {
+			retryableFunc := func() error { return im.Import(batch) }
+			_, err := im.backoff.Retry(ctx, retryableFunc)
+			if err != nil {
 				exitErr.Err = err
 			}
 			im.errors <- exitErr
@@ -249,7 +254,7 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 			im.s.idleDuration += time.Since(waitForBatch)
 			im.s.Unlock()
 
-			if err := im.flush(batch); err != nil {
+			if err := im.flush(ctx, batch); err != nil {
 				im.errors <- &ImportError{
 					Batch: batch,
 					Err:   err,
@@ -264,36 +269,22 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 	}
 }
 
-const (
-	// TODO: make configurable
-	backoffRetries     = 5
-	backoffFactor      = 1.7
-	backoffMinDuration = time.Second
-)
-
-func (im *Importer) flush(b []*TimeSeries) error {
-	var err error
-	for i := 0; i < backoffRetries; i++ {
-		err = im.Import(b)
-		if err == nil {
-			return nil
-		}
-		if errors.Is(err, ErrBadRequest) {
-			return err // fail fast if not recoverable
-		}
-		im.s.Lock()
-		im.s.retries++
-		im.s.Unlock()
-		backoff := float64(backoffMinDuration) * math.Pow(backoffFactor, float64(i))
-		time.Sleep(time.Duration(backoff))
+func (im *Importer) flush(ctx context.Context, b []*TimeSeries) error {
+	retryableFunc := func() error { return im.Import(b) }
+	attempts, err := im.backoff.Retry(ctx, retryableFunc)
+	if err != nil {
+		return fmt.Errorf("import failed with %d retries: %s", attempts, err)
 	}
-	return fmt.Errorf("import failed with %d retries: %s", backoffRetries, err)
+	im.s.Lock()
+	im.s.retries = attempts
+	im.s.Unlock()
+	return nil
 }
 
 // Ping sends a ping to im.addr.
 func (im *Importer) Ping() error {
 	url := fmt.Sprintf("%s/health", im.addr)
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}
@@ -317,7 +308,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	}
 
 	pr, pw := io.Pipe()
-	req, err := http.NewRequest("POST", im.importPath, pr)
+	req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}

app/vmctl/vm_native.go

@@ -2,177 +2,125 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log"
-	"net/http"
+	"sync"
 	"time"
 
-	"github.com/cheggaaa/pb/v3"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/cheggaaa/pb/v3"
 )
 
 type vmNativeProcessor struct {
-	filter    filter
-	rateLimit int64
+	filter native.Filter
 
-	dst          *vmNativeClient
-	src          *vmNativeClient
+	dst     *native.Client
+	src     *native.Client
+	backoff *backoff.Backoff
+
+	s            *stats
+	rateLimit    int64
 	interCluster bool
-}
-
-type vmNativeClient struct {
-	addr        string
-	user        string
-	password    string
-	extraLabels []string
-}
-
-type filter struct {
-	match     string
-	timeStart string
-	timeEnd   string
-	chunk     string
-}
-
-func (f filter) String() string {
-	s := fmt.Sprintf("\n\tfilter: match[]=%s", f.match)
-	if f.timeStart != "" {
-		s += fmt.Sprintf("\n\tstart: %s", f.timeStart)
-	}
-	if f.timeEnd != "" {
-		s += fmt.Sprintf("\n\tend: %s", f.timeEnd)
-	}
-	return s
+	cc           int
 }
 
 const (
-	nativeExportAddr  = "api/v1/export/native"
-	nativeImportAddr  = "api/v1/import/native"
-	nativeTenantsAddr = "admin/tenants"
-
-	nativeBarTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}`
+	nativeExportAddr = "api/v1/export/native"
+	nativeImportAddr = "api/v1/import/native"
+	nativeBarTpl     = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "█") "▒" "]" }} {{ percent . }}`
 )
 
-func (p *vmNativeProcessor) run(ctx context.Context) error {
-	if p.filter.chunk == "" {
-		return p.runWithFilter(ctx, p.filter)
+func (p *vmNativeProcessor) run(ctx context.Context, silent bool) error {
+	if p.cc == 0 {
+		p.cc = 1
+	}
+	p.s = &stats{
+		startTime: time.Now(),
 	}
 
-	startOfRange, err := time.Parse(time.RFC3339, p.filter.timeStart)
+	start, err := time.Parse(time.RFC3339, p.filter.TimeStart)
 	if err != nil {
-		return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeStart, p.filter.timeStart, time.RFC3339, err)
+		return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+			vmNativeFilterTimeStart, p.filter.TimeStart, time.RFC3339, err)
 	}
 
-	var endOfRange time.Time
-	if p.filter.timeEnd != "" {
-		endOfRange, err = time.Parse(time.RFC3339, p.filter.timeEnd)
+	end := time.Now().In(start.Location())
+	if p.filter.TimeEnd != "" {
+		end, err = time.Parse(time.RFC3339, p.filter.TimeEnd)
 		if err != nil {
-			return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeEnd, p.filter.timeEnd, time.RFC3339, err)
+			return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+				vmNativeFilterTimeEnd, p.filter.TimeEnd, time.RFC3339, err)
 		}
-	} else {
-		endOfRange = time.Now()
 	}
 
-	ranges, err := stepper.SplitDateRange(startOfRange, endOfRange, p.filter.chunk)
-	if err != nil {
-		return fmt.Errorf("failed to create date ranges for the given time filters: %v", err)
-	}
-
-	for rangeIdx, r := range ranges {
-		formattedStartTime := r[0].Format(time.RFC3339)
-		formattedEndTime := r[1].Format(time.RFC3339)
-		log.Printf("Processing range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
-		f := filter{
-			match:     p.filter.match,
-			timeStart: formattedStartTime,
-			timeEnd:   formattedEndTime,
-		}
-		err := p.runWithFilter(ctx, f)
-
+	ranges := [][]time.Time{{start, end}}
+	if p.filter.Chunk != "" {
+		ranges, err = stepper.SplitDateRange(start, end, p.filter.Chunk)
 		if err != nil {
-			log.Printf("processing failed for range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
-			return err
+			return fmt.Errorf("failed to create date ranges for the given time filters: %w", err)
 		}
 	}
+
+	tenants := []string{""}
+	if p.interCluster {
+		log.Printf("Discovering tenants...")
+		tenants, err = p.src.GetSourceTenants(ctx, p.filter)
+		if err != nil {
+			return fmt.Errorf("failed to get tenants: %w", err)
+		}
+		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
+		if !silent && !prompt(question) {
+			return nil
+		}
+	}
+
+	for _, tenantID := range tenants {
+		err := p.runBackfilling(ctx, tenantID, ranges, silent)
+		if err != nil {
+			return fmt.Errorf("migration failed: %s", err)
+		}
+	}
+
+	log.Println("Import finished!")
+	log.Print(p.s)
+
 	return nil
 }
 
-func (p *vmNativeProcessor) runWithFilter(ctx context.Context, f filter) error {
-	nativeImportAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.extraLabels)
+func (p *vmNativeProcessor) do(ctx context.Context, f native.Filter, srcURL, dstURL string) error {
 
+	retryableFunc := func() error { return p.runSingle(ctx, f, srcURL, dstURL) }
+	attempts, err := p.backoff.Retry(ctx, retryableFunc)
+	p.s.Lock()
+	p.s.retries += attempts
+	p.s.Unlock()
 	if err != nil {
-		return fmt.Errorf("failed to add labels to import path: %s", err)
-	}
-
-	if !p.interCluster {
-		srcURL := fmt.Sprintf("%s/%s", p.src.addr, nativeExportAddr)
-		dstURL := fmt.Sprintf("%s/%s", p.dst.addr, nativeImportAddr)
-
-		return p.runSingle(ctx, f, srcURL, dstURL)
-	}
-
-	tenants, err := p.getSourceTenants(ctx, f)
-	if err != nil {
-		return fmt.Errorf("failed to get source tenants: %s", err)
-	}
-
-	log.Printf("Discovered tenants: %v", tenants)
-	for _, tenant := range tenants {
-		// src and dst expected formats: http://vminsert:8480/ and http://vmselect:8481/
-		srcURL := fmt.Sprintf("%s/select/%s/prometheus/%s", p.src.addr, tenant, nativeExportAddr)
-		dstURL := fmt.Sprintf("%s/insert/%s/prometheus/%s", p.dst.addr, tenant, nativeImportAddr)
-
-		if err := p.runSingle(ctx, f, srcURL, dstURL); err != nil {
-			return fmt.Errorf("failed to migrate data for tenant %q: %s", tenant, err)
-		}
+		return fmt.Errorf("failed to migrate from %s to %s (retry attempts: %d): %w\nwith fileter %s", srcURL, dstURL, attempts, err, f)
 	}
 
 	return nil
 }
 
-func (p *vmNativeProcessor) runSingle(ctx context.Context, f filter, srcURL, dstURL string) error {
-	log.Printf("Initing export pipe from %q with filters: %s\n", srcURL, f)
+func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcURL, dstURL string) error {
 
-	exportReader, err := p.exportPipe(ctx, srcURL, f)
+	exportReader, err := p.src.ExportPipe(ctx, srcURL, f)
 	if err != nil {
-		return fmt.Errorf("failed to init export pipe: %s", err)
+		return fmt.Errorf("failed to init export pipe: %w", err)
 	}
 
 	pr, pw := io.Pipe()
-	sync := make(chan struct{})
+	done := make(chan struct{})
 	go func() {
-		defer func() { close(sync) }()
-		req, err := http.NewRequestWithContext(ctx, "POST", dstURL, pr)
-		if err != nil {
-			log.Fatalf("cannot create import request to %q: %s", p.dst.addr, err)
-		}
-		importResp, err := p.dst.do(req, http.StatusNoContent)
-		if err != nil {
-			log.Fatalf("import request failed: %s", err)
-		}
-		if err := importResp.Body.Close(); err != nil {
-			log.Fatalf("cannot close import response body: %s", err)
-		}
-	}()
-
-	fmt.Printf("Initing import process to %q:\n", dstURL)
-	pool := pb.NewPool()
-	bar := pb.ProgressBarTemplate(nativeBarTpl).New(0)
-	pool.Add(bar)
-	barReader := bar.NewProxyReader(exportReader)
-	if err := pool.Start(); err != nil {
-		log.Printf("error start process bars pool: %s", err)
-		return err
-	}
-	defer func() {
-		bar.Finish()
-		if err := pool.Stop(); err != nil {
-			fmt.Printf("failed to stop barpool: %+v\n", err)
+		defer func() { close(done) }()
+		if err := p.dst.ImportPipe(ctx, dstURL, pr); err != nil {
+			logger.Errorf("error initialize import pipe: %s", err)
+			return
 		}
 	}()
 
@@ -182,95 +130,176 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f filter, srcURL, dst
 		w = limiter.NewWriteLimiter(pw, rl)
 	}
 
-	_, err = io.Copy(w, barReader)
+	written, err := io.Copy(w, exportReader)
 	if err != nil {
-		return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
+		return fmt.Errorf("failed to write into %q: %s", p.dst.Addr, err)
 	}
 
+	p.s.Lock()
+	p.s.bytes += uint64(written)
+	p.s.requests++
+	p.s.Unlock()
+
 	if err := pw.Close(); err != nil {
 		return err
 	}
-	<-sync
+	<-done
 
-	log.Println("Import finished!")
 	return nil
 }
 
-func (p *vmNativeProcessor) getSourceTenants(ctx context.Context, f filter) ([]string, error) {
-	u := fmt.Sprintf("%s/%s", p.src.addr, nativeTenantsAddr)
-	req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
+func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error {
+	exportAddr := nativeExportAddr
+	srcURL := fmt.Sprintf("%s/%s", p.src.Addr, exportAddr)
+
+	importAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.ExtraLabels)
 	if err != nil {
-		return nil, fmt.Errorf("cannot create request to %q: %s", u, err)
+		return fmt.Errorf("failed to add labels to import path: %s", err)
+	}
+	dstURL := fmt.Sprintf("%s/%s", p.dst.Addr, importAddr)
+
+	if p.interCluster {
+		srcURL = fmt.Sprintf("%s/select/%s/prometheus/%s", p.src.Addr, tenantID, exportAddr)
+		dstURL = fmt.Sprintf("%s/insert/%s/prometheus/%s", p.dst.Addr, tenantID, importAddr)
 	}
 
-	params := req.URL.Query()
-	if f.timeStart != "" {
-		params.Set("start", f.timeStart)
+	barPrefix := "Requests to make"
+	initMessage := "Initing import process from %q to %q with filter %s"
+	initParams := []interface{}{srcURL, dstURL, p.filter.String()}
+	if p.interCluster {
+		barPrefix = fmt.Sprintf("Requests to make for tenant %s", tenantID)
+		initMessage = "Initing import process from %q to %q with filter %s for tenant %s"
+		initParams = []interface{}{srcURL, dstURL, p.filter.String(), tenantID}
 	}
-	if f.timeEnd != "" {
-		params.Set("end", f.timeEnd)
-	}
-	req.URL.RawQuery = params.Encode()
 
-	resp, err := p.src.do(req, http.StatusOK)
+	fmt.Println("") // extra line for better output formatting
+	log.Printf(initMessage, initParams...)
+
+	log.Printf("Exploring metrics...")
+	metrics, err := p.src.Explore(ctx, p.filter, tenantID)
 	if err != nil {
-		return nil, fmt.Errorf("tenants request failed: %s", err)
+		return fmt.Errorf("cannot get metrics from source %s: %w", p.src.Addr, err)
 	}
 
-	var r struct {
-		Tenants []string `json:"data"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
-		return nil, fmt.Errorf("cannot decode tenants response: %s", err)
+	if len(metrics) == 0 {
+		return fmt.Errorf("no metrics found")
 	}
 
-	if err := resp.Body.Close(); err != nil {
-		return nil, fmt.Errorf("cannot close tenants response body: %s", err)
-	}
-
-	return r.Tenants, nil
-}
-
-func (p *vmNativeProcessor) exportPipe(ctx context.Context, url string, f filter) (io.ReadCloser, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create request to %q: %s", p.src.addr, err)
-	}
-
-	params := req.URL.Query()
-	params.Set("match[]", f.match)
-	if f.timeStart != "" {
-		params.Set("start", f.timeStart)
-	}
-	if f.timeEnd != "" {
-		params.Set("end", f.timeEnd)
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// disable compression since it is meaningless for native format
-	req.Header.Set("Accept-Encoding", "identity")
-	resp, err := p.src.do(req, http.StatusOK)
-	if err != nil {
-		return nil, fmt.Errorf("export request failed: %s", err)
-	}
-	return resp.Body, nil
-}
-
-func (c *vmNativeClient) do(req *http.Request, expSC int) (*http.Response, error) {
-	if c.user != "" {
-		req.SetBasicAuth(c.user, c.password)
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error when performing request: %s", err)
-	}
-
-	if resp.StatusCode != expSC {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read response body for status code %d: %s", resp.StatusCode, err)
+	foundSeriesMsg := fmt.Sprintf("Found %d metrics to import", len(metrics))
+	if !p.interCluster {
+		// do not prompt for intercluster because there could be many tenants,
+		// and we don't want to interrupt the process when moving to the next tenant.
+		question := foundSeriesMsg + ". Continue?"
+		if !silent && !prompt(question) {
+			return nil
 		}
-		return nil, fmt.Errorf("unexpected response code %d: %s", resp.StatusCode, string(body))
+	} else {
+		log.Print(foundSeriesMsg)
 	}
-	return resp, err
+
+	processingMsg := fmt.Sprintf("Requests to make: %d", len(metrics)*len(ranges))
+	if len(ranges) > 1 {
+		processingMsg = fmt.Sprintf("Selected time range will be split into %d ranges according to %q step. %s", len(ranges), p.filter.Chunk, processingMsg)
+	}
+	log.Print(processingMsg)
+
+	var bar *pb.ProgressBar
+	if !silent {
+		bar = pb.ProgressBarTemplate(fmt.Sprintf(nativeBarTpl, barPrefix)).New(len(metrics) * len(ranges))
+		bar.Start()
+		defer bar.Finish()
+	}
+
+	filterCh := make(chan native.Filter)
+	errCh := make(chan error, p.cc)
+
+	var wg sync.WaitGroup
+	for i := 0; i < p.cc; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for f := range filterCh {
+				if err := p.do(ctx, f, srcURL, dstURL); err != nil {
+					errCh <- err
+					return
+				}
+				if bar != nil {
+					bar.Increment()
+				}
+			}
+		}()
+	}
+
+	// any error breaks the import
+	for s := range metrics {
+		for _, times := range ranges {
+			select {
+			case <-ctx.Done():
+				return fmt.Errorf("context canceled")
+			case infErr := <-errCh:
+				return fmt.Errorf("native error: %s", infErr)
+			case filterCh <- native.Filter{
+				Match:     fmt.Sprintf("{%s=%q}", nameLabel, s),
+				TimeStart: times[0].Format(time.RFC3339),
+				TimeEnd:   times[1].Format(time.RFC3339),
+			}:
+			}
+		}
+	}
+
+	close(filterCh)
+	wg.Wait()
+	close(errCh)
+
+	for err := range errCh {
+		return fmt.Errorf("import process failed: %s", err)
+	}
+
+	return nil
+}
+
+// stats represents client statistic
+// when processing data
+type stats struct {
+	sync.Mutex
+	startTime time.Time
+	bytes     uint64
+	requests  uint64
+	retries   uint64
+}
+
+func (s *stats) String() string {
+	s.Lock()
+	defer s.Unlock()
+
+	totalImportDuration := time.Since(s.startTime)
+	totalImportDurationS := totalImportDuration.Seconds()
+	bytesPerS := byteCountSI(0)
+	if s.bytes > 0 && totalImportDurationS > 0 {
+		bytesPerS = byteCountSI(int64(float64(s.bytes) / totalImportDurationS))
+	}
+
+	return fmt.Sprintf("VictoriaMetrics importer stats:\n"+
+		"  time spent while importing: %v;\n"+
+		"  total bytes: %s;\n"+
+		"  bytes/s: %s;\n"+
+		"  requests: %d;\n"+
+		"  requests retries: %d;",
+		totalImportDuration,
+		byteCountSI(int64(s.bytes)), bytesPerS,
+		s.requests, s.retries)
+}
+
+func byteCountSI(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB",
+		float64(b)/float64(div), "kMGTPE"[exp])
 }

app/vmctl/vm_native_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
 )
 
@@ -27,10 +28,10 @@ const (
 func Test_vmNativeProcessor_run(t *testing.T) {
 	t.Skip()
 	type fields struct {
-		filter    filter
+		filter    native.Filter
 		rateLimit int64
-		dst       *vmNativeClient
-		src       *vmNativeClient
+		dst       *native.Client
+		src       *native.Client
 	}
 	tests := []struct {
 		name    string
@@ -41,16 +42,16 @@ func Test_vmNativeProcessor_run(t *testing.T) {
 		{
 			name: "simulate syscall.SIGINT",
 			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
+				filter: native.Filter{
+					Match:     matchFilter,
+					TimeStart: timeStartFilter,
 				},
 				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
+				dst: &native.Client{
+					Addr: dstAddr,
 				},
-				src: &vmNativeClient{
-					addr: srcAddr,
+				src: &native.Client{
+					Addr: srcAddr,
 				},
 			},
 			closer: func(cancelFunc context.CancelFunc) {
@@ -62,16 +63,16 @@ func Test_vmNativeProcessor_run(t *testing.T) {
 		{
 			name: "simulate correct work",
 			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
+				filter: native.Filter{
+					Match:     matchFilter,
+					TimeStart: timeStartFilter,
 				},
 				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
+				dst: &native.Client{
+					Addr: dstAddr,
 				},
-				src: &vmNativeClient{
-					addr: srcAddr,
+				src: &native.Client{
+					Addr: srcAddr,
 				},
 			},
 			closer:  func(cancelFunc context.CancelFunc) {},
@@ -80,18 +81,18 @@ func Test_vmNativeProcessor_run(t *testing.T) {
 		{
 			name: "simulate correct work with chunking",
 			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
-					timeEnd:   timeEndFilter,
-					chunk:     stepper.StepMonth,
+				filter: native.Filter{
+					Match:     matchFilter,
+					TimeStart: timeStartFilter,
+					TimeEnd:   timeEndFilter,
+					Chunk:     stepper.StepMonth,
 				},
 				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
+				dst: &native.Client{
+					Addr: dstAddr,
 				},
-				src: &vmNativeClient{
-					addr: srcAddr,
+				src: &native.Client{
+					Addr: srcAddr,
 				},
 			},
 			closer:  func(cancelFunc context.CancelFunc) {},
@@ -110,7 +111,7 @@ func Test_vmNativeProcessor_run(t *testing.T) {
 
 			tt.closer(cancelFn)
 
-			if err := p.run(ctx); (err != nil) != tt.wantErr {
+			if err := p.run(ctx, true); (err != nil) != tt.wantErr {
 				t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})

app/vmgateway/README.md

@@ -206,6 +206,56 @@ mwIDAQAB
 ```
 This command will result in 3 keys loaded: 2 keys from files and 1 from command line.
 
+### Using OpenID discovery endpoint for JWT signature verification
+
+`vmgateway` supports using OpenID discovery endpoint for JWKS keys discovery.
+
+In order to enable [OpenID discovery](https://openid.net/specs/openid-connect-discovery-1_0.html) endpoint for JWT signature verification, you need to specify OpenID discovery endpoint URLs by using `auth.oidcDiscoveryEndpoints` flag.
+When `auth.oidcDiscoveryEndpoints` is specified `vmageteway` will fetch JWKS keys from the specified endpoint and use them for JWT signature verification.
+
+Example usage for tokens issued by Azure Active Directory:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.oidcDiscoveryEndpoints=https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration
+```
+
+Example usage for tokens issued by Google:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.oidcDiscoveryEndpoints=https://accounts.google.com/.well-known/openid-configuration
+```
+
+### Using JWKS endpoint for JWT signature verification
+
+`vmgateway` supports using JWKS endpoint for JWT signature verification.
+
+In order to enable JWKS endpoint for JWT signature verification, you need to specify JWKS endpoint URL by using `auth.jwksEndpoints` flag.
+When `auth.jwksEndpoints` is specified `vmageteway` will fetch public keys from the specified endpoint and use them for JWT signature verification.
+
+Example usage for tokens issued by Azure Active Directory:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.jwksEndpoints=https://login.microsoftonline.com/common/discovery/v2.0/keys
+```
+
+Example usage for tokens issued by Google:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.jwksEndpoints=https://www.googleapis.com/oauth2/v3/certs
+```
+
 ## Configuration
 
 The shortlist of configuration flags include the following:
@@ -213,6 +263,12 @@ The shortlist of configuration flags include the following:
 ```console
   -auth.httpHeader string
      HTTP header name to look for JWT authorization token (default "Authorization")
+  -auth.jwksEndpoints array
+     JWKS endpoints to fetch keys for JWT tokens signature verification
+     Supports an array of values separated by comma or specified via multiple flags.
+  -auth.oidcDiscoveryEndpoints array
+     OpenID Connect discovery endpoints to fetch keys for JWT tokens signature verification
+     Supports an array of values separated by comma or specified via multiple flags.
   -auth.publicKeyFiles array
      Path file with public key to verify JWT token signature
      Supports an array of values separated by comma or specified via multiple flags.

app/vminsert/common/insert_ctx.go

@@ -146,7 +146,7 @@ func (ctx *InsertCtx) FlushBufs() error {
 	}
 	// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
 	// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
-	// used at every ParseStream() call under lib/protoparser/*/streamparser.go
+	// used at every stream.Parse() call under lib/protoparser/*
 	err := vmstorage.AddRows(ctx.mrs)
 	ctx.Reset(0)
 	if err == nil {

app/vminsert/csvimport/request_handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -22,7 +23,7 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	})
 }

app/vminsert/datadog/request_handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -25,7 +26,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(series, extraLabels)
 	})
 }

app/vminsert/graphite/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -18,7 +19,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/influx/request_handler.go

@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -34,7 +35,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader) error {
-	return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, false, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, nil)
 	})
 }
@@ -52,7 +53,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, extraLabels)
 	})
 }

app/vminsert/main.go

@@ -48,13 +48,13 @@ var (
 		"See also -influxListenAddr.useProxyProtocol")
 	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbListenAddr.useProxyProtocol")
 	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
@@ -157,6 +157,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write":
+		if common.HandleVMProtoServerHandshake(w, r) {
+			return true
+		}
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()

app/vminsert/native/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -27,12 +27,12 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(block, extraLabels)
 	})
 }
 
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 

app/vminsert/opentsdb/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -18,7 +19,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/opentsdbhttp/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -27,7 +28,7 @@ func InsertHandler(req *http.Request) error {
 		if err != nil {
 			return err
 		}
-		return parser.ParseStream(req, func(rows []parser.Row) error {
+		return stream.Parse(req, func(rows []parser.Row) error {
 			return insertRows(rows, extraLabels)
 		})
 	default:

app/vminsert/prometheusimport/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,7 +29,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vminsert/prompush/push.go

@@ -21,7 +21,7 @@ func Push(wr *prompbmarshal.WriteRequest) {
 
 	tss := wr.Timeseries
 	for len(tss) > 0 {
-		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		// Process big tss in smaller blocks in order to reduce maximum memory usage
 		samplesCount := 0
 		i := 0
 		for i < len(tss) {

app/vminsert/promremotewrite/request_handler.go

@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -23,7 +23,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
 		return insertRows(tss, extraLabels)
 	})
 }

app/vminsert/vmimport/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -29,7 +30,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	})
 }

app/vmrestore/Makefile

@@ -75,6 +75,9 @@ vmrestore-linux-arm64:
 vmrestore-linux-ppc64le:
 	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmrestore-linux-s390x:
+	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmrestore-linux-386:
 	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmselect/graphiteql/parser.go

@@ -164,7 +164,7 @@ func (p *parser) parseString() (*StringExpr, error) {
 	return se, nil
 }
 
-// StringExpr represents string contant.
+// StringExpr represents string constant.
 type StringExpr struct {
 	// S contains unquoted string contents.
 	S string

app/vmselect/main.go

@@ -35,8 +35,9 @@ var (
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
 	resetCacheAuthKey    = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
-	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
-	vmalertProxyURL      = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
+	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
+		"See also -search.logQueryMemoryUsage")
+	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
 )
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)

app/vmselect/netstorage/netstorage.go

@@ -193,7 +193,7 @@ func getTmpResult() *result {
 func putTmpResult(r *result) {
 	currentTime := fasttime.UnixTimestamp()
 	if cap(r.rs.Values) > 1024*1024 && 4*len(r.rs.Values) < cap(r.rs.Values) && currentTime-r.lastResetTime > 10 {
-		// Reset r.rs in order to preseve memory usage after processing big time series with millions of rows.
+		// Reset r.rs in order to preserve memory usage after processing big time series with millions of rows.
 		r.rs = Result{}
 		r.lastResetTime = currentTime
 	}
@@ -936,7 +936,7 @@ func TagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagV
 
 // TSDBStatus returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
 //
-// It accepts aribtrary filters on time series in sq.
+// It accepts arbitrary filters on time series in sq.
 func TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
 	qt = qt.NewChild("get tsdb stats: %s, focusLabel=%q, topN=%d", sq, focusLabel, topN)
 	defer qt.Done()
@@ -1187,8 +1187,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-		metricName := bytesutil.InternBytes(sr.MetricBlockRef.MetricName)
-		brs := m[metricName]
+		// Do not intern mb.MetricName, since it leads to increased memory usage.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
+		metricName := sr.MetricBlockRef.MetricName
+		brs := m[string(metricName)]
 		if brs == nil {
 			brs = &blockRefs{}
 			brs.brs = brs.brsPrealloc[:0]
@@ -1198,8 +1200,9 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			addr:    addr,
 		})
 		if len(brs.brs) == 1 {
-			orderedMetricNames = append(orderedMetricNames, metricName)
-			m[metricName] = brs
+			metricNameStr := string(metricName)
+			orderedMetricNames = append(orderedMetricNames, metricNameStr)
+			m[metricNameStr] = brs
 		}
 	}
 	if err := sr.Error(); err != nil {

app/vmselect/prometheus/prometheus.go

@@ -759,6 +759,9 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		GetRequestURI: func() string {
+			return httpserver.GetRequestURI(r)
+		},
 	}
 	result, err := promql.Exec(qt, &ec, query, true)
 	if err != nil {
@@ -860,6 +863,9 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		GetRequestURI: func() string {
+			return httpserver.GetRequestURI(r)
+		},
 	}
 	result, err := promql.Exec(qt, &ec, query, false)
 	if err != nil {
@@ -1013,8 +1019,10 @@ func getRoundDigits(r *http.Request) int {
 
 func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) {
 	d := latencyOffset.Milliseconds()
-	if d <= 1000 {
-		d = 1000
+	if d < 0 {
+		// Zero latency offset may be useful for some use cases.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836
+		d = 0
 	}
 	return searchutils.GetDuration(r, "latency_offset", d)
 }

app/vmselect/prometheus/prometheus_test.go

@@ -200,7 +200,7 @@ func TestAdjustLastPoints(t *testing.T) {
 func TestGetLatencyOffsetMillisecondsSuccess(t *testing.T) {
 	f := func(url string, expectedOffset int64) {
 		t.Helper()
-		r, err := http.NewRequest("GET", url, nil)
+		r, err := http.NewRequest(http.MethodGet, url, nil)
 		if err != nil {
 			t.Fatalf("unexpected error in NewRequest(%q): %s", url, err)
 		}
@@ -219,7 +219,7 @@ func TestGetLatencyOffsetMillisecondsSuccess(t *testing.T) {
 func TestGetLatencyOffsetMillisecondsFailure(t *testing.T) {
 	f := func(url string) {
 		t.Helper()
-		r, err := http.NewRequest("GET", url, nil)
+		r, err := http.NewRequest(http.MethodGet, url, nil)
 		if err != nil {
 			t.Fatalf("unexpected error in NewRequest(%q): %s", url, err)
 		}

app/vmselect/promql/aggr.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"flag"
 	"fmt"
 	"math"
 	"sort"
@@ -15,6 +16,8 @@ import (
 	"github.com/cespare/xxhash/v2"
 )
 
+var maxSeriesPerAggrFunc = flag.Int("search.maxSeriesPerAggrFunc", 1e6, "The maximum number of time series an aggregate MetricsQL function can generate")
+
 var aggrFuncs = map[string]aggrFunc{
 	"any":            aggrFuncAny,
 	"avg":            newAggrFunc(aggrFuncAvg),
@@ -40,6 +43,7 @@ var aggrFuncs = map[string]aggrFunc{
 	"outliersk":      aggrFuncOutliersK,
 	"quantile":       aggrFuncQuantile,
 	"quantiles":      aggrFuncQuantiles,
+	"share":          aggrFuncShare,
 	"stddev":         newAggrFunc(aggrFuncStddev),
 	"stdvar":         newAggrFunc(aggrFuncStdvar),
 	"sum":            newAggrFunc(aggrFuncSum),
@@ -105,6 +109,16 @@ func removeGroupTags(metricName *storage.MetricName, modifier *metricsql.Modifie
 
 func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries, argOrig []*timeseries,
 	modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
+	m := aggrPrepareSeries(argOrig, modifier, maxSeries, keepOriginal)
+	rvs := make([]*timeseries, 0, len(m))
+	for _, tss := range m {
+		rv := afe(tss, modifier)
+		rvs = append(rvs, rv...)
+	}
+	return rvs, nil
+}
+
+func aggrPrepareSeries(argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) map[string][]*timeseries {
 	// Remove empty time series, e.g. series with all NaN samples,
 	// since such series are ignored by aggregate functions.
 	argOrig = removeEmptySeries(argOrig)
@@ -129,21 +143,7 @@ func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) [
 		m[k] = tss
 	}
 	bbPool.Put(bb)
-
-	srcTssCount := 0
-	dstTssCount := 0
-	rvs := make([]*timeseries, 0, len(m))
-	for _, tss := range m {
-		rv := afe(tss, modifier)
-		rvs = append(rvs, rv...)
-		srcTssCount += len(tss)
-		dstTssCount += len(rv)
-		if dstTssCount > 2000 && dstTssCount > 16*srcTssCount {
-			// This looks like count_values explosion.
-			return nil, fmt.Errorf(`too many timeseries after aggragation; got %d; want less than %d`, dstTssCount, 16*srcTssCount)
-		}
-	}
-	return rvs, nil
+	return m
 }
 
 func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
@@ -455,6 +455,37 @@ func aggrFuncMode(tss []*timeseries) []*timeseries {
 	return tss[:1]
 }
 
+func aggrFuncShare(afa *aggrFuncArg) ([]*timeseries, error) {
+	tss, err := getAggrTimeseries(afa.args)
+	if err != nil {
+		return nil, err
+	}
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+		for i := range tss[0].Values {
+			// Calculate sum for non-negative points at position i.
+			var sum float64
+			for _, ts := range tss {
+				v := ts.Values[i]
+				if math.IsNaN(v) || v < 0 {
+					continue
+				}
+				sum += v
+			}
+			// Divide every non-negative value at poisition i by sum in order to get its' share.
+			for _, ts := range tss {
+				v := ts.Values[i]
+				if math.IsNaN(v) || v < 0 {
+					ts.Values[i] = nan
+				} else {
+					ts.Values[i] = v / sum
+				}
+			}
+		}
+		return tss
+	}
+	return aggrFuncExt(afe, tss, &afa.ae.Modifier, afa.ae.Limit, true)
+}
+
 func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 	tss, err := getAggrTimeseries(afa.args)
 	if err != nil {
@@ -491,10 +522,6 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 				ts.Values[i] = (v - avg) / stddev
 			}
 		}
-		// Remove MetricGroup from all the tss.
-		for _, ts := range tss {
-			ts.MetricName.ResetMetricGroup()
-		}
 		return tss
 	}
 	return aggrFuncExt(afe, tss, &afa.ae.Modifier, afa.ae.Limit, true)
@@ -560,46 +587,56 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 		// Do nothing
 	}
 
-	afe := func(tss []*timeseries, modififer *metricsql.ModifierExpr) []*timeseries {
-		m := make(map[float64]bool)
+	afe := func(tss []*timeseries, modififer *metricsql.ModifierExpr) ([]*timeseries, error) {
+		m := make(map[float64]*timeseries)
 		for _, ts := range tss {
-			for _, v := range ts.Values {
+			for i, v := range ts.Values {
 				if math.IsNaN(v) {
 					continue
 				}
-				m[v] = true
-			}
-		}
-		values := make([]float64, 0, len(m))
-		for v := range m {
-			values = append(values, v)
-		}
-		sort.Float64s(values)
-
-		var rvs []*timeseries
-		for _, v := range values {
-			var dst timeseries
-			dst.CopyFromShallowTimestamps(tss[0])
-			dst.MetricName.RemoveTag(dstLabel)
-			dst.MetricName.AddTag(dstLabel, strconv.FormatFloat(v, 'f', -1, 64))
-			for i := range dst.Values {
-				count := 0
-				for _, ts := range tss {
-					if ts.Values[i] == v {
-						count++
+				dst := m[v]
+				if dst == nil {
+					if len(m) >= *maxSeriesPerAggrFunc {
+						return nil, fmt.Errorf("more than -search.maxSeriesPerAggrFunc=%d are generated by count_values()", *maxSeriesPerAggrFunc)
 					}
+					dst = &timeseries{}
+					dst.CopyFromShallowTimestamps(tss[0])
+					dst.MetricName.RemoveTag(dstLabel)
+					dst.MetricName.AddTag(dstLabel, strconv.FormatFloat(v, 'f', -1, 64))
+					values := dst.Values
+					for j := range values {
+						values[j] = nan
+					}
+					m[v] = dst
 				}
-				n := float64(count)
-				if n == 0 {
-					n = nan
+				values := dst.Values
+				if math.IsNaN(values[i]) {
+					values[i] = 1
+				} else {
+					values[i]++
 				}
-				dst.Values[i] = n
 			}
-			rvs = append(rvs, &dst)
 		}
-		return rvs
+		rvs := make([]*timeseries, 0, len(m))
+		for _, ts := range m {
+			rvs = append(rvs, ts)
+		}
+		return rvs, nil
 	}
-	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, false)
+
+	m := aggrPrepareSeries(args[1], &afa.ae.Modifier, afa.ae.Limit, false)
+	rvs := make([]*timeseries, 0, len(m))
+	for _, tss := range m {
+		rv, err := afe(tss, modifier)
+		if err != nil {
+			return nil, err
+		}
+		rvs = append(rvs, rv...)
+		if len(rvs) > *maxSeriesPerAggrFunc {
+			return nil, fmt.Errorf("more than -search.maxSeriesPerAggrFunc=%d are generated by count_values()", *maxSeriesPerAggrFunc)
+		}
+	}
+	return rvs, nil
 }
 
 func newAggrFuncTopK(isReverse bool) aggrFunc {
@@ -746,7 +783,7 @@ func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
 	}
 }
 
-func getIntK(k float64, kMax int) int {
+func getIntK(k float64, max int) int {
 	if math.IsNaN(k) {
 		return 0
 	}
@@ -754,8 +791,8 @@ func getIntK(k float64, kMax int) int {
 	if kn < 0 {
 		return 0
 	}
-	if kn > kMax {
-		return kMax
+	if kn > max {
+		return max
 	}
 	return kn
 }
@@ -901,7 +938,7 @@ func quantileSorted(phi float64, values []float64) float64 {
 func aggrFuncMAD(tss []*timeseries) []*timeseries {
 	// Calculate medians for each point across tss.
 	medians := getPerPointMedians(tss)
-	// Calculate MAD values multipled by tolerance for each point across tss.
+	// Calculate MAD values multiplied by tolerance for each point across tss.
 	// See https://en.wikipedia.org/wiki/Median_absolute_deviation
 	mads := getPerPointMADs(tss, medians)
 	tss[0].Values = append(tss[0].Values[:0], mads...)
@@ -920,7 +957,7 @@ func aggrFuncOutliersMAD(afa *aggrFuncArg) ([]*timeseries, error) {
 	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		// Calculate medians for each point across tss.
 		medians := getPerPointMedians(tss)
-		// Calculate MAD values multipled by tolerance for each point across tss.
+		// Calculate MAD values multiplied by tolerance for each point across tss.
 		// See https://en.wikipedia.org/wiki/Median_absolute_deviation
 		mads := getPerPointMADs(tss, medians)
 		for n := range mads {

app/vmselect/promql/eval.go

@@ -30,7 +30,11 @@ var (
 		"See https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3")
 	maxMemoryPerQuery = flagutil.NewBytes("search.maxMemoryPerQuery", 0, "The maximum amounts of memory a single query may consume. "+
 		"Queries requiring more memory are rejected. The total memory limit for concurrently executed queries can be estimated "+
-		"as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests")
+		"as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests . "+
+		"See also -search.logQueryMemoryUsage")
+	logQueryMemoryUsage = flagutil.NewBytes("search.logQueryMemoryUsage", 0, "Log queries, which require more memory than specified by this flag. "+
+		"This may help detecting and optimizing heavy queries. Query logging is disabled by default. "+
+		"See also -search.logSlowQueryDuration and -search.maxMemoryPerQuery")
 	noStaleMarkers = flag.Bool("search.noStaleMarkers", false, "Set this flag to true if the database doesn't contain Prometheus stale markers, "+
 		"so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets")
 )
@@ -123,6 +127,10 @@ type EvalConfig struct {
 	// EnforcedTagFilterss may contain additional label filters to use in the query.
 	EnforcedTagFilterss [][]storage.TagFilter
 
+	// The callback, which returns the request URI during logging.
+	// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
+	GetRequestURI func() string
+
 	timestamps     []int64
 	timestampsOnce sync.Once
 }
@@ -140,6 +148,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
+	ec.GetRequestURI = src.GetRequestURI
 
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
@@ -364,7 +373,9 @@ func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFunc
 		args: args,
 		ec:   ec,
 	}
+	qtChild := qt.NewChild("eval %s", ae.Name)
 	rv, err := af(afa)
+	qtChild.Done()
 	if err != nil {
 		return nil, fmt.Errorf(`cannot evaluate %q: %w`, ae.AppendString(nil), err)
 	}
@@ -466,7 +477,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 	// 1) execute the exprFirst
 	// 2) get common label filters for series returned at step 1
 	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-	//    during exprSecond exection instead of spending compute resources on extracting and processing these series
+	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
 	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
 	// 4) execute the exprSecond with possible additional filters found at step 3
 	//
@@ -1079,26 +1090,33 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs)))
 	rollupMemorySize = sumNoOverflow(mulNoOverflow(int64(rssLen), 1000), mulNoOverflow(rollupPoints, 16))
+	if maxMemory := int64(logQueryMemoryUsage.N); maxMemory > 0 && rollupMemorySize > maxMemory {
+		requestURI := ec.GetRequestURI()
+		logger.Warnf("remoteAddr=%s, requestURI=%s: the %s requires %d bytes of memory for processing; "+
+			"logging this query, since it exceeds the -search.logQueryMemoryUsage=%d; "+
+			"the query selects %d time series and generates %d points across all the time series; try reducing the number of selected time series",
+			ec.QuotedRemoteAddr, requestURI, expr.AppendString(nil), rollupMemorySize, maxMemory, timeseriesLen*len(rcs), rollupPoints)
+	}
 	if maxMemory := int64(maxMemoryPerQuery.N); maxMemory > 0 && rollupMemorySize > maxMemory {
 		rss.Cancel()
 		return nil, &UserReadableError{
-			Err: fmt.Errorf("not enough memory for processing %d data points across %d time series with %d points in each time series "+
+			Err: fmt.Errorf("not enough memory for processing %s, which returns %d data points across %d time series with %d points in each time series "+
 				"according to -search.maxMemoryPerQuery=%d; requested memory: %d bytes; "+
 				"possible solutions are: reducing the number of matching time series; increasing `step` query arg (step=%gs); "+
 				"increasing -search.maxMemoryPerQuery",
-				rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, maxMemory, rollupMemorySize, float64(ec.Step)/1e3),
+				expr.AppendString(nil), rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, maxMemory, rollupMemorySize, float64(ec.Step)/1e3),
 		}
 	}
 	rml := getRollupMemoryLimiter()
 	if !rml.Get(uint64(rollupMemorySize)) {
 		rss.Cancel()
 		return nil, &UserReadableError{
-			Err: fmt.Errorf("not enough memory for processing %d data points across %d time series with %d points in each time series; "+
+			Err: fmt.Errorf("not enough memory for processing %s, which returns %d data points across %d time series with %d points in each time series; "+
 				"total available memory for concurrent requests: %d bytes; "+
 				"requested memory: %d bytes; "+
 				"possible solutions are: reducing the number of matching time series; increasing `step` query arg (step=%gs); "+
 				"switching to node with more RAM; increasing -memory.allowedPercent",
-				rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, rml.MaxSize, uint64(rollupMemorySize), float64(ec.Step)/1e3),
+				expr.AppendString(nil), rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, rml.MaxSize, uint64(rollupMemorySize), float64(ec.Step)/1e3),
 		}
 	}
 	defer rml.Put(uint64(rollupMemorySize))

app/vmselect/promql/exec_test.go

@@ -3630,6 +3630,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`mad_over_time()`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(mad_over_time(rand(0)[200s:5s]), 0.001)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.243, 0.274, 0.256, 0.185, 0.266, 0.256},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`stddev_over_time()`, func(t *testing.T) {
 		t.Parallel()
 		q := `round(stddev_over_time(rand(0)[200s:5s]), 0.001)`
@@ -4784,6 +4795,85 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`share()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(round(share((
+			label_set(time()/100+10, "k", "v1"),
+			label_set(time()/200+5, "k", "v2"),
+			label_set(time()/110-10, "k", "v3"),
+			label_set(time()/90-5, "k", "v4"),
+		)), 0.001), "k")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.554, 0.521, 0.487, 0.462, 0.442, 0.426},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("k"),
+			Value: []byte("v1"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.277, 0.26, 0.243, 0.231, 0.221, 0.213},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("k"),
+			Value: []byte("v2"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 0.022, 0.055, 0.081, 0.1, 0.116},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("k"),
+			Value: []byte("v3"),
+		}}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.169, 0.197, 0.214, 0.227, 0.237, 0.245},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("k"),
+			Value: []byte("v4"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(share())`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum(share((
+			label_set(time()/100+10, "k", "v1"),
+			label_set(time()/200+5, "k", "v2"),
+			label_set(time()/110-10, "k", "v3"),
+			label_set(time()/90-5, "k", "v4"),
+		))), 0.001)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(share() by (k))`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(sum(share((
+			label_set(time()/100+10, "k", "v1"),
+			label_set(time()/200+5, "k", "v2", "a", "b"),
+			label_set(time()/110-10, "k", "v1", "a", "b"),
+			label_set(time()/90-5, "k", "v2"),
+		)) by (k)), 0.001)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`zscore()`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label(round(zscore((
@@ -6122,7 +6212,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() < 1300)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1200, 1200, 1200, 1200},
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -6133,7 +6223,18 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() > 1500)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1800, 2000},
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`interpolate(tail_head_and_middle)`, func(t *testing.T) {
+		t.Parallel()
+		q := `interpolate(time() > 1100 and time() < 1300 default time() > 1700 and time() < 1900)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1200, 1400, 1600, 1800, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -6492,6 +6593,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`range_trim_outliers()`, func(t *testing.T) {
+		t.Parallel()
+		q := `range_trim_outliers(0.5, time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`range_trim_spikes()`, func(t *testing.T) {
 		t.Parallel()
 		q := `range_trim_spikes(0.2, time())`
@@ -6503,6 +6615,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`range_trim_zscore()`, func(t *testing.T) {
+		t.Parallel()
+		q := `range_trim_zscore(0.9, time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1200, 1400, 1600, 1800, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`range_zscore()`, func(t *testing.T) {
+		t.Parallel()
+		q := `round(range_zscore(time()), 0.1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{-1.5, -0.9, -0.3, 0.3, 0.9, 1.5},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`range_quantile(0.5)`, func(t *testing.T) {
 		t.Parallel()
 		q := `range_quantile(0.5, time())`
@@ -7048,6 +7182,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`range_mad(time())`, func(t *testing.T) {
+		t.Parallel()
+		q := `range_mad(time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{300, 300, 300, 300, 300, 300},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`range_max(time())`, func(t *testing.T) {
 		t.Parallel()
 		q := `range_max(time())`
@@ -7377,6 +7522,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4}
 		f(q, resultExpected)
 	})
+	t.Run(`rollup_candlestick(high)`, func(t *testing.T) {
+		t.Parallel()
+		q := `rollup_candlestick(alias(round(rand(0),0.01),"foobar")[:10s], "high")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.9, 0.94, 0.97, 0.93, 0.98, 0.92},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foobar")
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("high"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`rollup_increase()`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(rollup_increase(time()))`
@@ -7410,6 +7571,61 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`rollup_rate()`, func(t *testing.T) {
+		t.Parallel()
+		q := `rollup_rate((2000-time())[600s])`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("avg"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("max"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 3, 2, 1, 0, -1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("min"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`rollup_rate(q, "max")`, func(t *testing.T) {
+		t.Parallel()
+		q := `rollup_rate((2000-time())[600s], "max")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`rollup_rate(q, "avg")`, func(t *testing.T) {
+		t.Parallel()
+		q := `rollup_rate((2000-time())[600s], "avg")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`rollup_scrape_interval()`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label(rollup_scrape_interval(1[5m:10S]), "rollup")`
@@ -7509,6 +7725,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`rollup_deriv(q, "max")`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(rollup_deriv(time()[100s:50s], "max"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`{}`, func(t *testing.T) {
 		t.Parallel()
 		q := `{}`
@@ -8306,8 +8533,12 @@ func TestExecError(t *testing.T) {
 	f(`end(1)`)
 	f(`step(1)`)
 	f(`running_sum(1, 2)`)
+	f(`range_mad()`)
 	f(`range_sum(1, 2)`)
+	f(`range_trim_outliers()`)
 	f(`range_trim_spikes()`)
+	f(`range_trim_zscore()`)
+	f(`range_zscore()`)
 	f(`range_first(1,  2)`)
 	f(`range_last(1, 2)`)
 	f(`range_linear_regression(1, 2)`)
@@ -8362,6 +8593,7 @@ func TestExecError(t *testing.T) {
 	f(`rate_over_sum()`)
 	f(`zscore_over_time()`)
 	f(`mode()`)
+	f(`share()`)
 	f(`zscore()`)
 	f(`prometheus_buckets()`)
 	f(`buckets_limit()`)
@@ -8385,6 +8617,8 @@ func TestExecError(t *testing.T) {
 	f(`changes_prometheus()`)
 	f(`delta()`)
 	f(`delta_prometheus()`)
+	f(`rollup_candlestick()`)
+	f(`rollup()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)
@@ -8471,6 +8705,12 @@ func TestExecError(t *testing.T) {
 	f(`ru()`)
 	f(`ru(1)`)
 	f(`ru(1,3,3)`)
+
+	// Invalid rollup tags
+	f(`rollup_rate(time()[5m], "")`)
+	f(`rollup_rate(time()[5m], "foo")`)
+	f(`rollup_rate(time()[5m], "foo", "bar")`)
+	f(`rollup_candlestick(time(), "foo")`)
 }
 
 func testResultsEqual(t *testing.T, result, resultExpected []netstorage.Result) {

app/vmselect/promql/rollup.go

@@ -56,6 +56,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"lag":                     newRollupFuncOneArg(rollupLag),
 	"last_over_time":          newRollupFuncOneArg(rollupLast),
 	"lifetime":                newRollupFuncOneArg(rollupLifetime),
+	"mad_over_time":           newRollupFuncOneArg(rollupMAD),
 	"max_over_time":           newRollupFuncOneArg(rollupMax),
 	"min_over_time":           newRollupFuncOneArg(rollupMin),
 	"mode_over_time":          newRollupFuncOneArg(rollupModeOverTime),
@@ -67,13 +68,13 @@ var rollupFuncs = map[string]newRollupFunc{
 	"rate":                    newRollupFuncOneArg(rollupDerivFast), // + rollupFuncsRemoveCounterResets
 	"rate_over_sum":           newRollupFuncOneArg(rollupRateOverSum),
 	"resets":                  newRollupFuncOneArg(rollupResets),
-	"rollup":                  newRollupFuncOneArg(rollupFake),
-	"rollup_candlestick":      newRollupFuncOneArg(rollupFake),
-	"rollup_delta":            newRollupFuncOneArg(rollupFake),
-	"rollup_deriv":            newRollupFuncOneArg(rollupFake),
-	"rollup_increase":         newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_rate":             newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_scrape_interval":  newRollupFuncOneArg(rollupFake),
+	"rollup":                  newRollupFuncOneOrTwoArgs(rollupFake),
+	"rollup_candlestick":      newRollupFuncOneOrTwoArgs(rollupFake),
+	"rollup_delta":            newRollupFuncOneOrTwoArgs(rollupFake),
+	"rollup_deriv":            newRollupFuncOneOrTwoArgs(rollupFake),
+	"rollup_increase":         newRollupFuncOneOrTwoArgs(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_rate":             newRollupFuncOneOrTwoArgs(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_scrape_interval":  newRollupFuncOneOrTwoArgs(rollupFake),
 	"scrape_interval":         newRollupFuncOneArg(rollupScrapeInterval),
 	"share_gt_over_time":      newRollupShareGT,
 	"share_le_over_time":      newRollupShareLE,
@@ -121,6 +122,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"lag":                     rollupLag,
 	"last_over_time":          rollupLast,
 	"lifetime":                rollupLifetime,
+	"mad_over_time":           rollupMAD,
 	"max_over_time":           rollupMax,
 	"min_over_time":           rollupMin,
 	"mode_over_time":          rollupModeOverTime,
@@ -280,6 +282,42 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
 	return aggrFuncNames, nil
 }
 
+// getRollupTag returns the possible second arg from the expr.
+//
+// The expr can have the following forms:
+// - rollup_func(q, tag)
+// - aggr_func(rollup_func(q, tag)) - this form is used during incremental aggregate calculations
+func getRollupTag(expr metricsql.Expr) (string, error) {
+	af, ok := expr.(*metricsql.AggrFuncExpr)
+	if ok {
+		// extract rollup_func() from aggr_func(rollup_func(q, tag))
+		if len(af.Args) != 1 {
+			logger.Panicf("BUG: unexpected number of args to %s; got %d; want 1", af.AppendString(nil), len(af.Args))
+		}
+		expr = af.Args[0]
+	}
+	fe, ok := expr.(*metricsql.FuncExpr)
+	if !ok {
+		logger.Panicf("BUG: unexpected expression; want *metricsql.FuncExpr; got %T; value: %s", expr, expr.AppendString(nil))
+	}
+	if len(fe.Args) < 2 {
+		return "", nil
+	}
+	if len(fe.Args) != 2 {
+		return "", fmt.Errorf("unexpected number of args; got %d; want %d", len(fe.Args), 2)
+	}
+	arg := fe.Args[1]
+
+	se, ok := arg.(*metricsql.StringExpr)
+	if !ok {
+		return "", fmt.Errorf("unexpected rollup tag type: %s; expecting string", arg.AppendString(nil))
+	}
+	if se.S == "" {
+		return "", fmt.Errorf("rollup tag cannot be empty")
+	}
+	return se.S, nil
+}
+
 func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start, end, step int64, maxPointsPerSeries int,
 	window, lookbackDelta int64, sharedTimestamps []int64) (
 	func(values []float64, timestamps []int64), []*rollupConfig, error) {
@@ -309,35 +347,69 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 			samplesScannedPerCall: samplesScannedPerCall,
 		}
 	}
-	appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
-		dst = append(dst, newRollupConfig(rollupMin, "min"))
-		dst = append(dst, newRollupConfig(rollupMax, "max"))
-		dst = append(dst, newRollupConfig(rollupAvg, "avg"))
-		return dst
+
+	appendRollupConfigs := func(dst []*rollupConfig, expr metricsql.Expr) ([]*rollupConfig, error) {
+		tag, err := getRollupTag(expr)
+		if err != nil {
+			return nil, fmt.Errorf("invalid args for %s: %w", expr.AppendString(nil), err)
+		}
+		switch tag {
+		case "min":
+			dst = append(dst, newRollupConfig(rollupMin, ""))
+		case "max":
+			dst = append(dst, newRollupConfig(rollupMax, ""))
+		case "avg":
+			dst = append(dst, newRollupConfig(rollupAvg, ""))
+		case "":
+			dst = append(dst, newRollupConfig(rollupMin, "min"))
+			dst = append(dst, newRollupConfig(rollupMax, "max"))
+			dst = append(dst, newRollupConfig(rollupAvg, "avg"))
+		default:
+			return nil, fmt.Errorf("unexpected second arg for %s: %q; want `min`, `max` or `avg`", expr.AppendString(nil), tag)
+		}
+		return dst, nil
 	}
 	var rcs []*rollupConfig
+	var err error
 	switch funcName {
 	case "rollup":
-		rcs = appendRollupConfigs(rcs)
+		rcs, err = appendRollupConfigs(rcs, expr)
 	case "rollup_rate", "rollup_deriv":
 		preFuncPrev := preFunc
 		preFunc = func(values []float64, timestamps []int64) {
 			preFuncPrev(values, timestamps)
 			derivValues(values, timestamps)
 		}
-		rcs = appendRollupConfigs(rcs)
+		rcs, err = appendRollupConfigs(rcs, expr)
 	case "rollup_increase", "rollup_delta":
 		preFuncPrev := preFunc
 		preFunc = func(values []float64, timestamps []int64) {
 			preFuncPrev(values, timestamps)
 			deltaValues(values)
 		}
-		rcs = appendRollupConfigs(rcs)
+		rcs, err = appendRollupConfigs(rcs, expr)
 	case "rollup_candlestick":
-		rcs = append(rcs, newRollupConfig(rollupOpen, "open"))
-		rcs = append(rcs, newRollupConfig(rollupClose, "close"))
-		rcs = append(rcs, newRollupConfig(rollupLow, "low"))
-		rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
+		tag, err := getRollupTag(expr)
+		if err != nil {
+			return nil, nil, fmt.Errorf("invalid args for %s: %w", expr.AppendString(nil), err)
+		}
+		switch tag {
+		case "open":
+			rcs = append(rcs, newRollupConfig(rollupOpen, "open"))
+		case "close":
+			rcs = append(rcs, newRollupConfig(rollupClose, "close"))
+		case "low":
+			rcs = append(rcs, newRollupConfig(rollupLow, "low"))
+		case "high":
+			rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
+		case "":
+			rcs = append(rcs, newRollupConfig(rollupOpen, "open"))
+			rcs = append(rcs, newRollupConfig(rollupClose, "close"))
+			rcs = append(rcs, newRollupConfig(rollupLow, "low"))
+			rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
+		default:
+			return nil, nil, fmt.Errorf("unexpected second arg for %s: %q; want `min`, `max` or `avg`", expr.AppendString(nil), tag)
+		}
 	case "rollup_scrape_interval":
 		preFuncPrev := preFunc
 		preFunc = func(values []float64, timestamps []int64) {
@@ -355,7 +427,7 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 				values[0] = values[1]
 			}
 		}
-		rcs = appendRollupConfigs(rcs)
+		rcs, err = appendRollupConfigs(rcs, expr)
 	case "aggr_over_time":
 		aggrFuncNames, err := getRollupAggrFuncNames(expr)
 		if err != nil {
@@ -374,6 +446,9 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 	default:
 		rcs = append(rcs, newRollupConfig(rf, ""))
 	}
+	if err != nil {
+		return nil, nil, err
+	}
 	return preFunc, rcs, nil
 }
 
@@ -383,7 +458,7 @@ func getRollupFunc(funcName string) newRollupFunc {
 }
 
 type rollupFuncArg struct {
-	// The value preceeding values if it fits staleness interval.
+	// The value preceding values if it fits staleness interval.
 	prevValue float64
 
 	// The timestamp for prevValue.
@@ -395,7 +470,7 @@ type rollupFuncArg struct {
 	// Timestamps for values.
 	timestamps []int64
 
-	// Real value preceeding values without restrictions on staleness interval.
+	// Real value preceding values without restrictions on staleness interval.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -585,7 +660,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
-			// Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo).
+			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
 			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
 			// when the window doesn't cover at least two raw samples (this is what most users expect).
 			//
@@ -843,6 +918,15 @@ func newRollupFuncTwoArgs(rf rollupFunc) newRollupFunc {
 	}
 }
 
+func newRollupFuncOneOrTwoArgs(rf rollupFunc) newRollupFunc {
+	return func(args []interface{}) (rollupFunc, error) {
+		if len(args) < 1 || len(args) > 2 {
+			return nil, fmt.Errorf("unexpected number of args; got %d; want 1...2", len(args))
+		}
+		return rf, nil
+	}
+}
+
 func newRollupHoltWinters(args []interface{}) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 3); err != nil {
 		return nil, err
@@ -1213,6 +1297,27 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) {
 	return rf, nil
 }
 
+func rollupMAD(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+
+	return mad(rfa.values)
+}
+
+func mad(values []float64) float64 {
+	// See https://en.wikipedia.org/wiki/Median_absolute_deviation
+	median := quantile(0.5, values)
+	a := getFloat64s()
+	ds := a.A[:0]
+	for _, v := range values {
+		ds = append(ds, math.Abs(v-median))
+	}
+	v := quantile(0.5, ds)
+	a.A = ds
+	putFloat64s(a)
+	return v
+}
+
 func rollupHistogram(rfa *rollupFuncArg) float64 {
 	values := rfa.values
 	tsm := rfa.tsm

app/vmselect/promql/rollup_result_cache.go

@@ -441,7 +441,7 @@ func mustSaveRollupResultCacheKeyPrefix(path string) {
 var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total")
 
 // Increment this value every time the format of the cache changes.
-const rollupResultCacheVersion = 8
+const rollupResultCacheVersion = 9
 
 func marshalRollupResultCacheKey(dst []byte, expr metricsql.Expr, window, step int64, etfs [][]storage.TagFilter) []byte {
 	dst = append(dst, rollupResultCacheVersion)

app/vmselect/promql/rollup_test.go

@@ -499,6 +499,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("resets", 5)
 	f("range_over_time", 111)
 	f("avg_over_time", 47.083333333333336)
+	f("mad_over_time", 10)
 	f("min_over_time", 12)
 	f("max_over_time", 123)
 	f("tmin_over_time", 0.08)

app/vmselect/promql/timeseries.go

@@ -82,15 +82,17 @@ func marshalTimeseriesFast(dst []byte, tss []*timeseries, maxSize int, step int6
 		logger.Panicf("BUG: tss cannot be empty")
 	}
 
-	// Calculate the required size for marshaled tss.
-	size := 0
-	for _, ts := range tss {
-		size += ts.marshaledFastSizeNoTimestamps()
-	}
-	// timestamps are stored only once for all the tss, since they are identical.
+	// timestamps are stored only once for all the tss, since they must be identical
 	assertIdenticalTimestamps(tss, step)
-	size += 8 * len(tss[0].Timestamps)
+	timestamps := tss[0].Timestamps
 
+	// Calculate the required size for marshaled tss.
+	size := 8 + 8                          // 8 bytes for len(tss) and 8 bytes for len(timestamps)
+	size += 8 * len(timestamps)            // encoded timestamps
+	size += 8 * len(tss) * len(timestamps) // encoded values
+	for _, ts := range tss {
+		size += marshaledFastMetricNameSize(&ts.MetricName)
+	}
 	if size > maxSize {
 		// Do not marshal tss, since it would occupy too much space
 		return dst
@@ -98,176 +100,133 @@ func marshalTimeseriesFast(dst []byte, tss []*timeseries, maxSize int, step int6
 
 	// Allocate the buffer for the marshaled tss before its' marshaling.
 	// This should reduce memory fragmentation and memory usage.
-	dst = bytesutil.ResizeNoCopyMayOverallocate(dst, size)
-	dst = marshalFastTimestamps(dst[:0], tss[0].Timestamps)
+	dstLen := len(dst)
+	dst = bytesutil.ResizeWithCopyMayOverallocate(dst, size+dstLen)
+	dst = dst[:dstLen]
+
+	// Marshal timestamps and values at first, so they are 8-byte aligned.
+	// This prevents from SIGBUS error on arm architectures.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	dst = encoding.MarshalUint64(dst, uint64(len(tss)))
+	dst = encoding.MarshalUint64(dst, uint64(len(timestamps)))
+	dst = marshalTimestampsFast(dst, timestamps)
 	for _, ts := range tss {
-		dst = ts.marshalFastNoTimestamps(dst)
+		dst = marshalValuesFast(dst, ts.Values)
+	}
+	for _, ts := range tss {
+		dst = marshalMetricNameFast(dst, &ts.MetricName)
 	}
 	return dst
 }
 
 // unmarshalTimeseriesFast unmarshals timeseries from src.
 //
-// The returned timeseries refer to src, so it is unsafe to modify it
-// until timeseries are in use.
+// The returned timeseries refer to src, so it is unsafe to modify it while timeseries are in use.
 func unmarshalTimeseriesFast(src []byte) ([]*timeseries, error) {
-	tail, timestamps, err := unmarshalFastTimestamps(src)
+	if len(src) < 16 {
+		return nil, fmt.Errorf("cannot unmarshal timeseries from %d bytes; need at least 16 bytes", len(src))
+	}
+	tssLen := encoding.UnmarshalUint64(src)
+	timestampsLen := encoding.UnmarshalUint64(src[8:])
+	src = src[16:]
+
+	// Unmarshal timestamps
+	tail, timestamps, err := unmarshalTimestampsFast(src, timestampsLen)
 	if err != nil {
 		return nil, err
 	}
 	src = tail
 
-	var tss []*timeseries
-	for len(src) > 0 {
+	tss := make([]*timeseries, tssLen)
+	for i := range tss {
 		var ts timeseries
-		ts.denyReuse = false
+		ts.denyReuse = true
 		ts.Timestamps = timestamps
+		tss[i] = &ts
+	}
 
-		tail, err := ts.unmarshalFastNoTimestamps(src)
+	// Unmarshal values
+	for _, ts := range tss {
+		tail, values, err := unmarshalValuesFast(src, timestampsLen)
+		if err != nil {
+			return nil, err
+		}
+		ts.Values = values
+		src = tail
+	}
+
+	// Unmarshal metric names for the time series
+	for _, ts := range tss {
+		tail, err := unmarshalMetricNameFast(&ts.MetricName, src)
 		if err != nil {
 			return nil, err
 		}
 		src = tail
+	}
 
-		tss = append(tss, &ts)
+	if len(src) > 0 {
+		return nil, fmt.Errorf("unexpected non-empty tail left after unmarshaling %d timeseries; len(tail)=%d", len(tss), len(src))
 	}
 	return tss, nil
 }
 
-// marshaledFastSizeNoTimestamps returns the size of marshaled ts
-// returned from marshalFastNoTimestamps.
-func (ts *timeseries) marshaledFastSizeNoTimestamps() int {
-	mn := &ts.MetricName
-	n := 2 + len(mn.MetricGroup)
+// marshaledFastMetricNameSize returns the size of marshaled mn returned from marshalMetricNameFast.
+func marshaledFastMetricNameSize(mn *storage.MetricName) int {
+	n := 0
+	n += 2 + len(mn.MetricGroup)
 	n += 2 // Length of tags.
 	for i := range mn.Tags {
 		tag := &mn.Tags[i]
 		n += 2 + len(tag.Key)
 		n += 2 + len(tag.Value)
 	}
-	n += 8 * len(ts.Values)
 	return n
 }
 
-// marshalFastNoTimestamps appends marshaled ts to dst and returns the result.
-//
-// It doesn't marshal timestamps.
-//
-// The result must be unmarshaled with unmarshalFastNoTimestamps.
-func (ts *timeseries) marshalFastNoTimestamps(dst []byte) []byte {
-	mn := &ts.MetricName
-	dst = marshalBytesFast(dst, mn.MetricGroup)
-	dst = encoding.MarshalUint16(dst, uint16(len(mn.Tags)))
-	// There is no need in tags' sorting - they must be sorted after unmarshaling.
-	for i := range mn.Tags {
-		tag := &mn.Tags[i]
-		dst = marshalBytesFast(dst, tag.Key)
-		dst = marshalBytesFast(dst, tag.Value)
-	}
-
-	// Do not marshal len(ts.Values), since it is already encoded as len(ts.Timestamps)
-	// during marshalFastTimestamps.
-	var valuesBuf []byte
-	if len(ts.Values) > 0 {
-		valuesBuf = float64ToByteSlice(ts.Values)
-	}
+func marshalValuesFast(dst []byte, values []float64) []byte {
+	// Do not marshal len(values), since it is already encoded as len(timestamps) at marshalTimestampsFast.
+	valuesBuf := float64ToByteSlice(values)
 	dst = append(dst, valuesBuf...)
 	return dst
 }
 
-func marshalFastTimestamps(dst []byte, timestamps []int64) []byte {
-	dst = encoding.MarshalUint32(dst, uint32(len(timestamps)))
-	var timestampsBuf []byte
-	if len(timestamps) > 0 {
-		timestampsBuf = int64ToByteSlice(timestamps)
+// it is unsafe modifying src while the returned values is in use.
+func unmarshalValuesFast(src []byte, valuesLen uint64) ([]byte, []float64, error) {
+	bufSize := valuesLen * 8
+	if uint64(len(src)) < bufSize {
+		return src, nil, fmt.Errorf("cannot unmarshal values; got %d ytes; want at least %d bytes", uint64(len(src)), bufSize)
 	}
+	values := byteSliceToFloat64(src[:bufSize])
+	return src[bufSize:], values, nil
+}
+
+func marshalTimestampsFast(dst []byte, timestamps []int64) []byte {
+	timestampsBuf := int64ToByteSlice(timestamps)
 	dst = append(dst, timestampsBuf...)
 	return dst
 }
 
 // it is unsafe modifying src while the returned timestamps is in use.
-func unmarshalFastTimestamps(src []byte) ([]byte, []int64, error) {
-	if len(src) < 4 {
-		return src, nil, fmt.Errorf("cannot decode len(timestamps); got %d bytes; want at least %d bytes", len(src), 4)
-	}
-	timestampsCount := int(encoding.UnmarshalUint32(src))
-	src = src[4:]
-	if timestampsCount == 0 {
-		return src, nil, nil
-	}
-
-	bufSize := timestampsCount * 8
-	if len(src) < bufSize {
+func unmarshalTimestampsFast(src []byte, timestampsLen uint64) ([]byte, []int64, error) {
+	bufSize := timestampsLen * 8
+	if uint64(len(src)) < bufSize {
 		return src, nil, fmt.Errorf("cannot unmarshal timestamps; got %d bytes; want at least %d bytes", len(src), bufSize)
 	}
 	timestamps := byteSliceToInt64(src[:bufSize])
-	src = src[bufSize:]
-
-	return src, timestamps, nil
+	return src[bufSize:], timestamps, nil
 }
 
-// unmarshalFastNoTimestamps unmarshals ts from src, so ts members reference src.
+// marshalMetricNameFast appends marshaled mn to dst and returns the result.
 //
-// It is expected that ts.Timestamps is already unmarshaled.
-//
-// It is unsafe to modify src while ts is in use.
-func (ts *timeseries) unmarshalFastNoTimestamps(src []byte) ([]byte, error) {
-	// ts members point to src, so they cannot be re-used.
-	ts.denyReuse = true
-
-	tail, err := unmarshalMetricNameFast(&ts.MetricName, src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal MetricName: %w", err)
-	}
-	src = tail
-
-	valuesCount := len(ts.Timestamps)
-	if valuesCount == 0 {
-		return src, nil
-	}
-	bufSize := valuesCount * 8
-	if len(src) < bufSize {
-		return src, fmt.Errorf("cannot unmarshal values; got %d bytes; want at least %d bytes", len(src), bufSize)
-	}
-	ts.Values = byteSliceToFloat64(src[:bufSize])
-
-	return src[bufSize:], nil
+// The result must be unmarshaled with unmarshalMetricNameFast.
+func marshalMetricNameFast(dst []byte, mn *storage.MetricName) []byte {
+	dst = marshalBytesFast(dst, mn.MetricGroup)
+	dst = encoding.MarshalUint16(dst, uint16(len(mn.Tags)))
+	// There is no need in tags' sorting - they must be sorted after unmarshaling.
+	return marshalMetricTagsFast(dst, mn.Tags)
 }
 
-func float64ToByteSlice(a []float64) (b []byte) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	sh.Data = uintptr(unsafe.Pointer(&a[0]))
-	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func int64ToByteSlice(a []int64) (b []byte) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	sh.Data = uintptr(unsafe.Pointer(&a[0]))
-	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func byteSliceToInt64(b []byte) (a []int64) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
-	sh.Data = uintptr(unsafe.Pointer(&b[0]))
-	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func byteSliceToFloat64(b []byte) (a []float64) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
-	sh.Data = uintptr(unsafe.Pointer(&b[0]))
-	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-// unmarshalMetricNameFast unmarshals mn from src, so mn members
-// hold references to src.
+// unmarshalMetricNameFast unmarshals mn from src, so mn members hold references to src.
 //
 // It is unsafe modifying src while mn is in use.
 func unmarshalMetricNameFast(mn *storage.MetricName, src []byte) ([]byte, error) {
@@ -320,9 +279,7 @@ func marshalMetricTagsFast(dst []byte, tags []storage.Tag) []byte {
 
 func marshalMetricNameSorted(dst []byte, mn *storage.MetricName) []byte {
 	dst = marshalBytesFast(dst, mn.MetricGroup)
-	sortMetricTags(mn)
-	dst = marshalMetricTagsFast(dst, mn.Tags)
-	return dst
+	return marshalMetricTagsSorted(dst, mn)
 }
 
 func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
@@ -348,6 +305,62 @@ func unmarshalBytesFast(src []byte) ([]byte, []byte, error) {
 	return src[n:], src[:n], nil
 }
 
+func float64ToByteSlice(a []float64) (b []byte) {
+	if len(a) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func int64ToByteSlice(a []int64) (b []byte) {
+	if len(a) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func byteSliceToInt64(b []byte) (a []int64) {
+	if len(b) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	// Make sure that the returned slice is properly aligned to 8 bytes.
+	// This prevents from SIGBUS error on arm architectures, which deny unaligned access.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	if sh.Data%8 != 0 {
+		logger.Panicf("BUG: the input byte slice b must be aligned to 8 bytes")
+	}
+	return
+}
+
+func byteSliceToFloat64(b []byte) (a []float64) {
+	if len(b) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	// Make sure that the returned slice is properly aligned to 8 bytes.
+	// This prevents from SIGBUS error on arm architectures, which deny unaligned access.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	if sh.Data%8 != 0 {
+		logger.Panicf("BUG: the input byte slice b must be aligned to 8 bytes")
+	}
+	return
+}
+
 func stringMetricName(mn *storage.MetricName) string {
 	var dst []byte
 	dst = append(dst, mn.MetricGroup...)

app/vmselect/promql/timeseries_test.go

@@ -1,7 +1,6 @@
 package promql
 
 import (
-	"fmt"
 	"os"
 	"reflect"
 	"testing"
@@ -14,91 +13,78 @@ func TestMain(m *testing.M) {
 	os.Exit(n)
 }
 
-func TestTimeseriesMarshalUnmarshalFast(t *testing.T) {
-	t.Run("single", func(t *testing.T) {
-		var tsOrig timeseries
-		buf := tsOrig.marshalFastNoTimestamps(nil)
-		n := tsOrig.marshaledFastSizeNoTimestamps()
-		if n != len(buf) {
-			t.Fatalf("unexpected marshaled size; got %d; want %d", n, len(buf))
-		}
-
-		var tsGot timeseries
-		tail, err := tsGot.unmarshalFastNoTimestamps(buf)
+func TestMarshalTimeseriesFast(t *testing.T) {
+	f := func(tss []*timeseries) {
+		t.Helper()
+		data := marshalTimeseriesFast(nil, tss, 1e9, 10)
+		tss2, err := unmarshalTimeseriesFast(data)
 		if err != nil {
 			t.Fatalf("cannot unmarshal timeseries: %s", err)
 		}
-		if len(tail) > 0 {
-			t.Fatalf("unexpected non-empty tail left: len(tail)=%d; tail=%X", len(tail), tail)
+		if !reflect.DeepEqual(tss, tss2) {
+			t.Fatalf("unexpected timeseries unmarshaled\ngot\n%#v\nwant\n%#v", tss2[0], tss[0])
 		}
-		tsOrig.denyReuse = true
-		tsOrig.MetricName.MetricGroup = []byte{}
-		if !reflect.DeepEqual(&tsOrig, &tsGot) {
-			t.Fatalf("unexpected ts\ngot:\n%s\nwant:\n%s", &tsGot, &tsOrig)
-		}
-	})
-	t.Run("multiple", func(t *testing.T) {
-		var dst []byte
-		var tssOrig []*timeseries
-		timestamps := []int64{2}
-		for i := 0; i < 10; i++ {
-			var ts timeseries
-			ts.denyReuse = true
-			ts.MetricName.MetricGroup = []byte(fmt.Sprintf("metricGroup %d", i))
-			ts.MetricName.Tags = []storage.Tag{{
-				Key:   []byte(fmt.Sprintf("key %d", i)),
-				Value: []byte(fmt.Sprintf("value %d", i)),
-			}}
-			ts.Values = []float64{float64(i) + 0.2}
-			ts.Timestamps = timestamps
+	}
 
-			dstLen := len(dst)
-			dst = ts.marshalFastNoTimestamps(dst)
-			n := ts.marshaledFastSizeNoTimestamps()
-			if n != len(dst)-dstLen {
-				t.Fatalf("unexpected marshaled size on iteration %d; got %d; want %d", i, n, len(dst)-dstLen)
-			}
+	// Single series
+	f([]*timeseries{{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte{},
+		},
+		denyReuse: true,
+	}})
+	f([]*timeseries{{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foobar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("tag1"),
+					Value: []byte("value1"),
+				},
+				{
+					Key:   []byte("tag2"),
+					Value: []byte("value2"),
+				},
+			},
+		},
+		Values:     []float64{1, 2, 3.234},
+		Timestamps: []int64{10, 20, 30},
+		denyReuse:  true,
+	}})
 
-			var tsGot timeseries
-			tsGot.Timestamps = ts.Timestamps
-			tail, err := tsGot.unmarshalFastNoTimestamps(dst[dstLen:])
-			if err != nil {
-				t.Fatalf("cannot unmarshal timeseries on iteration %d: %s", i, err)
-			}
-			if len(tail) > 0 {
-				t.Fatalf("unexpected non-empty tail left on iteration %d: len(tail)=%d; tail=%x", i, len(tail), tail)
-			}
-			if !reflect.DeepEqual(&ts, &tsGot) {
-				t.Fatalf("unexpected ts on iteration %d\ngot:\n%s\nwant:\n%s", i, &tsGot, &ts)
-			}
-
-			tssOrig = append(tssOrig, &ts)
-		}
-		buf := marshalTimeseriesFast(nil, tssOrig, 1e6, 123)
-		tssGot, err := unmarshalTimeseriesFast(buf)
-		if err != nil {
-			t.Fatalf("error in unmarshalTimeseriesFast: %s", err)
-		}
-		if !reflect.DeepEqual(tssOrig, tssGot) {
-			t.Fatalf("unexpected unmarshaled timeseries\ngot:\n%s\nwant:\n%s", tssGot, tssOrig)
-		}
-
-		src := dst
-		for i := 0; i < 10; i++ {
-			tsOrig := tssOrig[i]
-			var ts timeseries
-			ts.Timestamps = tsOrig.Timestamps
-			tail, err := ts.unmarshalFastNoTimestamps(src)
-			if err != nil {
-				t.Fatalf("cannot unmarshal timeseries[%d]: %s", i, err)
-			}
-			src = tail
-			if !reflect.DeepEqual(tsOrig, &ts) {
-				t.Fatalf("unexpected ts on iteration %d:\n%+v\nwant:\n%+v", i, &ts, tsOrig)
-			}
-		}
-		if len(src) > 0 {
-			t.Fatalf("unexpected tail left; len(tail)=%d; tail=%X", len(src), src)
-		}
+	// Multiple series
+	f([]*timeseries{
+		{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte("foobar"),
+				Tags: []storage.Tag{
+					{
+						Key:   []byte("tag1"),
+						Value: []byte("value1"),
+					},
+					{
+						Key:   []byte("tag2"),
+						Value: []byte("value2"),
+					},
+				},
+			},
+			Values:     []float64{1, 2.34, -33},
+			Timestamps: []int64{-10, 0, 10},
+			denyReuse:  true,
+		},
+		{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte("baz"),
+				Tags: []storage.Tag{
+					{
+						Key:   []byte("tag12"),
+						Value: []byte("value13"),
+					},
+				},
+			},
+			Values:     []float64{4, 1, 2.34},
+			Timestamps: []int64{-10, 0, 10},
+			denyReuse:  true,
+		},
 	})
 }