app/vmctl/terminal: fix builds for GOOS=freebsd and GOOS=openbsd

This is a follow-up for 8da9502df6

.github/ISSUE_TEMPLATE/question.yml

@@ -0,0 +1,32 @@
+name: Question
+description: Ask a question regarding VictoriaMetrics or its components
+labels: [question]
+body:
+  - type: textarea
+    id: describe-the-component
+    attributes:
+      label: Is your question request related to a specific component?
+      placeholder: |
+        VictoriaMetrics, vmagent, vmalert, vmui, etc...
+    validations:
+      required: false
+  - type: textarea
+    id: describe-the-question
+    attributes:
+      label: Describe the question in detail
+      description: |
+        A clear and concise description of the issue and the question.
+    validations:
+      required: true
+  - type: checkboxes
+    id: troubleshooting
+    attributes:
+      label: Troubleshooting docs
+      description: I am familiar with the following troubleshooting docs
+      options:
+        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
+          required: false
+        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
+          required: false
+        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
+          required: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis.yml

@@ -55,9 +55,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main.yml

@@ -30,9 +30,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
           check-latest: true
           cache: true
 
@@ -54,9 +54,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
           check-latest: true
           cache: true
 
@@ -79,9 +79,9 @@ jobs:
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
           check-latest: true
           cache: true
 

.github/workflows/nightly-build.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.0
+          go-version: 1.20.3
         id: go
 
       - name: Setup docker scan

.golangci.yml

@@ -1,14 +1,18 @@
 run:
   timeout: 2m
 
-enable:
+linters:
+  enable:
   - revive
 
 issues:
   exclude-rules:
-    - linters:
-      - staticcheck
-      text: "SA(4003|1019|5011):"
+  - linters:
+    - staticcheck
+    text: "SA(4003|1019|5011):"
+  include:
+  - EXC0012
+  - EXC0014
 
 linters-settings:
   errcheck:

Makefile

@@ -141,6 +141,8 @@ vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
 	vmalert-windows-amd64 \
 	vmauth-windows-amd64 \
+	vmbackup-windows-amd64 \
+	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
 victoria-metrics-crossbuild: \
@@ -186,7 +188,8 @@ release-victoria-metrics: \
 	release-victoria-metrics-darwin-amd64 \
 	release-victoria-metrics-darwin-arm64 \
 	release-victoria-metrics-freebsd-amd64 \
-	release-victoria-metrics-openbsd-amd64
+	release-victoria-metrics-openbsd-amd64 \
+	release-victoria-metrics-windows-amd64
 
 # adds i386 arch
 release-victoria-metrics-linux-386:
@@ -213,6 +216,9 @@ release-victoria-metrics-freebsd-amd64:
 release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
+release-victoria-metrics-windows-amd64:
+	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
+
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
@@ -222,6 +228,16 @@ release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
+release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
+	cd bin && \
+		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-metrics-windows-$(GOARCH)-prod.exe \
+		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-metrics-windows-$(GOARCH)-prod.exe \
+			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		victoria-metrics-windows-$(GOARCH)-prod.exe
+
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -295,26 +311,33 @@ release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
+	vmbackup-windows-$(GOARCH)-prod \
+	vmrestore-windows-$(GOARCH)-prod \
 	vmctl-windows-$(GOARCH)-prod
 	cd bin && \
 		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
+			vmbackup-windows-$(GOARCH)-prod.exe \
+			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
+			vmbackup-windows-$(GOARCH)-prod.exe \
+			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf \
 		vmagent-windows-$(GOARCH)-prod.exe \
 		vmalert-windows-$(GOARCH)-prod.exe \
 		vmauth-windows-$(GOARCH)-prod.exe \
+		vmbackup-windows-$(GOARCH)-prod.exe \
+		vmrestore-windows-$(GOARCH)-prod.exe \
 		vmctl-windows-$(GOARCH)-prod.exe
 
-
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -380,7 +403,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -2,6 +2,7 @@
 
 [![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics.svg?maxAge=604800)](https://hub.docker.com/r/victoriametrics/victoria-metrics)
+[![victoriametrics](https://snapcraft.io/victoriametrics/badge.svg)](https://snapcraft.io/victoriametrics)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
@@ -39,7 +40,8 @@ VictoriaMetrics has the following prominent features:
 * It can be used as long-term storage for Prometheus. See [these docs](#prometheus-setup) for details.
 * It can be used as a drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
 * It can be used as a drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
-* It features easy setup and operation:
+  VictoriaMetrics allows reducing infrastructure costs by more than 10x comparing to Graphite - see [this case study](https://docs.victoriametrics.com/CaseStudies.html#grammarly).
+* It is easy to setup and operate:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
     without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
@@ -105,6 +107,7 @@ Case studies:
 * [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Dig Security](https://docs.victoriametrics.com/CaseStudies.html#dig-security)
 * [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
 * [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
@@ -125,11 +128,22 @@ See also [articles and slides about VictoriaMetrics from our users](https://docs
 
 ## Operation
 
-### How to start VictoriaMetrics
+### Install
 
-Just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
+To quickly try VictoriaMetrics, just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
 See also [QuickStart guide](https://docs.victoriametrics.com/Quick-Start.html) for additional information.
 
+VictoriaMetrics can also be installed via these installation methods:
+
+* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
+* [Kubernetes operator for VictoriaMetrics](https://github.com/VictoriaMetrics/operator).
+* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
+* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
+* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
+* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
+
+### How to start VictoriaMetrics
+
 The following command-line flags are used the most:
 
 * `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
@@ -188,7 +202,7 @@ Changing scrape configuration is possible with text editor:
 vi $SNAP_DATA/var/snap/victoriametrics/current/etc/victoriametrics-scrape-config.yaml
 ```
 
-After changes were made, trigger config re-read with the command `curl 127.0.0.1:8248/-/reload`.
+After changes were made, trigger config re-read with the command `curl 127.0.0.1:8428/-/reload`.
 
 ## Prometheus setup
 
@@ -614,7 +628,6 @@ The `__graphite__` pseudo-label supports e.g. alternate regexp filters such as `
 
 VictoriaMetrics also supports Graphite query language - see [these docs](#graphite-render-api-usage).
 
-
 ## How to send data from OpenTSDB-compatible agents
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
@@ -729,20 +742,45 @@ All the Prometheus querying API handlers can be prepended with `/prometheus` pre
 
 ### Prometheus querying API enhancements
 
-VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used for enforcing additional label filters for queries. For example,
-`/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>` would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used
+for enforcing additional label filters for queries. For example, `/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>`
+would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`.
+This functionality can be used for limiting the scope of time series visible to the given tenant.
+It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics.
+See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
-VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
-`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries.
+For example, `/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically
+add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting
+the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically
+set by auth proxy sitting in front of VictoriaMetrics.
+See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
 VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats).
 
-VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
+VictoriaMetrics accepts `round_digits` query arg for [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query)
+and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) handlers. It can be used for rounding response values
+to the given number of digits after the decimal point.
+For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
 
-VictoriaMetrics accepts `limit` query arg for `/api/v1/labels` and `/api/v1/label/<labelName>/values` handlers for limiting the number of returned entries. For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels. If the provided `limit` value exceeds the corresponding `-search.maxTagKeys` / `-search.maxTagValues` command-line flag values, then limits specified in the command-line flags are used.
+VictoriaMetrics accepts `limit` query arg for [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels)
+and [`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues) handlers for limiting the number of returned entries.
+For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels.
+If the provided `limit` value exceeds the corresponding `-search.maxTagKeys` / `-search.maxTagValues` command-line flag values,
+then limits specified in the command-line flags are used.
 
-By default, VictoriaMetrics returns time series for the last 5 minutes from `/api/v1/series`, `/api/v1/labels` and `/api/v1/label/<labelName>/values` while the Prometheus API defaults to all time.  Explicitly set `start` and `end` to select the desired time range.
-VictoriaMetrics accepts `limit` query arg for `/api/v1/series` handlers for limiting the number of returned entries. For example, the query to `/api/v1/series?limit=5` returns a sample of up to 5 series, while ignoring the rest. If the provided `limit` value exceeds the corresponding `-search.maxSeries` command-line flag values, then limits specified in the command-line flags are used.
+By default, VictoriaMetrics returns time series for the last day starting at 00:00 UTC
+from [/api/v1/series](https://docs.victoriametrics.com/url-examples.html#apiv1series),
+[/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and
+[`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues),
+while the Prometheus API defaults to all time.  Explicitly set `start` and `end` to select the desired time range.
+VictoriaMetrics rounds the specified `start..end` time range to day granularity because of performance optimization concerns.
+If you need the exact set of label names and label values on the given time range, then send queries
+to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) or to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
+
+VictoriaMetrics accepts `limit` query arg at [/api/v1/series](https://docs.victoriametrics.com/url-examples.html#apiv1series)
+for limiting the number of returned entries. For example, the query to `/api/v1/series?limit=5` returns a sample of up to 5 series, while ignoring the rest of series.
+If the provided `limit` value exceeds the corresponding `-search.maxSeries` command-line flag values, then limits specified in the command-line flags are used.
 
 Additionally, VictoriaMetrics provides the following handlers:
 
@@ -791,10 +829,10 @@ VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series w
 
 ### Graphite Render API usage
 
-[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
+VictoriaMetrics supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
 at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
-When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
-Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points
+stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
 
 ### Graphite Metrics API usage
 
@@ -997,7 +1035,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
@@ -1046,7 +1084,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
@@ -1074,7 +1112,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 
 The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
@@ -1313,7 +1351,7 @@ See [allowed formats](#timestamp-formats) for these args.
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
-curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48' -d 'end=2022-06-06T19:29:07'
 ```
 By default, the last point on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden with `max_lookback` query arg.
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
@@ -1356,6 +1394,7 @@ By default VictoriaMetrics is tuned for an optimal resource usage under typical
 - `-search.maxSamplesPerQuery` limits the number of raw samples a single query can process. This allows limiting CPU usage for heavy queries.
 - `-search.maxPointsPerTimeseries` limits the number of calculated points, which can be returned per each matching time series from [range query](https://docs.victoriametrics.com/keyConcepts.html#range-query).
 - `-search.maxPointsSubqueryPerTimeseries` limits the number of calculated points, which can be generated per each matching time series during [subquery](https://docs.victoriametrics.com/MetricsQL.html#subqueries) evaluation.
+- `-search.maxSeriesPerAggrFunc` limits the number of time series, which can be generated by [MetricsQL aggregate functions](https://docs.victoriametrics.com/MetricsQL.html#aggregate-functions) in a single query.
 - `-search.maxSeries` limits the number of time series, which may be returned from [/api/v1/series](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers). This endpoint is used mostly by Grafana for auto-completion of metric names, label names and label values. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxSeries` to quite low value in order limit CPU and memory usage.
 - `-search.maxTagKeys` limits the number of items, which may be returned from [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names). This endpoint is used mostly by Grafana for auto-completion of label names. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagKeys` to quite low value in order to limit CPU and memory usage.
 - `-search.maxTagValues` limits the number of items, which may be returned from [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values). This endpoint is used mostly by Grafana for auto-completion of label values. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagValues` to quite low value in order to limit CPU and memory usage.
@@ -1433,12 +1472,14 @@ can be configured with the `-inmemoryDataFlushInterval` command-line flag (note
 In-memory parts are persisted to disk into `part` directories under the `<-storageDataPath>/data/small/YYYY_MM/` folder,
 where `YYYY_MM` is the month partition for the stored data. For example, `2022_11` is the partition for `parts`
 with [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) from `November 2022`.
+Each partition directory contains `parts.json` file with the actual list of parts in the partition.
 
-The `part` directory has the following name pattern: `rowsCount_blocksCount_minTimestamp_maxTimestamp`, where:
+Every `part` directory contains `metadata.json` file with the following fields:
 
-- `rowsCount` - the number of [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) stored in the part
-- `blocksCount` - the number of blocks stored in the part (see details about blocks below)
-- `minTimestamp` and `maxTimestamp` - minimum and maximum timestamps across raw samples stored in the part
+- `RowsCount` - the number of [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) stored in the part
+- `BlocksCount` - the number of blocks stored in the part (see details about blocks below)
+- `MinTimestamp` and `MaxTimestamp` - minimum and maximum timestamps across raw samples stored in the part
+- `MinDedupInterval` - the [deduplication interval](#deduplication) applied to the given part.
 
 Each `part` consists of `blocks` sorted by internal time series id (aka `TSID`).
 Each `block` contains up to 8K [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples),
@@ -1460,9 +1501,8 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
   and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
 
 Newly added `parts` either successfully appear in the storage or fail to appear.
-The newly added `parts` are being created in a temporary directory under `<-storageDataPath>/data/{small,big}/YYYY_MM/tmp` folder.
-When the newly added `part` is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html)
-to a temporary directory, then it is atomically moved to the storage directory.
+The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
+after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
 Thanks to this alogrithm, storage never contains partially created parts, even if hardware power off
 occurrs in the middle of writing the `part` to disk - such incompletely written `parts`
 are automatically deleted on the next VictoriaMetrics start.
@@ -1491,8 +1531,7 @@ Retention is configured with the `-retentionPeriod` command-line flag, which tak
 
 Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
 Data partitions outside the configured retention are deleted on the first day of the new month.
-Each partition consists of one or more data parts with the following name pattern `rowsCount_blocksCount_minTimestamp_maxTimestamp`.
-Data parts outside of the configured retention are eventually deleted during
+Each partition consists of one or more data parts. Data parts outside of the configured retention are eventually deleted during
 [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 
 The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
@@ -1885,7 +1924,14 @@ are added to all the metrics before sending them to the remote storage:
 
 ## Cache removal
 
-VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown (e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup. Sometimes it is needed to remove such caches on the next startup. This can be performed by placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.
+VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown
+(e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup.
+Sometimes it is needed to remove such caches on the next startup. This can be done in the following ways:
+
+- By manually removing the `<-storageDataPath>/cache` directory when VictoriaMetrics is stopped.
+- By placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics.
+  In this case VictoriaMetrics will automatically remove all the caches on the next start.
+  See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.
 
 ## Cache tuning
 
@@ -2037,17 +2083,10 @@ It is safe sharing the collected profiles from security point of view, since the
 
 ## Integrations
 
-* [Helm charts for single-node and cluster versions of VictoriaMetrics](https://github.com/VictoriaMetrics/helm-charts).
-* [Kubernetes operator for VictoriaMetrics](https://github.com/VictoriaMetrics/operator).
-* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
-  See [these docs](https://github.com/netdata/netdata#integrations).
 * [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
   See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
-* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
-* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
-* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
-
-* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
+* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
+  See [these docs](https://github.com/netdata/netdata#integrations).
 * [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/vmalert.html).
 
 ## Third-party contributions
@@ -2154,7 +2193,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
   -dryRun
-     Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
+     Whether to check config files without running VictoriaMetrics. The following config files are checked: -promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -2226,8 +2265,12 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s)
   -insert.maxQueueDuration duration
      The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringCacheExpireDuration duration
+     The expire duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -logNewSeries
      Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics
   -loggerDisableTimestamps
@@ -2263,11 +2306,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+     TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
   -opentsdbHTTPListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+     TCP and UDP address to listen for OpenTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
   -opentsdbListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
@@ -2337,6 +2380,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
@@ -2359,7 +2404,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.seriesLimitPerTarget int
      Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
-     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
@@ -2393,13 +2438,16 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.disableCache
      Whether to disable response caching. This may be useful during data backfilling
   -search.graphiteMaxPointsPerSeries int
-     The maximum number of points per series Graphite render API can return. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 1000000)
+     The maximum number of points per series Graphite render API can return (default 1000000)
   -search.graphiteStorageStep duration
-     The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
+     The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API (default 10s)
   -search.latencyOffset duration
      The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
+  -search.logQueryMemoryUsage size
+     Log queries, which require more memory than specified by this flag. This may help detecting and optimizing heavy queries. Query logging is disabled by default. See also -search.logSlowQueryDuration and -search.maxMemoryPerQuery
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -search.logSlowQueryDuration duration
-     Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
+     Log queries with execution time exceeding this value. Zero disables slow query logging. See also -search.logQueryMemoryUsage (default 5s)
   -search.maxConcurrentRequests int
      The maximum number of concurrent search requests. It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. See also -search.maxQueueDuration and -search.maxMemoryPerQuery (default 8)
   -search.maxExportDuration duration
@@ -2409,11 +2457,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxFederateSeries int
      The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
   -search.maxGraphiteSeries int
-     The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 300000)
+     The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
   -search.maxMemoryPerQuery size
-     The maximum amounts of memory a single query may consume. Queries requiring more memory are rejected. The total memory limit for concurrently executed queries can be estimated as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests
+     The maximum amounts of memory a single query may consume. Queries requiring more memory are rejected. The total memory limit for concurrently executed queries can be estimated as -search.maxMemoryPerQuery multiplied by -search.maxConcurrentRequests . See also -search.logQueryMemoryUsage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -search.maxPointsPerTimeseries int
      The maximum points per a single timeseries returned from /api/v1/query_range. This option doesn't limit the number of scanned raw samples in the database. The main purpose of this option is to limit the number of per-series points returned to graphing UI such as VMUI or Grafana. There is no sense in setting this limit to values bigger than the horizontal resolution of the graph (default 30000)
@@ -2432,6 +2480,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000)
   -search.maxSeries int
      The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 30000)
+  -search.maxSeriesPerAggrFunc int
+     The maximum number of time series an aggregate MetricsQL function can generate (default 1000000)
   -search.maxStalenessInterval duration
      The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag
   -search.maxStatusRequestDuration duration
@@ -2472,6 +2522,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum number of CPU cores to use for small merges. Default value is used if set to 0
   -snapshotAuthKey string
      authKey, which must be passed in query string to /snapshot* pages
+  -snapshotCreateTimeout duration
+     The timeout for creating new snapshot. If set, make sure that timeout is lower than backup period
   -snapshotsMaxAge value
      Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
      The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 0)

SECURITY.md

@@ -4,10 +4,10 @@
 
 | Version | Supported          |
 |---------|--------------------|
-| 1.81.x  | :white_check_mark: |
-| 1.80.x  | :x:                |
-| 1.79.x  | :white_check_mark: |
-| < 1.78  | :x:                |
+| [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
+| v1.87.x LTS release | :white_check_mark: |
+| v1.79.x LTS release | :white_check_mark: |
+| other releases  | :x:                |
 
 ## Reporting a Vulnerability
 

app/victoria-metrics/Makefile

@@ -39,6 +39,9 @@ victoria-metrics-freebsd-amd64-prod:
 victoria-metrics-openbsd-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
 
+victoria-metrics-windows-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-windows-amd64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 
@@ -82,6 +85,9 @@ victoria-metrics-linux-arm64:
 victoria-metrics-linux-ppc64le:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+victoria-metrics-linux-s390x:
+	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 victoria-metrics-linux-386:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
@@ -97,6 +103,9 @@ victoria-metrics-freebsd-amd64:
 victoria-metrics-openbsd-amd64:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
+victoria-metrics-windows-amd64:
+	GOARCH=amd64 APP_NAME=victoria-metrics $(MAKE) app-local-windows-goarch
+
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 

app/victoria-metrics/main.go

@@ -8,6 +8,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	vminsertcommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	vminsertrelabel "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -26,11 +28,13 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
-	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
-		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
+	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
+		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
+		"This can be changed with -promscrape.config.strictParse=false command-line flag")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
@@ -53,7 +57,13 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		if err := vminsertrelabel.CheckRelabelConfig(); err != nil {
+			logger.Fatalf("error when checking -relabelConfig: %s", err)
+		}
+		if err := vminsertcommon.CheckStreamAggrConfig(); err != nil {
+			logger.Fatalf("error when checking -streamAggr.config: %s", err)
+		}
+		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
 		return
 	}
 
@@ -92,7 +102,7 @@ func main() {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")

app/vmagent/Makefile

@@ -85,6 +85,9 @@ vmagent-linux-arm64:
 vmagent-linux-ppc64le:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmagent-linux-s390x:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/README.md

@@ -25,7 +25,9 @@ additionally to [discovering Prometheus-compatible targets and scraping metrics
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Can accept data via all the ingestion protocols supported by VictoriaMetrics - see [these docs](#how-to-push-data-to-vmagent).
 * Can aggregate incoming samples by time and by labels before sending them to remote storage - see [these docs](https://docs.victoriametrics.com/stream-aggregation.html).
-* Can replicate collected metrics simultaneously to multiple remote storage systems - see [these docs](#replication-and-high-availability).
+* Can replicate collected metrics simultaneously to multiple Prometheus-compatible remote storage systems - see [these docs](#replication-and-high-availability).
+* Can save egress network bandwidth usage costs when [VictoriaMetrics remote write protocol](#victoriametrics-remote-write-protocol)
+  is used for sending the data to VictoriaMetrics.
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as the connection
   to the remote storage is repaired. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
@@ -45,38 +47,42 @@ additionally to [discovering Prometheus-compatible targets and scraping metrics
 
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) (
 `vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags)),
-unpack it and pass the following flags to the `vmagent` binary in order to start scraping Prometheus-compatible targets:
+unpack it and pass the following flags to the `vmagent` binary in order to start scraping Prometheus-compatible targets
+and sending the data to the Prometheus-compatible remote storage:
 
-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`).
+* `-promscrape.config` with the path to [Prometheus config file](https://docs.victoriametrics.com/sd_configs.html) (usually located at `/etc/prometheus/prometheus.yml`).
   The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file,
   so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` command-line flag.
   In this case `vmagent` ignores unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified
-  multiple times to replicate data concurrently to an arbitrary number of remote storage systems. See [various use cases](#use-cases).
+* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics.
 
-Example command line:
+Example command for writing the data recieved via [supported push-based protocols](#how-to-push-data-to-vmagent)
+to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) located at `victoria-metrics-host:8428`:
+
+```console
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) if you need writing
+the data to [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+
+Example command for scraping Prometheus targets and writing the data to single-node VictoriaMetrics:
 
 ```console
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-Example of scrape configuration for `-promscrape.config` argument you can find [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus.yml)
-
 See [how to scrape Prometheus-compatible targets](#how-to-collect-metrics-in-prometheus-format) for more details.
 
 If you use single-node VictoriaMetrics, then you can discover and scrape Prometheus-compatible targets directly from VictoriaMetrics
 without the need to use `vmagent` - see [these docs](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter).
 
-If you don't need to scrape Prometheus-compatible targets, then the `-promscrape.config` option isn't needed.
-For example, the following command is sufficient for accepting data via [supported push-based protocols](#how-to-push-data-to-vmagent)
-and sending it to the provided `-remoteWrite.url`:
-
-```console
-/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
-```
+`vmagent` can save network bandwidth usage costs under high load when [VictoriaMetrics remote write protocol is used](#victoriametrics-remote-write-protocol).
 
 See [troubleshooting docs](#troubleshooting) if you encounter common issues with `vmagent`.
 
+See [various use cases](#use-cases) for vmagent.
+
 Pass `-help` to `vmagent` in order to see [the full list of supported command-line flags with their descriptions](#advanced-usage).
 
 ## How to push data to vmagent
@@ -98,7 +104,7 @@ additionally to pull-based Prometheus-compatible targets' scraping:
 
 `vmagent` should be restarted in order to update config options set via command-line args.
 `vmagent` supports multiple approaches for reloading configs from updated config files such as
-`-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
+`-promscrape.config`, `-remoteWrite.relabelConfig`, `-remoteWrite.urlRelabelConfig` and `-remoteWrite.streamAggr.config`:
 
 * Sending `SIGHUP` signal to `vmagent` process:
 
@@ -120,7 +126,8 @@ data to the remote storage. It re-tries sending the data to remote storage until
 The maximum on-disk size for the buffered metrics can be limited with `-remoteWrite.maxDiskUsagePerURL`.
 
 `vmagent` works on various architectures from the IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
-See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+The `vmagent` can save network bandwidth usage costs by using [VictoriaMetrics remote write protocol](#victoriametrics-remote-write-protocol).
 
 ### Drop-in replacement for Prometheus
 
@@ -147,6 +154,11 @@ If a single remote storage instance temporarily is out of service, then the coll
 `vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again
 and then it sends the buffered data to the remote storage in order to prevent data gaps.
 
+[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) already supports replication,
+so there is no need in specifying multiple `-remoteWrite.url` flags when writing data to the same cluster.
+See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#replication-and-data-safety).
+
+
 ### Relabeling and filtering
 
 `vmagent` can add, remove or update labels on the collected data before sending it to the remote storage. Additionally,
@@ -175,6 +187,32 @@ the `-remoteWrite.url` command-line flag should be configured as `<schema>://<vm
 according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format).
 There is also support for multitenant writes. See [these docs](#multitenancy).
 
+## VictoriaMetrics remote write protocol
+
+`vmagent` supports sending data to the configured `-remoteWrite.url` either via Prometheus remote write protocol
+or via VictoriaMetrics remote write protocol.
+
+VictoriaMetrics remote write protocol provides the following benefits comparing to Prometheus remote write protocol:
+
+- Reduced network bandwidth usage by 2x-5x. This allows saving network bandwidth usage costs when `vmagent` and
+  the configured remote storage systems are located in different datacenters, availability zones or regions.
+
+- Reduced disk read/write IO and disk space usage at `vmagent` when the remote storage is temporarily unavailable.
+  In this case `vmagent` buffers the incoming data to disk using the VictoriaMetrics remote write format.
+  This reduces disk read/write IO and disk space usage by 2x-5x comparing to Prometheus remote write format.
+
+`vmagent` automatically switches to VictoriaMetrics remote write protocol when it sends data to VictoriaMetrics components such as other `vmagent` instances,
+[single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html)
+or `vminsert` at [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+It is possible to force switch to VictoriaMetrics remote write protocol by specifying `-remoteWrite.forceVMProto`
+command-line flag for the corresponding `-remoteWrite.url`.
+It is possible to tune the compression level for VictoriaMetrics remote write protocol with `-remoteWrite.vmProtoCompressLevel` command-line flag.
+Bigger values reduce network usage at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of higher network usage.
+
+`vmagent` automatically switches to Prometheus remote write protocol when it sends data to old versions of VictoriaMetrics components
+or to other Prometheus-compatible remote storage systems. It is possible to force switch to Prometheus remote write protocol
+by specifying `-remoteWrite.forcePromProto` command-line flag for the corresponding `-remoteWrite.url`.
+
 ## Multitenancy
 
 By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`.
@@ -607,18 +645,19 @@ provide the following tools for debugging target-level and metric-level relabeli
 - Target-level debugging (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug target relabeling` link at the target, which must be debugged.
-  The opened page will show step-by-step results for the actual target relabeling rules applied to the discovered target labels.
+  The opened page shows step-by-step results for the actual target relabeling rules applied to the discovered target labels.
+  The page shows also the target URL generated after applying all the relabeling rules.
 
   The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
   is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page
   (`http://victoriametrics:8428/service-discovery` for single-node VictoriaMetrics), find the dropped target
-  and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules,
+  and click the `debug` link there. The opened page shows step-by-step results for the actual relabeling rules,
   which result to target drop.
 
 - Metric-level debugging (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug metrics relabeling` link at the target, which must be debugged.
-  The opened page will show step-by-step results for the actual metric relabeling rules applied to the given target labels.
+  The opened page shows step-by-step results for the actual metric relabeling rules applied to the given target labels.
 
 ## Prometheus staleness markers
 
@@ -880,7 +919,7 @@ If you have suggestions for improvements or have found a bug - please open an is
   The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
 * Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling).
-  Such storage systems include Prometheus, Cortex and Thanos, which typically emit `out of order sample` errors.
+  Such storage systems include Prometheus, Mimir, Cortex and Thanos, which typically emit `out of order sample` errors.
   The best solution is to use remote storage with [backfilling support](https://docs.victoriametrics.com/#backfilling) such as VictoriaMetrics.
 
 * `vmagent` buffers scraped data at the `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
@@ -1147,7 +1186,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/#query-tracing
   -dryRun
-     Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
+     Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -1211,8 +1250,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
      The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringCacheExpireDuration duration
+     The expire duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -kafka.consumer.topic array
      Kafka topic names for data consumption. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1268,11 +1311,11 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+     TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
   -opentsdbHTTPListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+     TCP and UDP address to listen for OpenTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
   -opentsdbListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
@@ -1340,6 +1383,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      How frequently to reload the full state from Kubernetes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
   -promscrape.maxResponseHeadersSize size
@@ -1362,7 +1407,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.seriesLimitPerTarget int
      Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
-     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
@@ -1420,6 +1465,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.flushInterval duration
      Interval for flushing the data to remote storage. This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url (default 1s)
+  -remoteWrite.forcePromProto array
+     Whether to force Prometheus remote write protocol for sending data to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
+     Supports array of values separated by comma or specified via multiple flags.
+  -remoteWrite.forceVMProto array
+     Whether to force VictoriaMetrics remote write protocol for sending data to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.headers array
      Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1467,6 +1518,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
      Optional path to file with relabeling configs, which are applied to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
+  -remoteWrite.keepDanglingQueues
+     Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.
@@ -1505,11 +1558,13 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.tmpDataPath string
      Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-     Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
+     Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
      Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
      Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.vmProtoCompressLevel int
+     The compression level for VictoriaMetrics remote write protocol. Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol
   -sortLabels
      Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit
   -tls

app/vmagent/csvimport/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,7 +26,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/datadog/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -28,7 +29,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }

app/vmagent/graphite/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,7 +20,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/influx/request_handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -36,7 +37,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(nil, db, rows, nil)
 	})
 }
@@ -54,7 +55,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/main.go

@@ -46,7 +46,8 @@ var (
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
 		"See also -influxListenAddr.useProxyProtocol")
@@ -56,18 +57,18 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
 	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
+	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
@@ -98,17 +99,20 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
 		return
 	}
 	if *dryRun {
-		if err := remotewrite.CheckRelabelConfigs(); err != nil {
-			logger.Fatalf("error when checking relabel configs: %s", err)
-		}
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exitting with 0 status code")
+		if err := remotewrite.CheckRelabelConfigs(); err != nil {
+			logger.Fatalf("error when checking relabel configs: %s", err)
+		}
+		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+		}
+		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
 	}
 
@@ -208,7 +212,7 @@ func getAuthTokenFromPath(path string) (*auth.Token, error) {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
@@ -253,6 +257,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write":
+		if common.HandleVMProtoServerHandshake(w, r) {
+			return true
+		}
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()

app/vmagent/native/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,12 +30,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmagent/opentsdb/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,7 +20,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/opentsdbhttp/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -24,7 +25,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/prometheusimport/request_handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -31,7 +32,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vmagent/prometheusimport/request_handler_test.go

@@ -23,7 +23,7 @@ var (
 func TestInsertHandler(t *testing.T) {
 	setUp()
 	defer tearDown()
-	req := httptest.NewRequest("POST", "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
+	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
 		t.Errorf("unxepected error %s", err)

app/vmagent/promremotewrite/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -27,7 +27,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/remotewrite/client.go

@@ -15,11 +15,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
+	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
@@ -69,8 +75,12 @@ var (
 type client struct {
 	sanitizedURL   string
 	remoteWriteURL string
-	fq             *persistentqueue.FastQueue
-	hc             *http.Client
+
+	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
+	useVMProto bool
+
+	fq *persistentqueue.FastQueue
+	hc *http.Client
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
@@ -122,19 +132,39 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+	hc := &http.Client{
+		Transport: tr,
+		Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
 		awsCfg:         awsCfg,
 		fq:             fq,
-		hc: &http.Client{
-			Transport: tr,
-			Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
-		},
-		stopCh: make(chan struct{}),
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
+
+	useVMProto := forceVMProto.GetOptionalArg(argIdx)
+	usePromProto := forcePromProto.GetOptionalArg(argIdx)
+	if useVMProto && usePromProto {
+		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
+	}
+	if !useVMProto && !usePromProto {
+		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
+		doRequest := func(url string) (*http.Response, error) {
+			return c.doRequest(url, nil)
+		}
+		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
+		if !useVMProto {
+			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
+				"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol", sanitizedURL)
+		}
+	}
+	c.useVMProto = useVMProto
+
 	return c
 }
 
@@ -291,38 +321,45 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlockHTTP returns false only if c.stopCh is closed.
-// Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.register(len(block), c.stopCh)
-	retryDuration := time.Second
-	retriesCount := 0
-	c.bytesSent.Add(len(block))
-	c.blocksSent.Inc()
-	sigv4Hash := ""
-	if c.awsCfg != nil {
-		sigv4Hash = awsapi.HashHex(block)
-	}
-
-again:
-	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
+func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+	reqBody := bytes.NewBuffer(body)
+	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
 	}
 	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	h.Set("Content-Encoding", "snappy")
-	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.useVMProto {
+		h.Set("Content-Encoding", "zstd")
+		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	} else {
+		h.Set("Content-Encoding", "snappy")
+		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	}
 	if c.awsCfg != nil {
+		sigv4Hash := awsapi.HashHex(body)
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
 			// there is no need in retry, request will be rejected by client.Do and retried by code below
 			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
+	return c.hc.Do(req)
+}
+
+// sendBlockHTTP sends the given block to c.remoteWriteURL.
+//
+// The function returns false only if c.stopCh is closed.
+// Otherwise it tries sending the block to remote storage indefinitely.
+func (c *client) sendBlockHTTP(block []byte) bool {
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
+	retriesCount := 0
+
+again:
 	startTime := time.Now()
-	resp, err := c.hc.Do(req)
+	resp, err := c.doRequest(c.remoteWriteURL, block)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
@@ -347,6 +384,8 @@ again:
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
 		c.requestsOKCount.Inc()
+		c.bytesSent.Add(len(block))
+		c.blocksSent.Inc()
 		return true
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()

app/vmagent/remotewrite/pendingseries.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -23,6 +24,9 @@ var (
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
+	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
+		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
+		"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -33,9 +37,10 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(pushBlock func(block []byte), significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte), isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.pushBlock = pushBlock
+	ps.wr.isVMRemoteWrite = isVMRemoteWrite
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
@@ -88,6 +93,9 @@ type writeRequest struct {
 	// pushBlock is called when whe write request is ready to be sent.
 	pushBlock func(block []byte)
 
+	// Whether to encode the write request with VictoriaMetrics remote write protocol.
+	isVMRemoteWrite bool
+
 	// How many significant figures must be left before sending the writeRequest to pushBlock.
 	significantFigures int
 
@@ -104,7 +112,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset pushBlock, significantFigures and roundDigits, since they are re-used.
+	// Do not reset lastFlushTime, pushBlock, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -126,7 +134,7 @@ func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
 	wr.adjustSampleValues()
 	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
-	pushWriteRequest(&wr.wr, wr.pushBlock)
+	pushWriteRequest(&wr.wr, wr.pushBlock, wr.isVMRemoteWrite)
 	wr.reset()
 }
 
@@ -188,7 +196,7 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte), isVMRemoteWrite bool) {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return
@@ -197,7 +205,11 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
 		zb := snappyBufPool.Get()
-		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		if isVMRemoteWrite {
+			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
+		} else {
+			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		}
 		writeRequestBufPool.Put(bb)
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
 			pushBlock(zb.B)
@@ -221,18 +233,18 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		pushWriteRequest(wr, pushBlock)
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples[n:]
-		pushWriteRequest(wr, pushBlock)
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples
 		return
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	pushWriteRequest(wr, pushBlock)
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries[n:]
-	pushWriteRequest(wr, pushBlock)
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries
 }
 

app/vmagent/remotewrite/pendingseries_test.go

@@ -2,40 +2,48 @@ package remotewrite
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/golang/snappy"
 )
 
 func TestPushWriteRequest(t *testing.T) {
-	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
+	for i, rowsCount := range rowsCounts {
+		expectedBlockLenProm := expectedBlockLensProm[i]
+		expectedBlockLenVM := expectedBlockLensVM[i]
 		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
-			testPushWriteRequest(t, rowsCount)
+			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
 		})
 	}
 }
 
-func testPushWriteRequest(t *testing.T, rowsCount int) {
-	wr := newTestWriteRequest(rowsCount, 10)
-	pushBlockLen := 0
-	pushBlock := func(block []byte) {
-		if pushBlockLen > 0 {
-			panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
+	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
+		t.Helper()
+		wr := newTestWriteRequest(rowsCount, 20)
+		pushBlockLen := 0
+		pushBlock := func(block []byte) {
+			if pushBlockLen > 0 {
+				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+			}
+			pushBlockLen = len(block)
+		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
+			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
+				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
 		}
-		pushBlockLen = len(block)
-	}
-	pushWriteRequest(wr, pushBlock)
-	b := prompbmarshal.MarshalWriteRequest(nil, wr)
-	zb := snappy.Encode(nil, b)
-	maxPushBlockLen := len(zb)
-	minPushBlockLen := maxPushBlockLen / 2
-	if pushBlockLen < minPushBlockLen {
-		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be at least %d bytes", pushBlockLen, minPushBlockLen)
-	}
-	if pushBlockLen > maxPushBlockLen {
-		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be smaller or equal to %d bytes", pushBlockLen, maxPushBlockLen)
 	}
+
+	// Check Prometheus remote write
+	f(false, expectedBlockLenProm, 0)
+
+	// Check VictoriaMetrics remote write
+	f(true, expectedBlockLenVM, 15)
 }
 
 func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {

app/vmagent/remotewrite/remotewrite.go

@@ -4,17 +4,22 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -24,18 +29,19 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 )
 
 var (
-	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
-		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
+		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
 	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
+	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
+		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -94,6 +100,8 @@ var allRelabelConfigs atomic.Value
 // since it may lead to high memory usage due to big number of buffers.
 var maxQueues = cgroup.AvailableCPUs() * 16
 
+const persistentQueueDirname = "persistent-queue"
+
 // InitSecretFlags must be called after flag.Parse and before any logging.
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
@@ -150,9 +158,8 @@ func Init() {
 		logger.Fatalf("cannot load relabel configs: %s", err)
 	}
 	allRelabelConfigs.Store(rcs)
-
-	configSuccess.Set(1)
-	configTimestamp.Set(fasttime.UnixTimestamp())
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
 	if len(*remoteWriteURLs) > 0 {
 		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
@@ -165,34 +172,56 @@ func Init() {
 		for {
 			select {
 			case <-sighupCh:
-			case <-stopCh:
+			case <-configReloaderStopCh:
 				return
 			}
-			configReloads.Inc()
-			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-			rcs, err := loadRelabelConfigs()
-			if err != nil {
-				configReloadErrors.Inc()
-				configSuccess.Set(0)
-				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-				continue
-			}
-
-			allRelabelConfigs.Store(rcs)
-			configSuccess.Set(1)
-			configTimestamp.Set(fasttime.UnixTimestamp())
-			logger.Infof("Successfully reloaded relabel configs")
+			reloadRelabelConfigs()
+			reloadStreamAggrConfigs()
 		}
 	}()
 }
 
+func reloadRelabelConfigs() {
+	relabelConfigReloads.Inc()
+	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		relabelConfigReloadErrors.Inc()
+		relabelConfigSuccess.Set(0)
+		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+		return
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	logger.Infof("successfully reloaded relabel configs")
+}
+
 var (
-	configReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	configReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	configSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
-	configTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
+	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
+func reloadStreamAggrConfigs() {
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for _, rwctxs := range rwctxsMap {
+			reinitStreamAggr(rwctxs)
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		reinitStreamAggr(rwctxsDefault)
+	}
+}
+
+func reinitStreamAggr(rwctxs []*remoteWriteCtx) {
+	for _, rwctx := range rwctxs {
+		rwctx.reinitStreamAggr()
+	}
+}
+
 func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
@@ -225,17 +254,47 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
+
+	if !*keepDanglingQueues {
+		// Remove dangling queues, if any.
+		// This is required for the case when the number of queues has been changed or URL have been changed.
+		// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
+		existingQueues := make(map[string]struct{}, len(rwctxs))
+		for _, rwctx := range rwctxs {
+			existingQueues[rwctx.fq.Dirname()] = struct{}{}
+		}
+
+		queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
+		files, err := os.ReadDir(queuesDir)
+		if err != nil {
+			logger.Fatalf("cannot read queues dir %q: %s", queuesDir, err)
+		}
+		removed := 0
+		for _, f := range files {
+			dirname := f.Name()
+			if _, ok := existingQueues[dirname]; !ok {
+				logger.Infof("removing dangling queue %q", dirname)
+				fullPath := filepath.Join(queuesDir, dirname)
+				fs.MustRemoveAll(fullPath)
+				removed++
+			}
+		}
+		if removed > 0 {
+			logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		}
+	}
+
 	return rwctxs
 }
 
-var stopCh = make(chan struct{})
+var configReloaderStopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
 // Stop stops remotewrite.
 //
 // It is expected that nobody calls Push during and after the call to this func.
 func Stop() {
-	close(stopCh)
+	close(configReloaderStopCh)
 	configReloaderWG.Wait()
 
 	for _, rwctx := range rwctxsDefault {
@@ -450,7 +509,7 @@ type remoteWriteCtx struct {
 	fq  *persistentqueue.FastQueue
 	c   *client
 
-	sas                 *streamaggr.Aggregators
+	sas                 atomic.Pointer[streamaggr.Aggregators]
 	streamAggrKeepInput bool
 
 	pss        []*pendingSeries
@@ -466,7 +525,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	pqURL.RawQuery = ""
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
-	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
 	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -475,6 +534,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
+
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
@@ -495,7 +555,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq.MustWriteBlock, sf, rd)
+		pss[i] = newPendingSeries(fq.MustWriteBlock, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{
@@ -514,10 +574,12 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(argIdx, 0)
 		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggrFile=%q: %s", sasFile, err)
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
-		rwctx.sas = sas
+		rwctx.sas.Store(sas)
 		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
 	}
 
 	return rwctx
@@ -532,8 +594,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
-	rwctx.sas.MustStop()
-	rwctx.sas = nil
+
+	sas := rwctx.sas.Swap(nil)
+	sas.MustStop()
+
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
@@ -564,8 +628,9 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 
 	// Apply stream aggregation if any
-	rwctx.sas.Push(tss)
-	if rwctx.sas == nil || rwctx.streamAggrKeepInput {
+	sas := rwctx.sas.Load()
+	sas.Push(tss)
+	if sas == nil || rwctx.streamAggrKeepInput {
 		// Push samples to the remote storage
 		rwctx.pushInternal(tss)
 	}
@@ -584,6 +649,36 @@ func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	pss[idx].Push(tss)
 }
 
+func (rwctx *remoteWriteCtx) reinitStreamAggr() {
+	sas := rwctx.sas.Load()
+	if sas == nil {
+		// There is no stream aggregation for rwctx
+		return
+	}
+
+	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
+	dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(rwctx.idx, 0)
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
+		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
+		return
+	}
+	if !sasNew.Equal(sas) {
+		sasOld := rwctx.sas.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded stream aggregation configs at -remoteWrite.streamAggr.config=%q", sasFile)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("the config at -remoteWrite.streamAggr.config=%q wasn't changed", sasFile)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+}
+
 var tssRelabelPool = &sync.Pool{
 	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
@@ -598,3 +693,20 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 	}
 	return rowsCount
 }
+
+// CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
+func CheckStreamAggrConfigs() error {
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
+	for idx, sasFile := range *streamAggrConfig {
+		if sasFile == "" {
+			continue
+		}
+		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(idx, 0)
+		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
+		if err != nil {
+			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
+		}
+		sas.MustStop()
+	}
+	return nil
+}

app/vmagent/vmimport/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmalert/Makefile

@@ -114,6 +114,9 @@ vmalert-linux-arm64:
 vmalert-linux-ppc64le:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmalert-linux-s390x:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmalert-linux-386:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert/README.md

@@ -28,7 +28,8 @@ Use this feature for the following cases:
 * Graphite datasource can be used for alerting and recording rules. See [these docs](#graphite);
 * Recording and Alerting rules backfilling (aka `replay`). See [these docs](#rules-backfilling);
 * Lightweight and without extra dependencies.
-* Supports [reusable templates](#reusable-templates) for annotations.
+* Supports [reusable templates](#reusable-templates) for annotations;
+* Load of recording and alerting rules from local filesystem, GCS and S3.
 
 ## Limitations
 
@@ -404,6 +405,25 @@ The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz`
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
+### Reading rules from object storage
+
+[Enterprise version](https://docs.victoriametrics.com/enterprise.html) of `vmalert` may read alerting and recording rules
+from object storage:
+
+- `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
+- `./bin/vmalert -rule=gs://bucket/bir/alert.rules` would read rules from the given path at GCS bucket
+
+S3 and GCS paths support only matching by prefix, e.g. `s3://bucket/dir/rule_` matches
+all files with prefix `rule_` in the folder `dir`.
+
+The following [command-line flags](#flags) can be used for fine-tuning access to S3 and GCS:
+
+- `-s3.credsFilePath` - path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+- `-s3.configFilePath` - path to file with S3 configs. Configs are loaded from default location if not set.
+- `-s3.configProfile` - profile name for S3 configs. If no set, the value of the environment variable will be loaded (`AWS_PROFILE` or `AWS_DEFAULT_PROFILE`).
+- `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
+- `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
+
 ### Topology examples
 
 The following sections are showing how `vmalert` may be used and configured
@@ -601,6 +621,12 @@ can read the same rules configuration as normal, evaluate them on the given time
 results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
 data source for backfilling.
 
+Please note, that response caching may lead to unexpected results during and after backfilling process.
+In order to avoid this you need to reset cache contents or disable caching when using backfilling
+as described in [backfilling docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#backfilling).
+
+See a blogpost about [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/).
+
 ### How it works
 
 In `replay` mode vmalert works as a cli-tool and exits immediately after work is done.
@@ -705,31 +731,42 @@ a review to the dashboard.
 
 ## Troubleshooting
 
-vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed,
-the data is already present in configured `-datasource.url`:
+### Data delay
+
+Data delay is one of the most common issues with rules execution.
+vmalert executes configured rules within certain intervals at specifics timestamps. 
+It expects that the data is already present in configured `-datasource.url` at the moment of time when rule is executed:
 
 <img alt="vmalert expected evaluation" src="vmalert_ts_normal.gif">
 
 Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations
-may get empty response from datasource and produce empty recording rules or reset alerts state:
+may get empty response from the datasource and produce empty recording rules or reset alerts state:
 
 <img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
 
-By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
-This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
-Usually, this results into a 30s shift for recording rules results.
-Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
+Try the following recommendations to reduce the chance of hitting the data delay issue:
 
-Try the following recommendations in such cases:
-
-* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics
-are delivered to the datasource;
+* Always configure group's `evaluationInterval` to be bigger or at least equal to 
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution);
+* Ensure that `[duration]` value is at least twice bigger than 
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution). For example,
+if expression is `rate(my_metric[2m]) > 0` then ensure that `my_metric` resolution is at least `1m` or better `30s`. 
+If you use VictoriaMetrics as datasource, `[duration]` can be omitted and VictoriaMetrics will adjust it automatically.
 * If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback`
-command-line flag to add a time shift for evaluations;
-* If time intervals between datapoints in datasource are irregular or `>=5min` - try changing vmalert's
-`-datasource.queryStep` command-line flag to specify how far search query can lookback for the recent datapoint. 
-The recommendation is to have the step at least two times bigger than `scrape_interval`, since
-there are no guarantees that scrape will not fail.
+command-line flag to add a time shift for evaluations. Or extend `[duration]` to tolerate the delay.
+For example, `max_over_time(errors_total[10m]) > 0` will be active even if there is no data in datasource for last `9m`.
+* If [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution)
+in datasource is inconsistent or `>=5min` - try changing vmalert's `-datasource.queryStep` command-line flag to specify 
+how far search query can lookback for the recent datapoint. The recommendation is to have the step 
+at least two times bigger than the resolution.
+
+> Please note, data delay is inevitable in distributed systems. And it is better to account for it instead of ignoring.
+
+By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s
+(see `-search.latencyOffset` command-line flag at vmselect). Such delay is needed to eliminate risk of incomplete
+data on the moment of querying, since metrics collectors won't be able to deliver the data in time.
+
+### Alerts state
 
 Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that
 alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only
@@ -752,6 +789,8 @@ HTTP request sent by vmalert to the `-datasource.url` during evaluation. If spec
 no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
 moment when rule was evaluated.
 
+### Debug mode
+
 vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration
 and vmalert will start printing additional log messages:
 ```terminal
@@ -877,7 +916,7 @@ The shortlist of configuration flags is the following:
   -evaluationInterval duration
      How often to evaluate the rules (default 1m0s)
   -external.alert.source string
-     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
+     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
   -external.label array
      Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
      Supports an array of values separated by comma or specified via multiple flags.
@@ -907,6 +946,10 @@ The shortlist of configuration flags is the following:
      Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880")
   -httpListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+  -insert.maxQueueDuration duration
+     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -923,6 +966,13 @@ The shortlist of configuration flags is the following:
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentInserts int
+     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
+  -memory.allowedBytes size
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
+  -memory.allowedPercent float
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
@@ -1089,8 +1139,8 @@ The shortlist of configuration flags is the following:
      Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
-  -replay.maxDatapointsPerQuery int
-     Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
+  -replay.maxDatapointsPerQuery /query_range
+     Max number of data points expected in one request. It affects the max time range for every /query_range request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
      Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration
@@ -1100,18 +1150,24 @@ The shortlist of configuration flags is the following:
   -replay.timeTo string
      The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
-     Path to the file with alert rules.
-     Supports patterns. Flag can be specified multiple times.
+     Path to the files with alerting and/or recording rules.
+     Supports hierarchical patterns and regexpes.
      Examples:
       -rule="/path/to/file". Path to a single file with alerting rules
-      -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-     absolute path to all .yaml files in root.
+      -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
      Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+     
+     Enterprise version of vmalert supports S3 and GCS paths to rules.
+     For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
+     S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
+     all files with prefix rule_ in folder dir.
+     See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
+     
      Supports an array of values separated by comma or specified via multiple flags.
   -rule.configCheckInterval duration
      Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
   -rule.maxResolveDuration duration
-     Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
+     Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group.
   -rule.resendDelay duration
      Minimum amount of time to wait before resending an alert to notifier
   -rule.templates array
@@ -1128,6 +1184,18 @@ The shortlist of configuration flags is the following:
      Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
      Whether to validate annotation and label templates (default true)
+  -s3.configFilePath string
+     Path to file with S3 configs. Configs are loaded from default location if not set.
+     See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.configProfile string
+     Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.credsFilePath string
+     Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+     See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.customEndpoint string
+     Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -s3.forcePathStyle
+     Prefixing endpoint with bucket name when set false, true by default. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default true)
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string

app/vmalert/config/config.go

@@ -5,16 +5,14 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
-	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -201,21 +199,45 @@ func (r *Rule) Validate() error {
 // ValidateTplFn must validate the given annotations
 type ValidateTplFn func(annotations map[string]string) error
 
+// cLogger is a logger with support of logs suppressing.
+// it is used when logs emitted by config package needs
+// to be suppressed.
+var cLogger = &log.Logger{}
+
+// ParseSilent parses rule configs from given file patterns without emitting logs
+func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	cLogger.Suppress(true)
+	defer cLogger.Suppress(false)
+
+	files, err := readFromFS(pathPatterns)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
+	}
+	return parse(files, validateTplFn, validateExpressions)
+}
+
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	var fp []string
-	for _, pattern := range pathPatterns {
-		matches, err := filepath.Glob(pattern)
-		if err != nil {
-			return nil, fmt.Errorf("error reading file pattern %s: %w", pattern, err)
-		}
-		fp = append(fp, matches...)
+	files, err := readFromFS(pathPatterns)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
+	groups, err := parse(files, validateTplFn, validateExpressions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
+	}
+	if len(groups) < 1 {
+		cLogger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	errGroup := new(utils.ErrGroup)
 	var groups []Group
-	for _, file := range fp {
+	for file, data := range files {
 		uniqueGroups := map[string]struct{}{}
-		gr, err := parseFile(file)
+		gr, err := parseConfig(data)
 		if err != nil {
 			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
 			continue
@@ -237,20 +259,19 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
 	if err := errGroup.Err(); err != nil {
 		return nil, err
 	}
-	if len(groups) < 1 {
-		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
-	}
+	sort.SliceStable(groups, func(i, j int) bool {
+		if groups[i].File != groups[j].File {
+			return groups[i].File < groups[j].File
+		}
+		return groups[i].Name < groups[j].Name
+	})
 	return groups, nil
 }
 
-func parseFile(path string) ([]Group, error) {
-	data, err := os.ReadFile(path)
+func parseConfig(data []byte) ([]Group, error) {
+	data, err := envtemplate.ReplaceBytes(data)
 	if err != nil {
-		return nil, fmt.Errorf("error reading alert rule file %q: %w", path, err)
-	}
-	data, err = envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
+		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
 	g := struct {
 		Groups []Group `yaml:"groups"`

app/vmalert/config/fs.go

@@ -0,0 +1,108 @@
+package config
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
+)
+
+// FS represent a file system abstract for reading files.
+type FS interface {
+	// Init initializes FS.
+	Init() error
+
+	// String must return human-readable representation of FS.
+	String() string
+
+	// List returns the list of file names which will be read via Read fn
+	List() ([]string, error)
+
+	// Read returns a list of read files in form of a map
+	// where key is a file name and value is a content of read file.
+	// Read must be called only after the successful Init call.
+	Read(files []string) (map[string][]byte, error)
+}
+
+var (
+	fsRegistryMu sync.Mutex
+	fsRegistry   = make(map[string]FS)
+)
+
+// readFromFS parses the given path list and inits FS for each item.
+// Once inited, readFromFS will try to read and return files from each FS.
+// readFromFS returns an error if at least one FS failed to init.
+// The function can be called multiple times but each unique path
+// will be inited only once.
+//
+// It is allowed to mix different FS types in path list.
+func readFromFS(paths []string) (map[string][]byte, error) {
+	var err error
+	result := make(map[string][]byte)
+	for _, path := range paths {
+
+		fsRegistryMu.Lock()
+		fs, ok := fsRegistry[path]
+		if !ok {
+			fs, err = newFS(path)
+			if err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
+			}
+			if err := fs.Init(); err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
+			}
+			fsRegistry[path] = fs
+		}
+		fsRegistryMu.Unlock()
+
+		list, err := fs.List()
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files from %q", fs)
+		}
+
+		cLogger.Infof("found %d files to read from %q", len(list), fs)
+
+		if len(list) < 1 {
+			continue
+		}
+
+		ts := time.Now()
+		files, err := fs.Read(list)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
+		}
+		cLogger.Infof("finished reading %d files in %v from %q", len(list), time.Since(ts), fs)
+
+		for k, v := range files {
+			if _, ok := result[k]; ok {
+				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
+			}
+			result[k] = v
+		}
+	}
+	return result, nil
+}
+
+// newFS creates FS based on the give path.
+// Supported file systems are: fs
+func newFS(path string) (FS, error) {
+	scheme := "fs"
+	n := strings.Index(path, "://")
+	if n >= 0 {
+		scheme = path[:n]
+		path = path[n+len("://"):]
+	}
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	switch scheme {
+	case "fs":
+		return &fslocal.FS{Pattern: path}, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmalert/config/fs_test.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNewFS(t *testing.T) {
+	f := func(path, expStr string) {
+		t.Helper()
+		fs, err := newFS(path)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
+		if fs.String() != expStr {
+			t.Fatalf("expected FS %q; got %q", expStr, fs.String())
+		}
+	}
+
+	f("/foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+	f("fs:///foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+}
+
+func TestNewFSNegative(t *testing.T) {
+	f := func(path, expErr string) {
+		t.Helper()
+		_, err := newFS(path)
+		if err == nil {
+			t.Fatalf("expected to have err: %s", expErr)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Fatalf("expected to have err %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("", "path cannot be empty")
+	f("fs://", "path cannot be empty")
+	f("foobar://baz", `unsupported scheme "foobar"`)
+}

app/vmalert/config/fslocal/fslocal.go

@@ -0,0 +1,49 @@
+package fslocal
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// FS represents a local file system
+type FS struct {
+	// Pattern is used for matching one or multiple files.
+	// The pattern may describe hierarchical names such as
+	// /usr/*/bin/ed (assuming the Separator is '/').
+	Pattern string
+}
+
+// Init verifies that configured Pattern is correct
+func (fs *FS) Init() error {
+	_, err := filepath.Glob(fs.Pattern)
+	return err
+}
+
+// String implements Stringer interface
+func (fs *FS) String() string {
+	return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern)
+}
+
+// List returns the list of file names which will be read via Read fn
+func (fs *FS) List() ([]string, error) {
+	matches, err := filepath.Glob(fs.Pattern)
+	if err != nil {
+		return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err)
+	}
+	return matches, nil
+}
+
+// Read returns a map of read files where
+// key is the file name and value is file's content.
+func (fs *FS) Read(files []string) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	for _, path := range files {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading file %q: %w", path, err)
+		}
+		result[path] = data
+	}
+	return result, nil
+}

app/vmalert/config/log/logger.go

@@ -0,0 +1,59 @@
+package log
+
+import (
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Logger is using lib/logger for logging
+// but can be suppressed via Suppress method
+type Logger struct {
+	mu       sync.RWMutex
+	disabled bool
+}
+
+// Suppress whether to ignore message logging.
+// Once suppressed, logging continues to be ignored
+// until logger is un-suppressed.
+func (l *Logger) Suppress(v bool) {
+	l.mu.Lock()
+	l.disabled = v
+	l.mu.Unlock()
+}
+
+func (l *Logger) isDisabled() bool {
+	l.mu.RLock()
+	defer l.mu.RUnlock()
+	return l.disabled
+}
+
+// Errorf logs error message.
+func (l *Logger) Errorf(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Errorf(format, args...)
+}
+
+// Warnf logs warning message.
+func (l *Logger) Warnf(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Warnf(format, args...)
+}
+
+// Infof logs info message.
+func (l *Logger) Infof(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Infof(format, args...)
+}
+
+// Panicf logs panic message and panics.
+// Panicf can't be suppressed
+func (l *Logger) Panicf(format string, args ...interface{}) {
+	logger.Panicf(format, args...)
+}

app/vmalert/config/log/logger_test.go

@@ -0,0 +1,54 @@
+package log
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+func TestOutput(t *testing.T) {
+	testOutput := &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+	defer logger.ResetOutputForTest()
+
+	log := &Logger{}
+
+	mustMatch := func(exp string) {
+		t.Helper()
+		if exp == "" {
+			if testOutput.String() != "" {
+				t.Errorf("expected output to be empty; got %q", testOutput.String())
+				return
+			}
+		}
+		if !strings.Contains(testOutput.String(), exp) {
+			t.Errorf("output %q should contain %q", testOutput.String(), exp)
+		}
+		fmt.Println(testOutput.String())
+		testOutput.Reset()
+	}
+
+	log.Warnf("foo")
+	mustMatch("foo")
+
+	log.Infof("info %d", 2)
+	mustMatch("info 2")
+
+	log.Errorf("error %s %d", "baz", 5)
+	mustMatch("error baz 5")
+
+	log.Suppress(true)
+
+	log.Warnf("foo")
+	mustMatch("")
+
+	log.Infof("info %d", 2)
+	mustMatch("")
+
+	log.Errorf("error %q %d", "baz", 5)
+	mustMatch("")
+
+}

app/vmalert/datasource/vm.go

@@ -174,7 +174,7 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 }
 
 func (s *VMStorage) newRequestPOST() (*http.Request, error) {
-	req, err := http.NewRequest("POST", s.datasourceURL, nil)
+	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
 		return nil, err
 	}

app/vmalert/group.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"net/url"
@@ -44,6 +45,9 @@ type Group struct {
 	// channel accepts new Group obj
 	// which supposed to update current group
 	updateCh chan *Group
+	// evalCancel stores the cancel fn for interrupting
+	// rules evaluation. Used on groups update() and close().
+	evalCancel context.CancelFunc
 
 	metrics *groupMetrics
 }
@@ -233,11 +237,24 @@ func (g *Group) updateWith(newGroup *Group) error {
 	return nil
 }
 
+// interruptEval interrupts in-flight rules evaluations
+// within the group. It is expected that g.evalCancel
+// will be repopulated after the call.
+func (g *Group) interruptEval() {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	if g.evalCancel != nil {
+		g.evalCancel()
+	}
+}
+
 func (g *Group) close() {
 	if g.doneCh == nil {
 		return
 	}
 	close(g.doneCh)
+	g.interruptEval()
 	<-g.finishedCh
 
 	g.metrics.iterationDuration.Unregister()
@@ -254,6 +271,26 @@ var skipRandSleepOnGroupStart bool
 func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
+	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	if !skipRandSleepOnGroupStart {
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(g.Interval)
+		}
+		randSleep -= sleepOffset
+		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		select {
+		case <-ctx.Done():
+			sleepTimer.Stop()
+			return
+		case <-g.doneCh:
+			sleepTimer.Stop()
+			return
+		case <-sleepTimer.C:
+		}
+	}
+
 	e := &executor{
 		rw:                       rw,
 		notifiers:                nts,
@@ -263,7 +300,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 
-	eval := func(ts time.Time) {
+	eval := func(ctx context.Context, ts time.Time) {
 		g.metrics.iterationTotal.Inc()
 
 		start := time.Now()
@@ -285,7 +322,13 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		g.LastEvaluation = start
 	}
 
-	eval(evalTS)
+	evalCtx, cancel := context.WithCancel(ctx)
+	g.mu.Lock()
+	g.evalCancel = cancel
+	g.mu.Unlock()
+	defer g.evalCancel()
+
+	eval(evalCtx, evalTS)
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
@@ -309,6 +352,14 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			return
 		case ng := <-g.updateCh:
 			g.mu.Lock()
+
+			// it is expected that g.evalCancel will be evoked
+			// somewhere else to unblock group from the rules evaluation.
+			// we recreate the evalCtx and g.evalCancel, so it can
+			// be called again.
+			evalCtx, cancel = context.WithCancel(ctx)
+			g.evalCancel = cancel
+
 			err := g.updateWith(ng)
 			if err != nil {
 				logger.Errorf("group %q: failed to update: %s", g.Name, err)
@@ -333,7 +384,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			}
 			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
-			eval(evalTS)
+			eval(evalCtx, evalTS)
 		}
 	}
 }
@@ -407,6 +458,11 @@ func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDur
 
 	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			// the context can be cancelled on graceful shutdown
+			// or on group update. So no need to handle the error as usual.
+			return nil
+		}
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
 	}

app/vmalert/group_test.go

@@ -474,3 +474,31 @@ func TestFaultyRW(t *testing.T) {
 		t.Fatalf("expected to get an error from faulty RW client, got nil instead")
 	}
 }
+
+func TestCloseWithEvalInterruption(t *testing.T) {
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("failed to parse rules: %s", err)
+	}
+
+	const delay = time.Second * 2
+	fq := &fakeQuerierWithDelay{delay: delay}
+
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], fq, evalInterval, nil)
+
+	go g.start(context.Background(), nil, nil, nil)
+
+	time.Sleep(evalInterval * 20)
+
+	go func() {
+		g.close()
+	}()
+
+	deadline := time.Tick(delay / 2)
+	select {
+	case <-deadline:
+		t.Fatalf("deadline for close exceeded")
+	case <-g.finishedCh:
+	}
+}

app/vmalert/helpers_test.go

@@ -104,6 +104,24 @@ func (fqr *fakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time
 	return cp, req, nil
 }
 
+type fakeQuerierWithDelay struct {
+	fakeQuerier
+	delay time.Duration
+}
+
+func (fqd *fakeQuerierWithDelay) Query(ctx context.Context, expr string, ts time.Time) ([]datasource.Metric, *http.Request, error) {
+	timer := time.NewTimer(fqd.delay)
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+	return fqd.fakeQuerier.Query(ctx, expr, ts)
+}
+
+func (fqd *fakeQuerierWithDelay) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqd
+}
+
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert

app/vmalert/main.go

@@ -28,13 +28,19 @@ import (
 )
 
 var (
-	rulePath = flagutil.NewArrayString("rule", `Path to the file with alert rules.
-Supports patterns. Flag can be specified multiple times.
+	rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules.
+Supports hierarchical patterns and regexpes.
 Examples:
  -rule="/path/to/file". Path to a single file with alerting rules
- -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-absolute path to all .yaml files in root.
-Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`)
+ -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
+Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+
+Enterprise version of vmalert supports S3 and GCS paths to rules.
+For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
+S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
+all files with prefix rule_ in folder dir.
+See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
+`)
 
 	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
 	for rules annotations templating. Flag can be specified multiple times.
@@ -51,13 +57,14 @@ absolute path to all .tpl files in root.`)
 
 	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
 	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
-		"which is by default equal to 3 evaluation intervals of the parent group.")
+		"which by default is 4 times evaluationInterval of the parent group.")
 	resendDelay            = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param.")
@@ -66,7 +73,8 @@ absolute path to all .tpl files in root.`)
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
 		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
-		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
+		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
@@ -312,6 +320,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 	// init reload metrics with positive values to improve alerting conditions
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
+	parseFn := config.Parse
 	for {
 		select {
 		case <-ctx.Done():
@@ -323,7 +332,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			}
 			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
 			configReloads.Inc()
+			// allow logs emitting during manual config reload
+			parseFn = config.Parse
 		case <-configCheckCh:
+			// disable logs emitting during per-interval config reload
+			parseFn = config.ParseSilent
 		}
 		if err := notifier.Reload(); err != nil {
 			configReloadErrors.Inc()
@@ -338,7 +351,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
+		newGroupsCfg, err := parseFn(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)

app/vmalert/manager.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"sort"
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -87,32 +86,12 @@ func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error
 	m.wg.Add(1)
 	id := g.ID()
 	go func() {
-		// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
-		if !skipRandSleepOnGroupStart {
-			randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
-			sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
-			if randSleep < sleepOffset {
-				randSleep += uint64(g.Interval)
-			}
-			randSleep -= sleepOffset
-			sleepTimer := time.NewTimer(time.Duration(randSleep))
-			select {
-			case <-ctx.Done():
-				sleepTimer.Stop()
-				return
-			case <-g.doneCh:
-				sleepTimer.Stop()
-				return
-			case <-sleepTimer.C:
-			}
-		}
+		defer m.wg.Done()
 		if restore {
 			g.start(ctx, m.notifiers, m.rw, m.rr)
 		} else {
 			g.start(ctx, m.notifiers, m.rw, nil)
 		}
-
-		m.wg.Done()
 	}()
 	m.groups[id] = g
 	return nil
@@ -168,6 +147,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	}
 	for _, ng := range groupsRegistry {
 		if err := m.startGroup(ctx, ng, restore); err != nil {
+			m.groupsMu.Unlock()
 			return err
 		}
 	}
@@ -181,6 +161,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 				old.updateCh <- new
 				wg.Done()
 			}(item.old, item.new)
+			item.old.interruptEval()
 		}
 		wg.Wait()
 	}

app/vmalert/notifier/alert.go

@@ -41,7 +41,7 @@ type Alert struct {
 	LastSent time.Time
 	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	// ID is the unique identifer for the Alert
+	// ID is the unique identifier for the Alert
 	ID uint64
 	// Restored is true if Alert was restored after restart
 	Restored bool
@@ -111,11 +111,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		ActiveAt: a.ActiveAt,
 		For:      a.For,
 	}
-	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
-	if err != nil {
-		return nil, fmt.Errorf("error getting a template: %w", err)
-	}
-	return templateAnnotations(annotations, tplData, tmpl, true)
+	return ExecTemplate(q, annotations, tplData)
 }
 
 // ExecTemplate executes the given template for given annotations map.
@@ -174,6 +170,8 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
 	if err != nil {
 		return fmt.Errorf("error cloning template before parse annotation: %w", err)
 	}
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tpl = tpl.Option("missingkey=zero")
 	tpl, err = tpl.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation template: %w", err)

app/vmalert/notifier/alert_test.go

@@ -200,6 +200,9 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
+			if err := ValidateTemplates(tc.annotations); err != nil {
+				t.Fatal(err)
+			}
 			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
 			if err != nil {
 				t.Fatal(err)

app/vmalert/notifier/alertmanager.go

@@ -64,7 +64,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
-	req, err := http.NewRequest("POST", am.addr, b)
+	req, err := http.NewRequest(http.MethodPost, am.addr, b)
 	if err != nil {
 		return err
 	}

app/vmalert/notifier/config.go

@@ -29,7 +29,7 @@ type Config struct {
 	// ConsulSDConfigs contains list of settings for service discovery via Consul
 	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
 	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
-	// DNSSDConfigs ontains list of settings for service discovery via DNS.
+	// DNSSDConfigs contains list of settings for service discovery via DNS.
 	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
 	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
 

app/vmalert/notifier/init.go

@@ -81,10 +81,6 @@ var (
 //
 // Init returns an error if both mods are used.
 func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
-	if externalLabels != nil || externalURL != "" {
-		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
-	}
-
 	externalURL = extURL
 	externalLabels = extLabels
 	eu, err := url.Parse(externalURL)

app/vmalert/notifier/init_test.go

@@ -0,0 +1,37 @@
+package notifier
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"testing"
+)
+
+func TestInit(t *testing.T) {
+	oldAddrs := *addrs
+	defer func() { *addrs = oldAddrs }()
+
+	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
+
+	fn, err := Init(nil, nil, "")
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+
+	nfs := fn()
+	if len(nfs) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(nfs))
+	}
+
+	targets := GetTargets()
+	if targets == nil || targets[TargetStatic] == nil {
+		t.Fatalf("expected to get static targets in response")
+	}
+
+	nf1 := targets[TargetStatic][0]
+	if nf1.Addr() != "127.0.0.1/api/v2/alerts" {
+		t.Fatalf("expected to get \"127.0.0.1/api/v2/alerts\"; got %q instead", nf1.Addr())
+	}
+	nf2 := targets[TargetStatic][1]
+	if nf2.Addr() != "127.0.0.2/api/v2/alerts" {
+		t.Fatalf("expected to get \"127.0.0.2/api/v2/alerts\"; got %q instead", nf2.Addr())
+	}
+}

app/vmalert/remotewrite/init_test.go

@@ -0,0 +1,20 @@
+package remotewrite
+
+import (
+	"context"
+	"testing"
+)
+
+func TestInit(t *testing.T) {
+	oldAddr := *addr
+	defer func() { *addr = oldAddr }()
+
+	*addr = "http://localhost:8428"
+	cl, err := Init(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cl.Close(); err != nil {
+		t.Fatal(err)
+	}
+}

app/vmalert/remotewrite/remotewrite.go

@@ -225,7 +225,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 func (c *Client) send(ctx context.Context, data []byte) error {
 	r := bytes.NewReader(data)
-	req, err := http.NewRequest("POST", c.addr, r)
+	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}

app/vmalert/templates/template.go

@@ -225,7 +225,7 @@ func templateFuncs() textTpl.FuncMap {
 		"toLower": strings.ToLower,
 
 		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
-		// This funcion is deprectated.
+		// This function is deprecated.
 		//
 		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
 		// these functions properly escape `\n` and `\r` chars according to their purpose.

app/vmalert/templates/template_test.go

@@ -76,6 +76,20 @@ func TestTemplateFuncs(t *testing.T) {
 	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
 	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
 	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
+
+	formatting("humanize", float64(127087), "127.1k")
+	formatting("humanize", float64(136458627186688), "136.5T")
+
+	formatting("humanizeDuration", 1, "1s")
+	formatting("humanizeDuration", 0.2, "200ms")
+	formatting("humanizeDuration", 42000, "11h 40m 0s")
+	formatting("humanizeDuration", 16790555, "194d 8h 2m 35s")
+
+	formatting("humanizePercentage", 1, "100%")
+	formatting("humanizePercentage", 0.8, "80%")
+	formatting("humanizePercentage", 0.015, "1.5%")
+
+	formatting("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
 func mkTemplate(current, replacement interface{}) textTemplate {

app/vmalert/web.go

@@ -57,7 +57,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	switch r.URL.Path {
 	case "/", "/vmalert", "/vmalert/":
-		if r.Method != "GET" {
+		if r.Method != http.MethodGet {
 			httpserver.Errorf(w, r, "path %q supports only GET method", r.URL.Path)
 			return false
 		}

app/vmalert/web.qtpl

@@ -52,7 +52,7 @@
         {% for _, g := range groups  %}
               <div class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
-                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                  {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
                 <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.ID] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>

app/vmalert/web.qtpl.go

@@ -227,7 +227,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 //line app/vmalert/web.qtpl:55
 			qw422016.N().FPrec(g.Interval, 0)
 //line app/vmalert/web.qtpl:55
-			qw422016.N().S(`s)</a>
+			qw422016.N().S(`s) #</a>
                  `)
 //line app/vmalert/web.qtpl:56
 			if rNotOk[g.ID] > 0 {

app/vmalert/web_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
@@ -19,9 +20,18 @@ func TestHandler(t *testing.T) {
 		},
 		state: newRuleState(10),
 	}
+	ar.state.add(ruleStateEntry{
+		time:    time.Now(),
+		at:      time.Now(),
+		samples: 10,
+	})
+	rr := &RecordingRule{
+		Name:  "record",
+		state: newRuleState(10),
+	}
 	g := &Group{
 		Name:  "group",
-		Rules: []Rule{ar},
+		Rules: []Rule{ar, rr},
 	}
 	m := &manager{groups: make(map[uint64]*Group)}
 	m.groups[0] = g
@@ -62,6 +72,14 @@ func TestHandler(t *testing.T) {
 	t.Run("/vmalert/rule", func(t *testing.T) {
 		a := ar.ToAPI()
 		getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		r := rr.ToAPI()
+		getResp(ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
+	})
+	t.Run("/vmalert/alert", func(t *testing.T) {
+		alerts := ar.AlertsToAPI()
+		for _, a := range alerts {
+			getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		}
 	})
 	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)

app/vmauth/Makefile

@@ -84,6 +84,9 @@ vmauth-linux-arm64:
 vmauth-linux-ppc64le:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmauth-linux-s390x:
+	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmauth-linux-386:
 	APP_NAME=vmauth CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmauth/README.md

@@ -17,7 +17,12 @@ and pass the following flag to `vmauth` binary in order to start authorizing and
 After that `vmauth` starts accepting HTTP requests on port `8427` and routing them according to the provided [-auth.config](#auth-config).
 The port can be modified via `-httpListenAddr` command-line flag.
 
-The auth config can be reloaded either by passing `SIGHUP` signal to `vmauth` or by querying `/-/reload` http endpoint.
+The auth config can be reloaded via the following ways:
+
+- By passing `SIGHUP` signal to `vmauth`.
+- By querying `/-/reload` http endpoint. This endpoint can be protected with `-reloadAuthKey` command-line flag. See [security docs](#security) for more details.
+- By specifying `-configCheckInterval` command-line flag to the interval between config re-reads. For example, `-configCheckInterval=5s` will re-read the config
+  and apply new changes every 5 seconds.
 
 Docker images for `vmauth` are available [here](https://hub.docker.com/r/victoriametrics/vmauth/tags).
 
@@ -28,7 +33,36 @@ accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.co
 
 ## Load balancing
 
-Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls.
+In the latter case `vmauth` balances load among the configured urls in least-loaded round-robin manner.
+`vmauth` retries failing `GET` requests across the configured list of urls.
+This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes
+in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+
+## Concurrency limiting
+
+`vmauth` limits the number of concurrent requests it can proxy according to the following command-line flags:
+
+- `-maxConcurrentRequests` limits the global number of concurrent requests `vmauth` can serve across all the configured users.
+- `-maxConcurrentPerUserRequests` limits the number of concurrent requests `vmauth` can serve per each configured user.
+
+It is also possible to set individual limits on the number of concurrent requests per each user
+with the `max_concurrent_requests` option - see [auth config example](#auth-config).
+
+`vmauth` responds with `429 Too Many Requests` HTTP error when the number of concurrent requests exceeds the configured limits.
+
+The following [metrics](#monitoring) related to concurrency limits are exposed by `vmauth`:
+
+- `vmauth_concurrent_requests_capacity` - the global limit on the number of concurrent requests `vmauth` can serve.
+  It is set via `-maxConcurrentRequests` command-line flag.
+- `vmauth_concurrent_requests_current` - the current number of concurrent requests `vmauth` processes.
+- `vmauth_concurrent_requests_limit_reached_total` - the number of requests rejected with `429 Too Many Requests` error
+  because of the global concurrency limit has been reached.
+- `vmauth_user_concurrent_requests_capacity{username="..."}` - the limit on the number of concurrent requests for the given `username`.
+- `vmauth_user_concurrent_requests_current{username="..."}` - the current number of concurrent requests for the given `username`.
+- `vmauth_user_concurrent_requests_limit_reached_total{username="foo"}` - the number of requests rejected with `429 Too Many Requests` error
+  because of the concurrency limit has been reached for the given `username`.
+
 
 ## Auth config
 
@@ -55,26 +89,27 @@ users:
   headers:
   - "X-Scope-OrgID: foobar"
 
-  # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://localhost:8428 .
+  # are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  #
+  # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests.
+  # Excess concurrent requests are rejected with 429 HTTP status code.
+  # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags.
 - username: "local-single-node"
   password: "***"
   url_prefix: "http://localhost:8428"
+  max_concurrent_requests: 10
 
-  # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
   # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
-- username: "local-single-node"
+- username: "local-single-node2"
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
-  # The user for querying account 123 in VictoriaMetrics cluster
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+  # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
   #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
   #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
@@ -84,10 +119,8 @@ users:
   - "http://vmselect1:8481/select/123/prometheus"
   - "http://vmselect2:8481/select/123/prometheus"
 
-  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+  # are load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
   # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
   #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
   #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write
@@ -232,6 +265,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
      Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+  -configCheckInterval duration
+     Interval for config file re-read. Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -265,7 +300,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -httpListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
   -logInvalidAuthTokens
      Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
@@ -284,8 +319,10 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentPerUserRequests int
+     The maximum number of concurrent requests vmauth can process per each configured user. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option in per-user config (default 300)
   -maxConcurrentRequests int
-     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend (default 1000)
+     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options (default 1000)
   -maxIdleConnsPerBackend int
      The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
   -memory.allowedBytes size

app/vmauth/auth_config.go

@@ -11,8 +11,10 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -23,6 +25,8 @@ import (
 var (
 	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "interval for config file re-read. "+
+		"Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.")
 )
 
 // AuthConfig represents auth config.
@@ -32,17 +36,43 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name        string     `yaml:"name,omitempty"`
-	BearerToken string     `yaml:"bearer_token,omitempty"`
-	Username    string     `yaml:"username,omitempty"`
-	Password    string     `yaml:"password,omitempty"`
-	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
-	URLMaps     []URLMap   `yaml:"url_map,omitempty"`
-	Headers     []Header   `yaml:"headers,omitempty"`
+	Name                  string     `yaml:"name,omitempty"`
+	BearerToken           string     `yaml:"bearer_token,omitempty"`
+	Username              string     `yaml:"username,omitempty"`
+	Password              string     `yaml:"password,omitempty"`
+	URLPrefix             *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMaps               []URLMap   `yaml:"url_map,omitempty"`
+	Headers               []Header   `yaml:"headers,omitempty"`
+	MaxConcurrentRequests int        `yaml:"max_concurrent_requests,omitempty"`
+
+	concurrencyLimitCh      chan struct{}
+	concurrencyLimitReached *metrics.Counter
 
 	requests *metrics.Counter
 }
 
+func (ui *UserInfo) beginConcurrencyLimit() error {
+	select {
+	case ui.concurrencyLimitCh <- struct{}{}:
+		return nil
+	default:
+		ui.concurrencyLimitReached.Inc()
+		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+	}
+}
+
+func (ui *UserInfo) endConcurrencyLimit() {
+	<-ui.concurrencyLimitCh
+}
+
+func (ui *UserInfo) getMaxConcurrentRequests() int {
+	mcr := ui.MaxConcurrentRequests
+	if mcr <= 0 {
+		mcr = *maxConcurrentPerUserRequests
+	}
+	return mcr
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -83,16 +113,77 @@ type SrcPath struct {
 	re        *regexp.Regexp
 }
 
-// URLPrefix represents pased `url_prefix`
+// URLPrefix represents passed `url_prefix`
 type URLPrefix struct {
-	n    uint32
-	urls []*url.URL
+	n   uint32
+	bus []*backendURL
 }
 
-func (up *URLPrefix) getNextURL() *url.URL {
+type backendURL struct {
+	brokenDeadline     uint64
+	concurrentRequests int32
+	url                *url.URL
+}
+
+func (bu *backendURL) isBroken() bool {
+	ct := fasttime.UnixTimestamp()
+	return ct < atomic.LoadUint64(&bu.brokenDeadline)
+}
+
+func (bu *backendURL) setBroken() {
+	deadline := fasttime.UnixTimestamp() + 3
+	atomic.StoreUint64(&bu.brokenDeadline, deadline)
+}
+
+func (bu *backendURL) put() {
+	atomic.AddInt32(&bu.concurrentRequests, -1)
+}
+
+func (up *URLPrefix) getBackendsCount() int {
+	return len(up.bus)
+}
+
+// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+//
+// backendURL.put() must be called on the returned backendURL after the request is complete.
+func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
+	bus := up.bus
+	if len(bus) == 1 {
+		// Fast path - return the only backend url.
+		bu := bus[0]
+		atomic.AddInt32(&bu.concurrentRequests, 1)
+		return bu
+	}
+
+	// Slow path - select other backend urls.
 	n := atomic.AddUint32(&up.n, 1)
-	idx := n % uint32(len(up.urls))
-	return up.urls[idx]
+
+	for i := uint32(0); i < uint32(len(bus)); i++ {
+		idx := (n + i) % uint32(len(bus))
+		bu := bus[idx]
+		if bu.isBroken() {
+			continue
+		}
+		if atomic.CompareAndSwapInt32(&bu.concurrentRequests, 0, 1) {
+			// Fast path - return the backend with zero concurrently executed requests.
+			return bu
+		}
+	}
+
+	// Slow path - return the backend with the minimum number of concurrently executed requests.
+	buMin := bus[n%uint32(len(bus))]
+	minRequests := atomic.LoadInt32(&buMin.concurrentRequests)
+	for _, bu := range bus {
+		if bu.isBroken() {
+			continue
+		}
+		if n := atomic.LoadInt32(&bu.concurrentRequests); n < minRequests {
+			buMin = bu
+			minRequests = n
+		}
+	}
+	atomic.AddInt32(&buMin.concurrentRequests, 1)
+	return buMin
 }
 
 // UnmarshalYAML unmarshals up from yaml.
@@ -121,31 +212,33 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 	default:
 		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
 	}
-	pus := make([]*url.URL, len(urls))
+	bus := make([]*backendURL, len(urls))
 	for i, u := range urls {
 		pu, err := url.Parse(u)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
 		}
-		pus[i] = pu
+		bus[i] = &backendURL{
+			url: pu,
+		}
 	}
-	up.urls = pus
+	up.bus = bus
 	return nil
 }
 
 // MarshalYAML marshals up to yaml.
 func (up *URLPrefix) MarshalYAML() (interface{}, error) {
 	var b []byte
-	if len(up.urls) == 1 {
-		u := up.urls[0].String()
+	if len(up.bus) == 1 {
+		u := up.bus[0].url.String()
 		b = strconv.AppendQuote(b, u)
 		return string(b), nil
 	}
 	b = append(b, '[')
-	for i, pu := range up.urls {
-		u := pu.String()
+	for i, bu := range up.bus {
+		u := bu.url.String()
 		b = strconv.AppendQuote(b, u)
-		if i+1 < len(up.urls) {
+		if i+1 < len(up.bus) {
 			b = append(b, ',')
 		}
 	}
@@ -215,10 +308,20 @@ func stopAuthConfig() {
 }
 
 func authConfigReloader(sighupCh <-chan os.Signal) {
+	var refreshCh <-chan time.Time
+	// initialize auth refresh interval
+	if *configCheckInterval > 0 {
+		ticker := time.NewTicker(*configCheckInterval)
+		defer ticker.Stop()
+		refreshCh = ticker.C
+	}
+
 	for {
 		select {
 		case <-stopCh:
 			return
+		case <-refreshCh:
+			procutil.SelfSIGHUP()
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; loading -auth.config=%q", *authConfigPath)
 			m, err := readAuthConfig(*authConfigPath)
@@ -298,29 +401,44 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
+		name := ui.name()
 		if ui.BearerToken != "" {
-			name := "bearer_token"
-			if ui.Name != "" {
-				name = ui.Name
-			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 		}
 		if ui.Username != "" {
-			name := ui.Username
-			if ui.Name != "" {
-				name = ui.Name
-			}
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 		}
+		mcr := ui.getMaxConcurrentRequests()
+		ui.concurrencyLimitCh = make(chan struct{}, mcr)
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+			return float64(cap(ui.concurrencyLimitCh))
+		})
+		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+			return float64(len(ui.concurrencyLimitCh))
+		})
 		byAuthToken[at1] = ui
 		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
+func (ui *UserInfo) name() string {
+	if ui.Name != "" {
+		return ui.Name
+	}
+	if ui.Username != "" {
+		return ui.Username
+	}
+	if ui.BearerToken != "" {
+		return "bearer_token"
+	}
+	return ""
+}
+
 func getAuthTokens(bearerToken, username, password string) (string, string) {
 	if bearerToken != "" {
 		// Accept the bearerToken as Basic Auth username with empty password
@@ -342,12 +460,12 @@ func getAuthToken(bearerToken, username, password string) string {
 }
 
 func (up *URLPrefix) sanitize() error {
-	for i, pu := range up.urls {
-		puNew, err := sanitizeURLPrefix(pu)
+	for _, bu := range up.bus {
+		puNew, err := sanitizeURLPrefix(bu.url)
 		if err != nil {
 			return err
 		}
-		up.urls[i] = puNew
+		bu.url = puNew
 	}
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -218,11 +218,13 @@ users:
 - username: foo
   password: bar
   url_prefix: http://aaa:343/bbb
+  max_concurrent_requests: 5
 `, map[string]*UserInfo{
 		getAuthToken("", "foo", "bar"): {
-			Username:  "foo",
-			Password:  "bar",
-			URLPrefix: mustParseURL("http://aaa:343/bbb"),
+			Username:              "foo",
+			Password:              "bar",
+			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
+			MaxConcurrentRequests: 5,
 		},
 	})
 
@@ -390,15 +392,17 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	pus := make([]*url.URL, len(us))
+	bus := make([]*backendURL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		pus[i] = pu
+		bus[i] = &backendURL{
+			url: pu,
+		}
 	}
 	return &URLPrefix{
-		urls: pus,
+		bus: bus,
 	}
 }

app/vmauth/example_config.yml

@@ -18,26 +18,27 @@ users:
   headers:
   - "X-Scope-OrgID: foobar"
 
-  # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be proxied to http://localhost:8428 .
+  # are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  #
+  # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests.
+  # Excess concurrent requests are rejected with 429 HTTP status code.
+  # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags.
 - username: "local-single-node"
   password: "***"
   url_prefix: "http://localhost:8428"
+  max_concurrent_requests: 10
 
-  # The user for querying local single-node VictoriaMetrics with extra_label team=dev.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be routed to http://localhost:8428 with extra_label=team=dev query arg.
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
   # For example, http://vmauth:8427/api/v1/query is routed to http://localhost:8428/api/v1/query?extra_label=team=dev
-- username: "local-single-node"
+- username: "local-single-node2"
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
-  # The user for querying account 123 in VictoriaMetrics cluster
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
+  # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
   #   - http://vmselect1:8481/select/123/prometheus/api/v1/select
   #   - http://vmselect2:8481/select/123/prometheus/api/v1/select
@@ -47,10 +48,8 @@ users:
   - "http://vmselect1:8481/select/123/prometheus"
   - "http://vmselect2:8481/select/123/prometheus"
 
-  # The user for inserting Prometheus data into VictoriaMetrics cluster under account 42
-  # See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
-  # will be load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
+  # are load-balanced between http://vminsert1:8480/insert/42/prometheus and http://vminsert2:8480/insert/42/prometheus
   # For example, http://vmauth:8427/api/v1/write is proxied to the following urls in a round-robin manner:
   #   - http://vminsert1:8480/insert/42/prometheus/api/v1/write
   #   - http://vminsert2:8480/insert/42/prometheus/api/v1/write

app/vmauth/main.go

@@ -28,12 +28,16 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
 		"See also -maxConcurrentRequests")
 	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend")
+		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
+		"in per-user config")
 	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
@@ -107,32 +111,54 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	ui.requests.Inc()
-	targetURL, headers, err := createTargetURL(ui, r.URL)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
-		return true
-	}
 
 	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-	default:
-		concurrentRequestsLimitReachedTotal.Inc()
-		w.Header().Add("Retry-After", "10")
-		err := &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh)),
-			StatusCode: http.StatusTooManyRequests,
+		if err := ui.beginConcurrencyLimit(); err != nil {
+			handleConcurrencyLimitError(w, r, err)
+			<-concurrencyLimitCh
+			return true
 		}
-		httpserver.Errorf(w, r, "%s", err)
+	default:
+		concurrentRequestsLimitReached.Inc()
+		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
+		handleConcurrencyLimitError(w, r, err)
 		return true
 	}
-	processRequest(w, r, targetURL, headers)
+	processRequest(w, r, ui)
+	ui.endConcurrencyLimit()
 	<-concurrencyLimitCh
 	return true
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+	u := normalizeURL(r.URL)
+	up, headers, err := ui.getURLPrefixAndHeaders(u)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
+		return
+	}
+	maxAttempts := up.getBackendsCount()
+	for i := 0; i < maxAttempts; i++ {
+		bu := up.getLeastLoadedBackendURL()
+		targetURL := mergeURLs(bu.url, u)
+		ok := tryProcessingRequest(w, r, targetURL, headers)
+		bu.put()
+		if ok {
+			return
+		}
+		bu.setBroken()
+	}
+	err = &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
+		StatusCode: http.StatusServiceUnavailable,
+	}
+	httpserver.Errorf(w, r, "%s", err)
+}
+
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
@@ -142,12 +168,20 @@ func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL,
 	transportOnce.Do(transportInit)
 	res, err := transport.RoundTrip(req)
 	if err != nil {
-		err = &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("error when proxying the request to %q: %s", targetURL, err),
-			StatusCode: http.StatusBadGateway,
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		if r.Method == http.MethodPost || r.Method == http.MethodPut {
+			// It is impossible to retry POST and PUT requests,
+			// since we already proxied the request body to the backend.
+			err = &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
 		}
-		httpserver.Errorf(w, r, "%s", err)
-		return
+		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying the request to %q: %s", remoteAddr, requestURI, targetURL, err)
+		return false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
@@ -162,8 +196,9 @@ func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL,
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-		return
+		return true
 	}
+	return true
 }
 
 var copyBufPool bytesutil.ByteBufferPool
@@ -269,7 +304,7 @@ func concurrencyLimitInit() {
 	})
 }
 
-var concurrentRequestsLimitReachedTotal = metrics.NewCounter("vmauth_concurrent_requests_limit_reached_total")
+var concurrentRequestsLimitReached = metrics.NewCounter("vmauth_concurrent_requests_limit_reached_total")
 
 func usage() {
 	const s = `
@@ -279,3 +314,12 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
 `
 	flagutil.Usage(s)
 }
+
+func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	w.Header().Add("Retry-After", "10")
+	err = &httpserver.ErrorWithStatusCode{
+		Err:        err,
+		StatusCode: http.StatusTooManyRequests,
+	}
+	httpserver.Errorf(w, r, "%s", err)
+}

app/vmauth/target_url.go

@@ -7,11 +7,6 @@ import (
 	"strings"
 )
 
-func (up *URLPrefix) mergeURLs(requestURI *url.URL) *url.URL {
-	pu := up.getNextURL()
-	return mergeURLs(pu, requestURI)
-}
-
 func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	targetURL := *uiURL
 	targetURL.Path += requestURI.Path
@@ -35,12 +30,27 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header, error) {
+	for _, e := range ui.URLMaps {
+		for _, sp := range e.SrcPaths {
+			if sp.match(u.Path) {
+				return e.URLPrefix, e.Headers, nil
+			}
+		}
+	}
+	if ui.URLPrefix != nil {
+		return ui.URLPrefix, ui.Headers, nil
+	}
+	missingRouteRequests.Inc()
+	return nil, nil, fmt.Errorf("missing route for %q", u.String())
+}
+
+func normalizeURL(uOrig *url.URL) *url.URL {
 	u := *uOrig
 	// Prevent from attacks with using `..` in r.URL.Path
 	u.Path = path.Clean(u.Path)
 	if !strings.HasSuffix(u.Path, "/") && strings.HasSuffix(uOrig.Path, "/") {
-		// The path.Clean() removes traling slash.
+		// The path.Clean() removes trailing slash.
 		// Return it back if needed.
 		// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752
 		u.Path += "/"
@@ -52,16 +62,5 @@ func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1554
 		u.Path = ""
 	}
-	for _, e := range ui.URLMaps {
-		for _, sp := range e.SrcPaths {
-			if sp.match(u.Path) {
-				return e.URLPrefix.mergeURLs(&u), e.Headers, nil
-			}
-		}
-	}
-	if ui.URLPrefix != nil {
-		return ui.URLPrefix.mergeURLs(&u), ui.Headers, nil
-	}
-	missingRouteRequests.Inc()
-	return nil, nil, fmt.Errorf("missing route for %q", u.String())
+	return &u
 }

app/vmauth/target_url_test.go

@@ -13,10 +13,14 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, headers, err := createTargetURL(ui, u)
+		u = normalizeURL(u)
+		up, headers, err := ui.getURLPrefixAndHeaders(u)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
+		bu := up.getLeastLoadedBackendURL()
+		target := mergeURLs(bu.url, u)
+		bu.put()
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
@@ -119,15 +123,16 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, headers, err := createTargetURL(ui, u)
+		u = normalizeURL(u)
+		up, headers, err := ui.getURLPrefixAndHeaders(u)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if target != nil {
-			t.Fatalf("unexpected target=%q; want empty string", target)
+		if up != nil {
+			t.Fatalf("unexpected non-empty up=%#v", up)
 		}
 		if headers != nil {
-			t.Fatalf("unexpected headers=%q; want empty string", headers)
+			t.Fatalf("unexpected non-empty headers=%q", headers)
 		}
 	}
 	f(&UserInfo{}, "/foo/bar")

app/vmbackup/Makefile

@@ -39,6 +39,9 @@ vmbackup-freebsd-amd64-prod:
 vmbackup-openbsd-amd64-prod:
 	APP_NAME=vmbackup $(MAKE) app-via-docker-openbsd-amd64
 
+vmbackup-windows-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-windows-amd64
+
 package-vmbackup:
 	APP_NAME=vmbackup $(MAKE) package-via-docker
 
@@ -75,6 +78,9 @@ vmbackup-linux-arm64:
 vmbackup-linux-ppc64le:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmbackup-linux-s390x:
+	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmbackup-linux-386:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
@@ -90,5 +96,8 @@ vmbackup-freebsd-amd64:
 vmbackup-openbsd-amd64:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
+vmbackup-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmbackup $(MAKE) app-local-windows-goarch
+
 vmbackup-pure:
 	APP_NAME=vmbackup $(MAKE) app-local-pure

app/vmbackup/main.go

@@ -49,29 +49,35 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
+	// Storing snapshot delete function to be able to call it in case
+	// of error since logger.Fatal will exit the program without
+	// calling deferred functions.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055
+	deleteSnapshot := func() {}
+
 	if len(*snapshotCreateURL) > 0 {
 		// create net/url object
-		createUrl, err := url.Parse(*snapshotCreateURL)
+		createURL, err := url.Parse(*snapshotCreateURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotCreateURL: %s", err)
 		}
 		if len(*snapshotName) > 0 {
 			logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case")
 		}
-		logger.Infof("Snapshot create url %s", createUrl.Redacted())
+		logger.Infof("Snapshot create url %s", createURL.Redacted())
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
 			if err != nil {
 				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
 			}
 		}
-		deleteUrl, err := url.Parse(*snapshotDeleteURL)
+		deleteURL, err := url.Parse(*snapshotDeleteURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotDeleteURL: %s", err)
 		}
-		logger.Infof("Snapshot delete url %s", deleteUrl.Redacted())
+		logger.Infof("Snapshot delete url %s", deleteURL.Redacted())
 
-		name, err := snapshot.Create(createUrl.String())
+		name, err := snapshot.Create(createURL.String())
 		if err != nil {
 			logger.Fatalf("cannot create snapshot: %s", err)
 		}
@@ -80,32 +86,48 @@ func main() {
 			logger.Fatalf("cannot set snapshotName flag: %v", err)
 		}
 
-		defer func() {
-			err := snapshot.Delete(deleteUrl.String(), name)
+		deleteSnapshot = func() {
+			err := snapshot.Delete(deleteURL.String(), name)
 			if err != nil {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
-		}()
+		}
 	} else if len(*snapshotName) == 0 {
 		logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
 	}
-	if err := snapshot.Validate(*snapshotName); err != nil {
-		logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
-	}
 
 	go httpserver.Serve(*httpListenAddr, false, nil)
 
+	err := makeBackup()
+	deleteSnapshot()
+	if err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func makeBackup() error {
+	if err := snapshot.Validate(*snapshotName); err != nil {
+		return fmt.Errorf("invalid -snapshotName=%q: %s", *snapshotName, err)
+	}
+
 	srcFS, err := newSrcFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	dstFS, err := newDstFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	originFS, err := newOriginFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	a := &actions.Backup{
 		Concurrency: *concurrency,
@@ -114,18 +136,12 @@ func main() {
 		Origin:      originFS,
 	}
 	if err := a.Run(); err != nil {
-		logger.Fatalf("cannot create backup: %s", err)
+		return err
 	}
 	srcFS.MustStop()
 	dstFS.MustStop()
 	originFS.MustStop()
-
-	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop http server for metrics: %s", err)
-	}
-	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+	return nil
 }
 
 func usage() {

app/vmctl/Makefile

@@ -78,6 +78,9 @@ vmctl-linux-arm64:
 vmctl-linux-ppc64le:
 	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
+vmctl-linux-s390x:
+	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
 vmctl-linux-386:
 	APP_NAME=vmctl CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmctl/README.md

@@ -483,6 +483,10 @@ Processing ranges: 8798 / 8798 [████████████████
 2022/10/19 16:45:37 Total time: 1m19.406283424s
 ```
 
+Migrating big volumes of data may result in remote read client reaching the timeout.
+Consider increasing the value of `--remote-read-http-timeout` (default `5m`) command-line flag when seeing
+timeouts or `context canceled` errors.
+
 ### Filtering
 
 The filtering consists of two parts: by labels and time.
@@ -733,21 +737,33 @@ or higher.
 
 See `./vmctl vm-native --help` for details and full list of flags.
 
-In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`)
-and processing is done by "destination" (`dst`). Because of that, `vmctl` doesn't actually know how much data will be
-processed and can't show the progress bar. It will show the current processing speed and total number of processed bytes:
+Migration in `vm-native` mode takes two steps:
+1. Explore the list of the metrics to migrate via `/api/v1/series` API;
+2. Migrate explored metrics one-by-one.
 
 ```
-./vmctl vm-native --vm-native-src-addr=http://localhost:8528  \
-  --vm-native-dst-addr=http://localhost:8428 \
-  --vm-native-filter-match='{job="vmagent"}' \
-  --vm-native-filter-time-start='2020-01-01T20:07:00Z'
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ 
+    --vm-native-dst-addr=http://localhost:8428 \
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \
+    --vm-native-filter-match='{__name__=~"vm_cache_.*"}'    
 VictoriaMetrics Native import mode
-Initing export pipe from "http://localhost:8528" with filters:
-        filter: match[]={job="vmagent"}
-Initing import process to "http://localhost:8428":
-Total: 336.75 KiB ↖ Speed: 454.46 KiB p/s
-2020/10/13 17:04:59 Total time: 952.143376ms
+
+2023/03/02 09:22:02 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__=~"vm_cache_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:22:02 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:22:04 Requests to make: 9
+Requests to make: 9 / 9 [███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/03/02 09:22:06 Import finished!
+2023/03/02 09:22:06 VictoriaMetrics importer stats:
+  time spent while importing: 3.632638875s;
+  total bytes: 7.8 MB;
+  bytes/s: 2.1 MB;
+  requests: 9;
+  requests retries: 0;
+2023/03/02 09:22:06 Total time: 3.633127625s
 ```
 
 Importing tips:
@@ -755,6 +771,7 @@ Importing tips:
 1. Migrating big volumes of data may result in reaching the safety limits on `src` side.
 Please verify that `-search.maxExportDuration` and `-search.maxExportSeries` were set with
 proper values for `src`. If hitting the limits, follow the recommendations [here](https://docs.victoriametrics.com/#how-to-export-data-in-native-format).
+If hitting `the number of matching timeseries exceeds...` error, adjust filters to match less time series or update `-search.maxSeries` command-line flag on vmselect/vmsingle;
 2. Migrating all the metrics from one VM to another may collide with existing application metrics
 (prefixed with `vm_`) at destination and lead to confusion when using
 [official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
@@ -764,73 +781,84 @@ To avoid such situation try to filter out VM process metrics via `--vm-native-fi
 4. `vmctl` doesn't provide relabeling or other types of labels management in this mode.
 Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/vmctl/issues/4#issuecomment-683424375).
 5. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
-and specify `accountID` param.
+and specify `accountID` param. Example formats:
+
+```console
+# Migrating from cluster to single
+--vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+--vm-native-dst-addr=http://<dst-vmsingle>:8428
+
+ # Migrating from single to cluster
+--vm-native-src-addr=http://<src-vmsingle>:8428
+--vm-native-src-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+
+# Migrating single to single
+--vm-native-src-addr=http://<src-vmsingle>:8428
+--vm-native-dst-addr=http://<dst-vmsingle>:8428
+
+# Migrating cluster to cluster
+--vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+--vm-native-dst-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+```
 6. When migrating large volumes of data it might be useful to use `--vm-native-step-interval` flag to split single process into smaller steps.
+7. `vmctl` supports `--vm-concurrency` which controls the number of concurrent workers that process the input from source query results.
+Please note that each import request can load up to a single vCPU core on VictoriaMetrics. So try to set it according
+to allocated CPU resources of your VictoriaMetrics installation.
+8. `vmctl` supports `--vm-native-src-headers` and `--vm-native-dst-headers` which defines headers to send with each request
+to the corresponding source address.
+9. `vmctl` supports `--vm-native-disable-http-keep-alive` to allow `vmctl` to use non-persistent HTTP connections to avoid
+error `use of closed network connection` when run a longer export.
+10. Migrating data with overlapping time range for destination data can produce duplicates series at destination.
+To avoid duplicates on the destination set `-dedup.minScrapeInterval=1ms` for `vmselect` and `vmstorage`.
+This will instruct `vmselect` and `vmstorage` to ignore duplicates with match timestamps.
+
+In this mode `vmctl` acts as a proxy between two VM instances, where time series filtering is done by "source" (`src`)
+and processing is done by "destination" (`dst`). So no extra memory or CPU resources required on `vmctl` side. Only
+`src` and `dst` resource matter.
 
 #### Using time-based chunking of migration
 
-It is possible split migration process into set of smaller batches based on time. This is especially useful when migrating large volumes of data as this adds indication of progress and ability to restore process from certain point in case of failure.
+It is possible split migration process into set of smaller batches based on time. This is especially useful when 
+migrating large volumes of data as this adds indication of progress and ability to restore process from certain point 
+in case of failure.
 
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`.
-Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for export process.
+To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`, `minute`.
+Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for 
+export process.
 
 Every range is being processed independently, which means that:
 - after range processing is finished all data within range is migrated
-- if process fails on one of stages it is guaranteed that data of prior stages is already written, so it is possible to restart process starting from failed range
+- if process fails on one of stages it is guaranteed that data of prior stages is already written,
+so it is possible to restart process starting from failed range.
 
-It is recommended using the `month` step when migrating the data over multiple months, since the migration with `day` and `hour` steps may take longer time to complete
-because of additional overhead.
+It is recommended using the `month` step when migrating the data over multiple months, 
+since the migration with `day` and `hour` steps may take longer time to complete because of additional overhead.
 
 Usage example:
 ```console
-./vmctl vm-native 
-    --vm-native-filter-time-start 2022-06-17T00:07:00Z \
-    --vm-native-filter-time-end 2022-10-03T00:07:00Z \
-    --vm-native-src-addr http://localhost:8428 \
-    --vm-native-dst-addr http://localhost:8528 \
-    --vm-native-step-interval=month
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ 
+    --vm-native-dst-addr=http://localhost:8428 \
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \
+    --vm-native-step-interval=month \
+    --vm-native-filter-match='{__name__=~"vm_cache_.*"}'    
 VictoriaMetrics Native import mode
-2022/08/30 19:48:24 Processing range 1/5: 2022-06-17T00:07:00Z - 2022-06-30T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-06-17T00:07:00Z
-        end: 2022-06-30T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 28.89 KiB p/s 
-2022/08/30 19:48:24 Processing range 2/5: 2022-07-01T00:00:00Z - 2022-07-31T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-07-01T00:00:00Z
-        end: 2022-07-31T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 164.35 KiB p/s 
-2022/08/30 19:48:24 Processing range 3/5: 2022-08-01T00:00:00Z - 2022-08-31T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-08-01T00:00:00Z
-        end: 2022-08-31T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 191.42 KiB p/s 
-2022/08/30 19:48:24 Processing range 4/5: 2022-09-01T00:00:00Z - 2022-09-30T23:59:59Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-09-01T00:00:00Z
-        end: 2022-09-30T23:59:59Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 141.04 KiB p/s 
-2022/08/30 19:48:24 Processing range 5/5: 2022-10-01T00:00:00Z - 2022-10-03T00:07:00Z 
-2022/08/30 19:48:24 Initing export pipe from "http://localhost:8428" with filters: 
-        filter: match[]={__name__!=""}
-        start: 2022-10-01T00:00:00Z
-        end: 2022-10-03T00:07:00Z
-Initing import process to "http://localhost:8428":
-2022/08/30 19:48:24 Import finished!
-Total: 16 B ↗ Speed: 186.32 KiB p/s 
-2022/08/30 19:48:24 Total time: 12.680582ms
+
+2023/03/02 09:18:05 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__=~"vm_cache_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:18:05 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:18:07 Selected time range will be split into 5 ranges according to "month" step. Requests to make: 45.
+Requests to make: 45 / 45 [█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/03/02 09:18:12 Import finished!
+2023/03/02 09:18:12 VictoriaMetrics importer stats:
+  time spent while importing: 7.111870667s;
+  total bytes: 7.7 MB;
+  bytes/s: 1.1 MB;
+  requests: 45;
+  requests retries: 0;
+2023/03/02 09:18:12 Total time: 7.112405875s
 ```
 
 #### Cluster-to-cluster migration mode
@@ -842,70 +870,41 @@ Cluster-to-cluster uses `/admin/tenants` endpoint (available starting from [v1.8
 To use this mode you need to set `--vm-intercluster` flag to `true`, `--vm-native-src-addr` flag to 'http://vmselect:8481/' and `--vm-native-dst-addr` value to http://vminsert:8480/:
 
 ```console
-./bin/vmctl vm-native --vm-intercluster=true --vm-native-src-addr=http://localhost:8481/ --vm-native-dst-addr=http://172.17.0.3:8480/
+  ./vmctl vm-native --vm-native-src-addr=http://127.0.0.1:8481/ \
+  --vm-native-dst-addr=http://127.0.0.1:8480/ \
+  --vm-native-filter-match='{__name__="vm_app_uptime_seconds"}' \
+  --vm-native-filter-time-start='2023-02-01T00:00:00Z' \
+  --vm-native-step-interval=day \  
+--vm-intercluster
 VictoriaMetrics Native import mode
-2022/12/05 21:20:06 Discovered tenants: [123:1 12812919:1 1289198:1 1289:1283 12:1 1:0 1:1 1:1231231 1:1271727 1:12819 1:281 812891298:1]
-2022/12/05 21:20:06 Initing export pipe from "http://localhost:8481/select/123:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/123:1/prometheus/api/v1/import/native":
-Total: 61.13 MiB ↖ Speed: 2.05 MiB p/s 
-Total: 61.13 MiB ↗ Speed: 2.30 MiB p/s 
-2022/12/05 21:20:33 Initing export pipe from "http://localhost:8481/select/12812919:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/12812919:1/prometheus/api/v1/import/native":
-Total: 43.14 MiB ↘ Speed: 1.86 MiB p/s  
-Total: 43.14 MiB ↙ Speed: 2.36 MiB p/s 
-2022/12/05 21:20:51 Initing export pipe from "http://localhost:8481/select/1289198:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1289198:1/prometheus/api/v1/import/native":
-Total: 16.64 MiB ↗ Speed: 2.66 MiB p/s 
-Total: 16.64 MiB ↘ Speed: 2.19 MiB p/s 
-2022/12/05 21:20:59 Initing export pipe from "http://localhost:8481/select/1289:1283/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1289:1283/prometheus/api/v1/import/native":
-Total: 43.33 MiB ↙ Speed: 1.94 MiB p/s 
-Total: 43.33 MiB ↖ Speed: 2.35 MiB p/s 
-2022/12/05 21:21:18 Initing export pipe from "http://localhost:8481/select/12:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/12:1/prometheus/api/v1/import/native":
-Total: 63.78 MiB ↙ Speed: 1.96 MiB p/s 
-Total: 63.78 MiB ↖ Speed: 2.28 MiB p/s 
-2022/12/05 21:21:46 Initing export pipe from "http://localhost:8481/select/1:0/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:0/prometheus/api/v1/import/native":
-2022/12/05 21:21:46 Import finished!
-Total: 330 B ↗ Speed: 3.53 MiB p/s 
-2022/12/05 21:21:46 Initing export pipe from "http://localhost:8481/select/1:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1/prometheus/api/v1/import/native":
-Total: 63.81 MiB ↙ Speed: 1.96 MiB p/s 
-Total: 63.81 MiB ↖ Speed: 2.28 MiB p/s 
-2022/12/05 21:22:14 Initing export pipe from "http://localhost:8481/select/1:1231231/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1231231/prometheus/api/v1/import/native":
-Total: 63.84 MiB ↙ Speed: 1.93 MiB p/s 
-Total: 63.84 MiB ↖ Speed: 2.29 MiB p/s 
-2022/12/05 21:22:42 Initing export pipe from "http://localhost:8481/select/1:1271727/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:1271727/prometheus/api/v1/import/native":
-Total: 54.37 MiB ↘ Speed: 1.90 MiB p/s 
-Total: 54.37 MiB ↙ Speed: 2.37 MiB p/s 
-2022/12/05 21:23:05 Initing export pipe from "http://localhost:8481/select/1:12819/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:12819/prometheus/api/v1/import/native":
-Total: 17.01 MiB ↙ Speed: 1.75 MiB p/s 
-Total: 17.01 MiB ↖ Speed: 2.15 MiB p/s 
-2022/12/05 21:23:13 Initing export pipe from "http://localhost:8481/select/1:281/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/1:281/prometheus/api/v1/import/native":
-Total: 63.89 MiB ↘ Speed: 1.90 MiB p/s 
-Total: 63.89 MiB ↙ Speed: 2.29 MiB p/s 
-2022/12/05 21:23:42 Initing export pipe from "http://localhost:8481/select/812891298:1/prometheus/api/v1/export/native" with filters: 
-        filter: match[]={__name__!=""}
-Initing import process to "http://172.17.0.3:8480/insert/812891298:1/prometheus/api/v1/import/native":
-Total: 63.84 MiB ↖ Speed: 1.99 MiB p/s 
-Total: 63.84 MiB ↗ Speed: 2.26 MiB p/s 
-2022/12/05 21:24:10 Total time: 4m4.1466565s
+2023/02/28 10:41:42 Discovering tenants...
+2023/02/28 10:41:42 The following tenants were discovered: [0:0 1:0 2:0 3:0 4:0]
+2023/02/28 10:41:42 Initing import process from "http://127.0.0.1:8481/select/0:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/0:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 0:0 
+2023/02/28 10:41:42 Exploring metrics...
+2023/02/28 10:41:42 Found 1 metrics to import 
+2023/02/28 10:41:42 Selected time range will be split into 28 ranges according to "day" step. 
+Requests to make for tenant 0:0: 28 / 28 [███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+
+2023/02/28 10:41:45 Initing import process from "http://127.0.0.1:8481/select/1:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/1:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 1:0 
+2023/02/28 10:41:45 Exploring metrics...
+2023/02/28 10:41:45 Found 1 metrics to import 
+2023/02/28 10:41:45 Selected time range will be split into 28 ranges according to "day" step. Requests to make: 28 
+Requests to make for tenant 1:0: 28 / 28 [████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+
+...
+
+2023/02/28 10:42:49 Import finished!
+2023/02/28 10:42:49 VictoriaMetrics importer stats:
+  time spent while importing: 1m6.714210417s;
+  total bytes: 39.7 MB;
+  bytes/s: 594.4 kB;
+  requests: 140;
+  requests retries: 0;
+2023/02/28 10:42:49 Total time: 1m7.147971417s
 ```
 
 ## Verifying exported blocks from VictoriaMetrics
@@ -972,6 +971,7 @@ a sign of network issues or VM being overloaded. See the logs during import for
 By default `vmctl` waits confirmation from user before starting the import. If this is unwanted
 behavior and no user interaction required - pass `-s` flag to enable "silence" mode:
 
+See below the example of `vm-native` migration process:
 ```
     -s Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
 ```

app/vmctl/auth/auth.go

@@ -0,0 +1,222 @@
+package auth
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+)
+
+// HTTPClientConfig represents http client config.
+type HTTPClientConfig struct {
+	BasicAuth   *BasicAuthConfig
+	BearerToken string
+	Headers     string
+}
+
+// NewConfig creates auth config for the given hcc.
+func (hcc *HTTPClientConfig) NewConfig() (*Config, error) {
+	opts := &Options{
+		BasicAuth:   hcc.BasicAuth,
+		BearerToken: hcc.BearerToken,
+		Headers:     hcc.Headers,
+	}
+	return opts.NewConfig()
+}
+
+// BasicAuthConfig represents basic auth config.
+type BasicAuthConfig struct {
+	Username     string
+	Password     string
+	PasswordFile string
+}
+
+// ConfigOptions options which helps build Config
+type ConfigOptions func(config *HTTPClientConfig)
+
+// Generate returns Config based on the given params
+func Generate(filterOptions ...ConfigOptions) (*Config, error) {
+	authCfg := &HTTPClientConfig{}
+	for _, option := range filterOptions {
+		option(authCfg)
+	}
+
+	return authCfg.NewConfig()
+}
+
+// WithBasicAuth returns AuthConfigOptions and initialized BasicAuthConfig based on given params
+func WithBasicAuth(username, password string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if username != "" || password != "" {
+			config.BasicAuth = &BasicAuthConfig{
+				Username: username,
+				Password: password,
+			}
+		}
+	}
+}
+
+// WithBearer returns AuthConfigOptions and set BearerToken or BearerTokenFile based on given params
+func WithBearer(token string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if token != "" {
+			config.BearerToken = token
+		}
+	}
+}
+
+// WithHeaders returns AuthConfigOptions and set Headers based on the given params
+func WithHeaders(headers string) ConfigOptions {
+	return func(config *HTTPClientConfig) {
+		if headers != "" {
+			config.Headers = headers
+		}
+	}
+}
+
+// Config is auth config.
+type Config struct {
+	getAuthHeader      func() string
+	authHeaderLock     sync.Mutex
+	authHeader         string
+	authHeaderDeadline uint64
+
+	headers []keyValue
+
+	authDigest string
+}
+
+// SetHeaders sets the configured ac headers to req.
+func (ac *Config) SetHeaders(req *http.Request, setAuthHeader bool) {
+	reqHeaders := req.Header
+	for _, h := range ac.headers {
+		reqHeaders.Set(h.key, h.value)
+	}
+	if setAuthHeader {
+		if ah := ac.GetAuthHeader(); ah != "" {
+			reqHeaders.Set("Authorization", ah)
+		}
+	}
+}
+
+// GetAuthHeader returns optional `Authorization: ...` http header.
+func (ac *Config) GetAuthHeader() string {
+	f := ac.getAuthHeader
+	if f == nil {
+		return ""
+	}
+	ac.authHeaderLock.Lock()
+	defer ac.authHeaderLock.Unlock()
+	if fasttime.UnixTimestamp() > ac.authHeaderDeadline {
+		ac.authHeader = f()
+		// Cache the authHeader for a second.
+		ac.authHeaderDeadline = fasttime.UnixTimestamp() + 1
+	}
+	return ac.authHeader
+}
+
+type authContext struct {
+	// getAuthHeader must return <value> for 'Authorization: <value>' http request header
+	getAuthHeader func() string
+
+	// authDigest must contain the digest for the used authorization
+	// The digest must be changed whenever the original config changes.
+	authDigest string
+}
+
+func (ac *authContext) initFromBasicAuthConfig(ba *BasicAuthConfig) error {
+	if ba.Username == "" {
+		return fmt.Errorf("missing `username` in `basic_auth` section")
+	}
+	if ba.Password != "" {
+		ac.getAuthHeader = func() string {
+			// See https://en.wikipedia.org/wiki/Basic_access_authentication
+			token := ba.Username + ":" + ba.Password
+			token64 := base64.StdEncoding.EncodeToString([]byte(token))
+			return "Basic " + token64
+		}
+		ac.authDigest = fmt.Sprintf("basic(username=%q, password=%q)", ba.Username, ba.Password)
+		return nil
+	}
+	return nil
+}
+
+func (ac *authContext) initFromBearerToken(bearerToken string) error {
+	ac.getAuthHeader = func() string {
+		return "Bearer " + bearerToken
+	}
+	ac.authDigest = fmt.Sprintf("bearer(token=%q)", bearerToken)
+	return nil
+}
+
+// Options contain options, which must be passed to NewConfig.
+type Options struct {
+	// BasicAuth contains optional BasicAuthConfig.
+	BasicAuth *BasicAuthConfig
+
+	// BearerToken contains optional bearer token.
+	BearerToken string
+
+	// Headers contains optional http request headers in the form 'Foo: bar'.
+	Headers string
+}
+
+// NewConfig creates auth config from the given opts.
+func (opts *Options) NewConfig() (*Config, error) {
+	var ac authContext
+	if opts.BasicAuth != nil {
+		if ac.getAuthHeader != nil {
+			return nil, fmt.Errorf("cannot use both `authorization` and `basic_auth`")
+		}
+		if err := ac.initFromBasicAuthConfig(opts.BasicAuth); err != nil {
+			return nil, err
+		}
+	}
+	if opts.BearerToken != "" {
+		if ac.getAuthHeader != nil {
+			return nil, fmt.Errorf("cannot simultaneously use `authorization`, `basic_auth` and `bearer_token`")
+		}
+		if err := ac.initFromBearerToken(opts.BearerToken); err != nil {
+			return nil, err
+		}
+	}
+
+	headers, err := parseHeaders(opts.Headers)
+	if err != nil {
+		return nil, err
+	}
+	c := &Config{
+		getAuthHeader: ac.getAuthHeader,
+		headers:       headers,
+		authDigest:    ac.authDigest,
+	}
+	return c, nil
+}
+
+type keyValue struct {
+	key   string
+	value string
+}
+
+func parseHeaders(headers string) ([]keyValue, error) {
+	if len(headers) == 0 {
+		return nil, nil
+	}
+
+	var headersSplitByDelimiter = strings.Split(headers, "^^")
+
+	kvs := make([]keyValue, len(headersSplitByDelimiter))
+	for i, h := range headersSplitByDelimiter {
+		n := strings.IndexByte(h, ':')
+		if n < 0 {
+			return nil, fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, h)
+		}
+		kv := &kvs[i]
+		kv.key = strings.TrimSpace(h[:n])
+		kv.value = strings.TrimSpace(h[n+1:])
+	}
+	return kvs, nil
+}

app/vmctl/backoff/backoff.go

@@ -0,0 +1,61 @@
+package backoff
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	backoffRetries     = 5
+	backoffFactor      = 1.7
+	backoffMinDuration = time.Second
+)
+
+// retryableFunc describes call back which will repeat on errors
+type retryableFunc func() error
+
+// ErrBadRequest is an error returned on bad request
+var ErrBadRequest = errors.New("bad request")
+
+// Backoff describes object with backoff policy params
+type Backoff struct {
+	retries     int
+	factor      float64
+	minDuration time.Duration
+}
+
+// New initialize backoff object
+func New() *Backoff {
+	return &Backoff{
+		retries:     backoffRetries,
+		factor:      backoffFactor,
+		minDuration: backoffMinDuration,
+	}
+}
+
+// Retry process retries until all attempts are completed
+func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
+	var attempt uint64
+	for i := 0; i < b.retries; i++ {
+		// @TODO we should use context to cancel retries
+		err := cb()
+		if err == nil {
+			return attempt, nil
+		}
+		if errors.Is(err, ErrBadRequest) || errors.Is(err, context.Canceled) {
+			logger.Errorf("unrecoverable error: %s", err)
+			return attempt, err // fail fast if not recoverable
+		}
+		attempt++
+		backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
+		dur := time.Duration(backoff)
+		logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
+		time.Sleep(time.Duration(backoff))
+	}
+	return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
+}

app/vmctl/backoff/backoff_test.go

@@ -0,0 +1,96 @@
+package backoff
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestRetry_Do(t *testing.T) {
+	counter := 0
+	tests := []struct {
+		name               string
+		backoffRetries     int
+		backoffFactor      float64
+		backoffMinDuration time.Duration
+		retryableFunc      retryableFunc
+		ctx                context.Context
+		withCancel         bool
+		want               uint64
+		wantErr            bool
+	}{
+		{
+			name: "return bad request",
+			retryableFunc: func() error {
+				return ErrBadRequest
+			},
+			ctx:     context.Background(),
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name: "empty retries values",
+			retryableFunc: func() error {
+				time.Sleep(time.Millisecond * 100)
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:               "only one retry test",
+			backoffRetries:     5,
+			backoffFactor:      1.7,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					counter++
+					if counter%2 == 0 {
+						return fmt.Errorf("got some error")
+					}
+					if counter%3 == 0 {
+						return nil
+					}
+				}
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    1,
+			wantErr: false,
+		},
+		{
+			name:               "all retries failed test",
+			backoffRetries:     5,
+			backoffFactor:      0.1,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					return fmt.Errorf("got some error")
+				}
+				return nil
+			},
+			ctx:     context.Background(),
+			want:    5,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := New()
+			got, err := r.Retry(tt.ctx, tt.retryableFunc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Retry() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

app/vmctl/flags.go

@@ -325,13 +325,19 @@ const (
 	vmNativeFilterTimeEnd   = "vm-native-filter-time-end"
 	vmNativeStepInterval    = "vm-native-step-interval"
 
-	vmNativeSrcAddr     = "vm-native-src-addr"
-	vmNativeSrcUser     = "vm-native-src-user"
-	vmNativeSrcPassword = "vm-native-src-password"
+	vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive"
 
-	vmNativeDstAddr     = "vm-native-dst-addr"
-	vmNativeDstUser     = "vm-native-dst-user"
-	vmNativeDstPassword = "vm-native-dst-password"
+	vmNativeSrcAddr        = "vm-native-src-addr"
+	vmNativeSrcUser        = "vm-native-src-user"
+	vmNativeSrcPassword    = "vm-native-src-password"
+	vmNativeSrcHeaders     = "vm-native-src-headers"
+	vmNativeSrcBearerToken = "vm-native-src-bearer-token"
+
+	vmNativeDstAddr        = "vm-native-dst-addr"
+	vmNativeDstUser        = "vm-native-dst-user"
+	vmNativeDstPassword    = "vm-native-dst-password"
+	vmNativeDstHeaders     = "vm-native-dst-headers"
+	vmNativeDstBearerToken = "vm-native-dst-bearer-token"
 )
 
 var (
@@ -344,8 +350,9 @@ var (
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{
-			Name:  vmNativeFilterTimeStart,
-			Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Name:     vmNativeFilterTimeStart,
+			Usage:    "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
 		},
 		&cli.StringFlag{
 			Name:  vmNativeFilterTimeEnd,
@@ -355,6 +362,11 @@ var (
 			Name:  vmNativeStepInterval,
 			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
 		},
+		&cli.BoolFlag{
+			Name:  vmNativeDisableHTTPKeepAlive,
+			Usage: "Disable HTTP persistent connections for requests made to VictoriaMetrics components during export",
+			Value: false,
+		},
 		&cli.StringFlag{
 			Name: vmNativeSrcAddr,
 			Usage: "VictoriaMetrics address to perform export from. \n" +
@@ -372,6 +384,16 @@ var (
 			Usage:   "VictoriaMetrics password for basic auth",
 			EnvVars: []string{"VM_NATIVE_SRC_PASSWORD"},
 		},
+		&cli.StringFlag{
+			Name: vmNativeSrcHeaders,
+			Usage: "Optional HTTP headers to send with each request to the corresponding source address. \n" +
+				"For example, --vm-native-src-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding source address. \n" +
+				"Multiple headers must be delimited by '^^': --vm-native-src-headers='header1:value1^^header2:value2'",
+		},
+		&cli.StringFlag{
+			Name:  vmNativeSrcBearerToken,
+			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-src-addr`",
+		},
 		&cli.StringFlag{
 			Name: vmNativeDstAddr,
 			Usage: "VictoriaMetrics address to perform import to. \n" +
@@ -389,6 +411,16 @@ var (
 			Usage:   "VictoriaMetrics password for basic auth",
 			EnvVars: []string{"VM_NATIVE_DST_PASSWORD"},
 		},
+		&cli.StringFlag{
+			Name: vmNativeDstHeaders,
+			Usage: "Optional HTTP headers to send with each request to the corresponding destination address. \n" +
+				"For example, --vm-native-dst-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding destination address. \n" +
+				"Multiple headers must be delimited by '^^': --vm-native-dst-headers='header1:value1^^header2:value2'",
+		},
+		&cli.StringFlag{
+			Name:  vmNativeDstBearerToken,
+			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-dst-addr`",
+		},
 		&cli.StringSliceFlag{
 			Name:  vmExtraLabel,
 			Value: nil,
@@ -406,6 +438,11 @@ var (
 				fmt.Sprintf(" In this mode --%s flag format is: 'http://vmselect:8481/'. --%s flag format is: http://vminsert:8480/. \n", vmNativeSrcAddr, vmNativeDstAddr) +
 				" TenantID will be appended automatically after discovering tenants from src.",
 		},
+		&cli.UintFlag{
+			Name:  vmConcurrency,
+			Usage: "Number of workers concurrently performing import requests to VM",
+			Value: 2,
+		},
 	}
 )
 
@@ -485,7 +522,7 @@ var (
 		},
 		&cli.DurationFlag{
 			Name:  remoteReadHTTPTimeout,
-			Usage: "Timeout defines timeout for HTTP write request to remote storage",
+			Usage: "Timeout defines timeout for HTTP requests made by remote read client",
 		},
 		&cli.StringFlag{
 			Name:  remoteReadHeaders,

app/vmctl/influx/influx.go

@@ -162,7 +162,8 @@ func (c *Client) Explore() ([]*Series, error) {
 	for _, s := range series {
 		fields, ok := mFields[s.Measurement]
 		if !ok {
-			return nil, fmt.Errorf("can't find field keys for measurement %q", s.Measurement)
+			log.Printf("skip measurement %q since it has no fields", s.Measurement)
+			continue
 		}
 		for _, field := range fields {
 			is := &Series{

app/vmctl/main.go

@@ -11,7 +11,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/terminal"
 	"github.com/urfave/cli/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
@@ -20,7 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 )
 
 func main() {
@@ -35,9 +39,6 @@ func main() {
 		Name:    "vmctl",
 		Usage:   "VictoriaMetrics command-line tool",
 		Version: buildinfo.Version,
-		// Disable `-version` flag to avoid conflict with lib/buildinfo flags
-		// see https://github.com/urfave/cli/issues/1560
-		HideVersion: true,
 		Commands: []*cli.Command{
 			{
 				Name:  "opentsdb",
@@ -65,13 +66,13 @@ func main() {
 					// disable progress bars since openTSDB implementation
 					// does not use progress bar pool
 					vmCfg.DisableProgressBar = true
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency))
-					return otsdbProcessor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return otsdbProcessor.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -100,7 +101,7 @@ func main() {
 					}
 
 					vmCfg := initConfigVM(c)
-					importer, err = vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -112,7 +113,7 @@ func main() {
 						c.String(influxMeasurementFieldSeparator),
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode))
-					return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return processor.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -137,7 +138,7 @@ func main() {
 
 					vmCfg := initConfigVM(c)
 
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -152,7 +153,7 @@ func main() {
 						},
 						cc: c.Int(remoteReadConcurrency),
 					}
-					return rmp.run(ctx, c.Bool(globalSilent), c.Bool(globalVerbose))
+					return rmp.run(ctx, isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -163,7 +164,7 @@ func main() {
 					fmt.Println("Prometheus import mode")
 
 					vmCfg := initConfigVM(c)
-					importer, err = vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -186,13 +187,13 @@ func main() {
 						im: importer,
 						cc: c.Int(promConcurrency),
 					}
-					return pp.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return pp.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
 				Name:  "vm-native",
 				Usage: "Migrate time series between VictoriaMetrics installations via native binary format",
-				Flags: vmNativeFlags,
+				Flags: mergeFlags(globalFlags, vmNativeFlags),
 				Action: func(c *cli.Context) error {
 					fmt.Println("VictoriaMetrics Native import mode")
 
@@ -200,28 +201,51 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
+					var srcExtraLabels []string
+					srcAddr := strings.Trim(c.String(vmNativeSrcAddr), "/")
+					srcAuthConfig, err := auth.Generate(
+						auth.WithBasicAuth(c.String(vmNativeSrcUser), c.String(vmNativeSrcPassword)),
+						auth.WithBearer(c.String(vmNativeSrcBearerToken)),
+						auth.WithHeaders(c.String(vmNativeSrcHeaders)))
+					if err != nil {
+						return fmt.Errorf("error initilize auth config for source: %s", srcAddr)
+					}
+
+					dstAddr := strings.Trim(c.String(vmNativeDstAddr), "/")
+					dstExtraLabels := c.StringSlice(vmExtraLabel)
+					dstAuthConfig, err := auth.Generate(
+						auth.WithBasicAuth(c.String(vmNativeDstUser), c.String(vmNativeDstPassword)),
+						auth.WithBearer(c.String(vmNativeDstBearerToken)),
+						auth.WithHeaders(c.String(vmNativeDstHeaders)))
+					if err != nil {
+						return fmt.Errorf("error initilize auth config for destination: %s", dstAddr)
+					}
+
 					p := vmNativeProcessor{
 						rateLimit:    c.Int64(vmRateLimit),
 						interCluster: c.Bool(vmInterCluster),
-						filter: filter{
-							match:     c.String(vmNativeFilterMatch),
-							timeStart: c.String(vmNativeFilterTimeStart),
-							timeEnd:   c.String(vmNativeFilterTimeEnd),
-							chunk:     c.String(vmNativeStepInterval),
+						filter: native.Filter{
+							Match:     c.String(vmNativeFilterMatch),
+							TimeStart: c.String(vmNativeFilterTimeStart),
+							TimeEnd:   c.String(vmNativeFilterTimeEnd),
+							Chunk:     c.String(vmNativeStepInterval),
 						},
-						src: &vmNativeClient{
-							addr:     strings.Trim(c.String(vmNativeSrcAddr), "/"),
-							user:     c.String(vmNativeSrcUser),
-							password: c.String(vmNativeSrcPassword),
+						src: &native.Client{
+							AuthCfg:              srcAuthConfig,
+							Addr:                 srcAddr,
+							ExtraLabels:          srcExtraLabels,
+							DisableHTTPKeepAlive: c.Bool(vmNativeDisableHTTPKeepAlive),
 						},
-						dst: &vmNativeClient{
-							addr:        strings.Trim(c.String(vmNativeDstAddr), "/"),
-							user:        c.String(vmNativeDstUser),
-							password:    c.String(vmNativeDstPassword),
-							extraLabels: c.StringSlice(vmExtraLabel),
+						dst: &native.Client{
+							AuthCfg:              dstAuthConfig,
+							Addr:                 dstAddr,
+							ExtraLabels:          dstExtraLabels,
+							DisableHTTPKeepAlive: c.Bool(vmNativeDisableHTTPKeepAlive),
 						},
+						backoff: backoff.New(),
+						cc:      c.Int(vmConcurrency),
 					}
-					return p.run(ctx)
+					return p.run(ctx, isNonInteractive(c))
 				},
 			},
 			{
@@ -247,7 +271,7 @@ func main() {
 						return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1)
 					}
 					var blocksCount uint64
-					if err := parser.ParseStream(f, isBlockGzipped, func(block *parser.Block) error {
+					if err := stream.Parse(f, isBlockGzipped, func(block *stream.Block) error {
 						atomic.AddUint64(&blocksCount, 1)
 						return nil
 					}); err != nil {
@@ -294,3 +318,8 @@ func initConfigVM(c *cli.Context) vm.Config {
 		DisableProgressBar: c.Bool(vmDisableProgressBar),
 	}
 }
+
+func isNonInteractive(c *cli.Context) bool {
+	isTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
+	return c.Bool(globalSilent) || !isTerminal
+}

app/vmctl/native/client.go

@@ -0,0 +1,186 @@
+package native
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
+)
+
+const (
+	nativeTenantsAddr = "admin/tenants"
+	nativeSeriesAddr  = "api/v1/series"
+	nameLabel         = "__name__"
+)
+
+// Client is an HTTP client for exporting and importing
+// time series via native protocol.
+type Client struct {
+	AuthCfg              *auth.Config
+	Addr                 string
+	ExtraLabels          []string
+	DisableHTTPKeepAlive bool
+}
+
+// LabelValues represents series from api/v1/series response
+type LabelValues map[string]string
+
+// Response represents response from api/v1/series
+type Response struct {
+	Status string        `json:"status"`
+	Series []LabelValues `json:"data"`
+}
+
+// Explore finds series by provided filter from api/v1/series
+func (c *Client) Explore(ctx context.Context, f Filter, tenantID string) (map[string]struct{}, error) {
+	url := fmt.Sprintf("%s/%s", c.Addr, nativeSeriesAddr)
+	if tenantID != "" {
+		url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeSeriesAddr)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
+	}
+
+	params := req.URL.Query()
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	params.Set("match[]", f.Match)
+	req.URL.RawQuery = params.Encode()
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("series request failed: %s", err)
+	}
+
+	var response Response
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return nil, fmt.Errorf("cannot decode series response: %s", err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, fmt.Errorf("cannot close series response body: %s", err)
+	}
+	names := make(map[string]struct{})
+	for _, series := range response.Series {
+		// TODO: consider tweaking /api/v1/series API to return metric names only
+		// this could make explore response much lighter.
+		for key, value := range series {
+			if key != nameLabel {
+				continue
+			}
+			if _, ok := names[value]; ok {
+				continue
+			}
+			names[value] = struct{}{}
+		}
+	}
+	return names, nil
+}
+
+// ImportPipe uses pipe reader in request to process data
+func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
+	if err != nil {
+		return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
+	}
+
+	importResp, err := c.do(req, http.StatusNoContent)
+	if err != nil {
+		return fmt.Errorf("import request failed: %s", err)
+	}
+	if err := importResp.Body.Close(); err != nil {
+		return fmt.Errorf("cannot close import response body: %s", err)
+	}
+	return nil
+}
+
+// ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
+func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
+	}
+
+	params := req.URL.Query()
+	params.Set("match[]", f.Match)
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	req.URL.RawQuery = params.Encode()
+
+	// disable compression since it is meaningless for native format
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("export request failed: %w", err)
+	}
+	return resp.Body, nil
+}
+
+// GetSourceTenants discovers tenants by provided filter
+func (c *Client) GetSourceTenants(ctx context.Context, f Filter) ([]string, error) {
+	u := fmt.Sprintf("%s/%s", c.Addr, nativeTenantsAddr)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", u, err)
+	}
+
+	params := req.URL.Query()
+	if f.TimeStart != "" {
+		params.Set("start", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		params.Set("end", f.TimeEnd)
+	}
+	req.URL.RawQuery = params.Encode()
+
+	resp, err := c.do(req, http.StatusOK)
+	if err != nil {
+		return nil, fmt.Errorf("tenants request failed: %s", err)
+	}
+
+	var r struct {
+		Tenants []string `json:"data"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		return nil, fmt.Errorf("cannot decode tenants response: %s", err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, fmt.Errorf("cannot close tenants response body: %s", err)
+	}
+
+	return r.Tenants, nil
+}
+
+func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
+	if c.AuthCfg != nil {
+		c.AuthCfg.SetHeaders(req, true)
+	}
+	var httpClient = &http.Client{Transport: &http.Transport{DisableKeepAlives: c.DisableHTTPKeepAlive}}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error when performing request: %w", err)
+	}
+
+	if resp.StatusCode != expSC {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body for status code %d: %s", resp.StatusCode, err)
+		}
+		return nil, fmt.Errorf("unexpected response code %d: %s", resp.StatusCode, string(body))
+	}
+	return resp, err
+}

app/vmctl/native/filter.go

@@ -0,0 +1,22 @@
+package native
+
+import "fmt"
+
+// Filter represents request filter
+type Filter struct {
+	Match     string
+	TimeStart string
+	TimeEnd   string
+	Chunk     string
+}
+
+func (f Filter) String() string {
+	s := fmt.Sprintf("\n\tfilter: match[]=%s", f.Match)
+	if f.TimeStart != "" {
+		s += fmt.Sprintf("\n\tstart: %s", f.TimeStart)
+	}
+	if f.TimeEnd != "" {
+		s += fmt.Sprintf("\n\tend: %s", f.TimeEnd)
+	}
+	return s
+}

app/vmctl/prometheus_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"testing"
 	"time"
@@ -58,7 +59,7 @@ func Test_prometheusProcessor_run(t *testing.T) {
 					return client
 				},
 				im: func(vmCfg vm.Config) *vm.Importer {
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
 					if err != nil {
 						t.Fatalf("error init importer: %s", err)
 					}
@@ -95,7 +96,7 @@ func Test_prometheusProcessor_run(t *testing.T) {
 					return client
 				},
 				im: func(vmCfg vm.Config) *vm.Importer {
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
 					if err != nil {
 						t.Fatalf("error init importer: %s", err)
 					}

app/vmctl/remote_read_test.go

@@ -110,6 +110,7 @@ func TestRemoteRead(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
 			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
 			defer remoteReadServer.Close()
 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
@@ -139,7 +140,7 @@ func TestRemoteRead(t *testing.T) {
 
 			tt.vmCfg.Addr = remoteWriteServer.URL()
 
-			importer, err := vm.NewImporter(tt.vmCfg)
+			importer, err := vm.NewImporter(ctx, tt.vmCfg)
 			if err != nil {
 				t.Fatalf("failed to create VM importer: %s", err)
 			}
@@ -156,7 +157,6 @@ func TestRemoteRead(t *testing.T) {
 				cc: 1,
 			}
 
-			ctx := context.Background()
 			err = rmp.run(ctx, true, false)
 			if err != nil {
 				t.Fatalf("failed to run remote read processor: %s", err)
@@ -263,6 +263,7 @@ func TestSteamRemoteRead(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
 			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
 			defer remoteReadServer.Close()
 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
@@ -292,7 +293,7 @@ func TestSteamRemoteRead(t *testing.T) {
 
 			tt.vmCfg.Addr = remoteWriteServer.URL()
 
-			importer, err := vm.NewImporter(tt.vmCfg)
+			importer, err := vm.NewImporter(ctx, tt.vmCfg)
 			if err != nil {
 				t.Fatalf("failed to create VM importer: %s", err)
 			}
@@ -309,7 +310,6 @@ func TestSteamRemoteRead(t *testing.T) {
 				cc: 1,
 			}
 
-			ctx := context.Background()
 			err = rmp.run(ctx, true, false)
 			if err != nil {
 				t.Fatalf("failed to run remote read processor: %s", err)

app/vmctl/remoteread/remoteread.go

@@ -21,7 +21,7 @@ import (
 )
 
 const (
-	defaultReadTimeout = 30 * time.Second
+	defaultReadTimeout = 5 * time.Minute
 	remoteReadPath     = "/api/v1/read"
 	healthPath         = "/-/healthy"
 )
@@ -157,7 +157,7 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 // Ping checks the health of the read source
 func (c *Client) Ping() error {
 	url := c.addr + healthPath
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", url, err)
 	}
@@ -174,7 +174,7 @@ func (c *Client) Ping() error {
 func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback) error {
 	r := bytes.NewReader(data)
 	url := c.addr + remoteReadPath
-	req, err := http.NewRequest("POST", url, r)
+	req, err := http.NewRequest(http.MethodPost, url, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}

app/vmctl/terminal/terminal.go

@@ -0,0 +1,6 @@
+package terminal
+
+// IsTerminal returns true if the file descriptor is terminal
+func IsTerminal(fd int) bool {
+	return isTerminal(fd)
+}

app/vmctl/terminal/unix.go

@@ -0,0 +1,13 @@
+//go:build aix || linux || solaris || zos
+// +build aix linux solaris zos
+
+package terminal
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETS
+
+func isTerminal(fd int) bool {
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
+}

app/vmctl/terminal/unix_bsd.go

@@ -0,0 +1,13 @@
+//go:build darwin || freebsd || openbsd
+// +build darwin freebsd openbsd
+
+package terminal
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TIOCGETA
+
+func isTerminal(fd int) bool {
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
+}

app/vmctl/terminal/windows..go

@@ -0,0 +1,8 @@
+//go:build windows
+// +build windows
+
+package terminal
+
+func isTerminal(fd int) bool {
+	return true
+}

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -2,39 +2,70 @@ package remote_read_integration
 
 import (
 	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"sort"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 )
 
+// LabelValues represents series from api/v1/series response
+type LabelValues map[string]string
+
+// Response represents response from api/v1/series
+type Response struct {
+	Status string        `json:"status"`
+	Series []LabelValues `json:"data"`
+}
+
+// RemoteWriteServer represents fake remote write server with database
 type RemoteWriteServer struct {
-	server *httptest.Server
-	series []vm.TimeSeries
+	server         *httptest.Server
+	series         []vm.TimeSeries
+	expectedSeries []vm.TimeSeries
 }
 
 // NewRemoteWriteServer prepares test remote write server
 func NewRemoteWriteServer(t *testing.T) *RemoteWriteServer {
 	rws := &RemoteWriteServer{series: make([]vm.TimeSeries, 0)}
 	mux := http.NewServeMux()
+
 	mux.Handle("/api/v1/import", rws.getWriteHandler(t))
 	mux.Handle("/health", rws.handlePing())
+	mux.Handle("/api/v1/series", rws.seriesHandler())
+	mux.Handle("/api/v1/export/native", rws.exportNativeHandler())
+	mux.Handle("/api/v1/import/native", rws.importNativeHandler(t))
 	rws.server = httptest.NewServer(mux)
 	return rws
 }
 
-// Close closes the server.
+// Close closes the server
 func (rws *RemoteWriteServer) Close() {
 	rws.server.Close()
 }
 
-func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
+// Series saves generated series for fake database
+func (rws *RemoteWriteServer) Series(series []vm.TimeSeries) {
 	rws.series = append(rws.series, series...)
 }
 
+// ExpectedSeries saves expected results to check in the handler
+func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
+	rws.expectedSeries = append(rws.expectedSeries, series...)
+}
+
+// URL returns server url
 func (rws *RemoteWriteServer) URL() string {
 	return rws.server.URL
 }
@@ -68,13 +99,14 @@ func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
 			rows.Reset()
 		}
 
-		if !reflect.DeepEqual(tss, rws.series) {
+		if !reflect.DeepEqual(tss, rws.expectedSeries) {
 			w.WriteHeader(http.StatusInternalServerError)
-			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.series, tss)
+			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.expectedSeries, tss)
 			return
 		}
 
 		w.WriteHeader(http.StatusNoContent)
+		return
 	})
 }
 
@@ -84,3 +116,146 @@ func (rws *RemoteWriteServer) handlePing() http.Handler {
 		_, _ = w.Write([]byte("OK"))
 	})
 }
+
+func (rws *RemoteWriteServer) seriesHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var labelValues []LabelValues
+		for _, ser := range rws.series {
+			metricNames := make(LabelValues)
+			if ser.Name != "" {
+				metricNames["__name__"] = ser.Name
+			}
+			for _, p := range ser.LabelPairs {
+				metricNames[p.Name] = p.Value
+			}
+			labelValues = append(labelValues, metricNames)
+		}
+
+		resp := Response{
+			Status: "success",
+			Series: labelValues,
+		}
+
+		err := json.NewEncoder(w).Encode(resp)
+		if err != nil {
+			log.Printf("error send series: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func (rws *RemoteWriteServer) exportNativeHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		now := time.Now()
+		err := prometheus.ExportNativeHandler(now, w, r)
+		if err != nil {
+			log.Printf("error export series via native protocol: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+		return
+	})
+}
+
+func (rws *RemoteWriteServer) importNativeHandler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		common.StartUnmarshalWorkers()
+		defer common.StopUnmarshalWorkers()
+
+		var gotTimeSeries []vm.TimeSeries
+
+		err := stream.Parse(r.Body, false, func(block *stream.Block) error {
+			mn := &block.MetricName
+			var timeseries vm.TimeSeries
+			timeseries.Name = string(mn.MetricGroup)
+			timeseries.Timestamps = append(timeseries.Timestamps, block.Timestamps...)
+			timeseries.Values = append(timeseries.Values, block.Values...)
+
+			for i := range mn.Tags {
+				tag := &mn.Tags[i]
+				timeseries.LabelPairs = append(timeseries.LabelPairs, vm.LabelPair{
+					Name:  string(tag.Key),
+					Value: string(tag.Value),
+				})
+			}
+
+			gotTimeSeries = append(gotTimeSeries, timeseries)
+
+			return nil
+		})
+		if err != nil {
+			log.Printf("error parse stream blocks: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		// got timeseries should be sorted
+		// because they are processed independently
+		sort.SliceStable(gotTimeSeries, func(i, j int) bool {
+			iv, jv := gotTimeSeries[i], gotTimeSeries[j]
+			switch {
+			case iv.Values[0] != jv.Values[0]:
+				return iv.Values[0] < jv.Values[0]
+			case iv.Timestamps[0] != jv.Timestamps[0]:
+				return iv.Timestamps[0] < jv.Timestamps[0]
+			default:
+				return iv.Name < jv.Name
+			}
+		})
+
+		if !reflect.DeepEqual(gotTimeSeries, rws.expectedSeries) {
+			w.WriteHeader(http.StatusInternalServerError)
+			t.Fatalf("datasets not equal, expected: %#v;\n got: %#v", rws.expectedSeries, gotTimeSeries)
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+		return
+	})
+}
+
+// GenerateVNSeries generates test timeseries
+func GenerateVNSeries(start, end, numOfSeries, numOfSamples int64) []vm.TimeSeries {
+	var ts []vm.TimeSeries
+	j := 0
+	for i := 0; i < int(numOfSeries); i++ {
+		if i%3 == 0 {
+			j++
+		}
+
+		timeSeries := vm.TimeSeries{
+			Name: fmt.Sprintf("vm_metric_%d", j),
+			LabelPairs: []vm.LabelPair{
+				{Name: "job", Value: strconv.Itoa(i)},
+			},
+		}
+
+		ts = append(ts, timeSeries)
+	}
+
+	for i := range ts {
+		t, v := generateTimeStampsAndValues(i, start, end, numOfSamples)
+		ts[i].Timestamps = t
+		ts[i].Values = v
+	}
+
+	return ts
+}
+
+func generateTimeStampsAndValues(idx int, startTime, endTime, numOfSamples int64) ([]int64, []float64) {
+	delta := (endTime - startTime) / numOfSamples
+
+	var timestamps []int64
+	var values []float64
+	t := startTime
+	for t != endTime {
+		v := 100 * int64(idx)
+		timestamps = append(timestamps, t*1000)
+		values = append(values, float64(v))
+		t = t + delta
+	}
+
+	return timestamps, values
+}

app/vmctl/vm/vm.go

@@ -3,15 +3,16 @@ package vm
 import (
 	"bufio"
 	"compress/gzip"
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
@@ -75,7 +76,8 @@ type Importer struct {
 	wg   sync.WaitGroup
 	once sync.Once
 
-	s *stats
+	s       *stats
+	backoff *backoff.Backoff
 }
 
 // ResetStats resets im stats.
@@ -107,7 +109,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 }
 
 // NewImporter creates new Importer for the given cfg.
-func NewImporter(cfg Config) (*Importer, error) {
+func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 	if cfg.Concurrency < 1 {
 		return nil, fmt.Errorf("concurrency can't be lower than 1")
 	}
@@ -136,6 +138,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
+		backoff:    backoff.New(),
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
@@ -154,7 +157,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		}
 		go func(bar *pb.ProgressBar) {
 			defer im.wg.Done()
-			im.startWorker(bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
+			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
 		}(bar)
 	}
 	im.ResetStats()
@@ -205,7 +208,7 @@ func (im *Importer) Close() {
 	})
 }
 
-func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigures, roundDigits int) {
+func (im *Importer) startWorker(ctx context.Context, bar *pb.ProgressBar, batchSize, significantFigures, roundDigits int) {
 	var batch []*TimeSeries
 	var dataPoints int
 	var waitForBatch time.Time
@@ -219,7 +222,9 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 			exitErr := &ImportError{
 				Batch: batch,
 			}
-			if err := im.Import(batch); err != nil {
+			retryableFunc := func() error { return im.Import(batch) }
+			_, err := im.backoff.Retry(ctx, retryableFunc)
+			if err != nil {
 				exitErr.Err = err
 			}
 			im.errors <- exitErr
@@ -249,7 +254,7 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 			im.s.idleDuration += time.Since(waitForBatch)
 			im.s.Unlock()
 
-			if err := im.flush(batch); err != nil {
+			if err := im.flush(ctx, batch); err != nil {
 				im.errors <- &ImportError{
 					Batch: batch,
 					Err:   err,
@@ -264,36 +269,22 @@ func (im *Importer) startWorker(bar *pb.ProgressBar, batchSize, significantFigur
 	}
 }
 
-const (
-	// TODO: make configurable
-	backoffRetries     = 5
-	backoffFactor      = 1.7
-	backoffMinDuration = time.Second
-)
-
-func (im *Importer) flush(b []*TimeSeries) error {
-	var err error
-	for i := 0; i < backoffRetries; i++ {
-		err = im.Import(b)
-		if err == nil {
-			return nil
-		}
-		if errors.Is(err, ErrBadRequest) {
-			return err // fail fast if not recoverable
-		}
-		im.s.Lock()
-		im.s.retries++
-		im.s.Unlock()
-		backoff := float64(backoffMinDuration) * math.Pow(backoffFactor, float64(i))
-		time.Sleep(time.Duration(backoff))
+func (im *Importer) flush(ctx context.Context, b []*TimeSeries) error {
+	retryableFunc := func() error { return im.Import(b) }
+	attempts, err := im.backoff.Retry(ctx, retryableFunc)
+	if err != nil {
+		return fmt.Errorf("import failed with %d retries: %s", attempts, err)
 	}
-	return fmt.Errorf("import failed with %d retries: %s", backoffRetries, err)
+	im.s.Lock()
+	im.s.retries = attempts
+	im.s.Unlock()
+	return nil
 }
 
 // Ping sends a ping to im.addr.
 func (im *Importer) Ping() error {
 	url := fmt.Sprintf("%s/health", im.addr)
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}
@@ -317,7 +308,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	}
 
 	pr, pw := io.Pipe()
-	req, err := http.NewRequest("POST", im.importPath, pr)
+	req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
 	if err != nil {
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}

app/vmctl/vm_native.go

@@ -2,177 +2,126 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log"
-	"net/http"
+	"sync"
 	"time"
 
-	"github.com/cheggaaa/pb/v3"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/cheggaaa/pb/v3"
 )
 
 type vmNativeProcessor struct {
-	filter    filter
-	rateLimit int64
+	filter native.Filter
 
-	dst          *vmNativeClient
-	src          *vmNativeClient
+	dst     *native.Client
+	src     *native.Client
+	backoff *backoff.Backoff
+
+	s            *stats
+	rateLimit    int64
 	interCluster bool
-}
-
-type vmNativeClient struct {
-	addr        string
-	user        string
-	password    string
-	extraLabels []string
-}
-
-type filter struct {
-	match     string
-	timeStart string
-	timeEnd   string
-	chunk     string
-}
-
-func (f filter) String() string {
-	s := fmt.Sprintf("\n\tfilter: match[]=%s", f.match)
-	if f.timeStart != "" {
-		s += fmt.Sprintf("\n\tstart: %s", f.timeStart)
-	}
-	if f.timeEnd != "" {
-		s += fmt.Sprintf("\n\tend: %s", f.timeEnd)
-	}
-	return s
+	cc           int
 }
 
 const (
-	nativeExportAddr  = "api/v1/export/native"
-	nativeImportAddr  = "api/v1/import/native"
-	nativeTenantsAddr = "admin/tenants"
-
-	nativeBarTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}`
+	nativeExportAddr = "api/v1/export/native"
+	nativeImportAddr = "api/v1/import/native"
+	nativeBarTpl     = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "█") "▒" "]" }} {{ percent . }}`
 )
 
-func (p *vmNativeProcessor) run(ctx context.Context) error {
-	if p.filter.chunk == "" {
-		return p.runWithFilter(ctx, p.filter)
+func (p *vmNativeProcessor) run(ctx context.Context, silent bool) error {
+	if p.cc == 0 {
+		p.cc = 1
+	}
+	p.s = &stats{
+		startTime: time.Now(),
 	}
 
-	startOfRange, err := time.Parse(time.RFC3339, p.filter.timeStart)
+	start, err := time.Parse(time.RFC3339, p.filter.TimeStart)
 	if err != nil {
-		return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeStart, p.filter.timeStart, time.RFC3339, err)
+		return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+			vmNativeFilterTimeStart, p.filter.TimeStart, time.RFC3339, err)
 	}
 
-	var endOfRange time.Time
-	if p.filter.timeEnd != "" {
-		endOfRange, err = time.Parse(time.RFC3339, p.filter.timeEnd)
+	end := time.Now().In(start.Location())
+	if p.filter.TimeEnd != "" {
+		end, err = time.Parse(time.RFC3339, p.filter.TimeEnd)
 		if err != nil {
-			return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeEnd, p.filter.timeEnd, time.RFC3339, err)
+			return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+				vmNativeFilterTimeEnd, p.filter.TimeEnd, time.RFC3339, err)
 		}
-	} else {
-		endOfRange = time.Now()
 	}
 
-	ranges, err := stepper.SplitDateRange(startOfRange, endOfRange, p.filter.chunk)
-	if err != nil {
-		return fmt.Errorf("failed to create date ranges for the given time filters: %v", err)
-	}
-
-	for rangeIdx, r := range ranges {
-		formattedStartTime := r[0].Format(time.RFC3339)
-		formattedEndTime := r[1].Format(time.RFC3339)
-		log.Printf("Processing range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
-		f := filter{
-			match:     p.filter.match,
-			timeStart: formattedStartTime,
-			timeEnd:   formattedEndTime,
-		}
-		err := p.runWithFilter(ctx, f)
-
+	ranges := [][]time.Time{{start, end}}
+	if p.filter.Chunk != "" {
+		ranges, err = stepper.SplitDateRange(start, end, p.filter.Chunk)
 		if err != nil {
-			log.Printf("processing failed for range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
-			return err
+			return fmt.Errorf("failed to create date ranges for the given time filters: %w", err)
 		}
 	}
+
+	tenants := []string{""}
+	if p.interCluster {
+		log.Printf("Discovering tenants...")
+		tenants, err = p.src.GetSourceTenants(ctx, p.filter)
+		if err != nil {
+			return fmt.Errorf("failed to get tenants: %w", err)
+		}
+		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
+		if !silent && !prompt(question) {
+			return nil
+		}
+	}
+
+	for _, tenantID := range tenants {
+		err := p.runBackfilling(ctx, tenantID, ranges, silent)
+		if err != nil {
+			return fmt.Errorf("migration failed: %s", err)
+		}
+	}
+
+	log.Println("Import finished!")
+	log.Print(p.s)
+
 	return nil
 }
 
-func (p *vmNativeProcessor) runWithFilter(ctx context.Context, f filter) error {
-	nativeImportAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.extraLabels)
+func (p *vmNativeProcessor) do(ctx context.Context, f native.Filter, srcURL, dstURL string) error {
 
+	retryableFunc := func() error { return p.runSingle(ctx, f, srcURL, dstURL) }
+	attempts, err := p.backoff.Retry(ctx, retryableFunc)
+	p.s.Lock()
+	p.s.retries += attempts
+	p.s.Unlock()
 	if err != nil {
-		return fmt.Errorf("failed to add labels to import path: %s", err)
-	}
-
-	if !p.interCluster {
-		srcURL := fmt.Sprintf("%s/%s", p.src.addr, nativeExportAddr)
-		dstURL := fmt.Sprintf("%s/%s", p.dst.addr, nativeImportAddr)
-
-		return p.runSingle(ctx, f, srcURL, dstURL)
-	}
-
-	tenants, err := p.getSourceTenants(ctx, f)
-	if err != nil {
-		return fmt.Errorf("failed to get source tenants: %s", err)
-	}
-
-	log.Printf("Discovered tenants: %v", tenants)
-	for _, tenant := range tenants {
-		// src and dst expected formats: http://vminsert:8480/ and http://vmselect:8481/
-		srcURL := fmt.Sprintf("%s/select/%s/prometheus/%s", p.src.addr, tenant, nativeExportAddr)
-		dstURL := fmt.Sprintf("%s/insert/%s/prometheus/%s", p.dst.addr, tenant, nativeImportAddr)
-
-		if err := p.runSingle(ctx, f, srcURL, dstURL); err != nil {
-			return fmt.Errorf("failed to migrate data for tenant %q: %s", tenant, err)
-		}
+		return fmt.Errorf("failed to migrate from %s to %s (retry attempts: %d): %w\nwith fileter %s", srcURL, dstURL, attempts, err, f)
 	}
 
 	return nil
 }
 
-func (p *vmNativeProcessor) runSingle(ctx context.Context, f filter, srcURL, dstURL string) error {
-	log.Printf("Initing export pipe from %q with filters: %s\n", srcURL, f)
+func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcURL, dstURL string) error {
 
-	exportReader, err := p.exportPipe(ctx, srcURL, f)
+	exportReader, err := p.src.ExportPipe(ctx, srcURL, f)
 	if err != nil {
-		return fmt.Errorf("failed to init export pipe: %s", err)
+		return fmt.Errorf("failed to init export pipe: %w", err)
 	}
 
 	pr, pw := io.Pipe()
-	sync := make(chan struct{})
+	done := make(chan struct{})
 	go func() {
-		defer func() { close(sync) }()
-		req, err := http.NewRequestWithContext(ctx, "POST", dstURL, pr)
-		if err != nil {
-			log.Fatalf("cannot create import request to %q: %s", p.dst.addr, err)
-		}
-		importResp, err := p.dst.do(req, http.StatusNoContent)
-		if err != nil {
-			log.Fatalf("import request failed: %s", err)
-		}
-		if err := importResp.Body.Close(); err != nil {
-			log.Fatalf("cannot close import response body: %s", err)
-		}
-	}()
-
-	fmt.Printf("Initing import process to %q:\n", dstURL)
-	pool := pb.NewPool()
-	bar := pb.ProgressBarTemplate(nativeBarTpl).New(0)
-	pool.Add(bar)
-	barReader := bar.NewProxyReader(exportReader)
-	if err := pool.Start(); err != nil {
-		log.Printf("error start process bars pool: %s", err)
-		return err
-	}
-	defer func() {
-		bar.Finish()
-		if err := pool.Stop(); err != nil {
-			fmt.Printf("failed to stop barpool: %+v\n", err)
+		defer func() { close(done) }()
+		if err := p.dst.ImportPipe(ctx, dstURL, pr); err != nil {
+			logger.Errorf("error initialize import pipe: %s", err)
+			return
 		}
 	}()
 
@@ -182,95 +131,193 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f filter, srcURL, dst
 		w = limiter.NewWriteLimiter(pw, rl)
 	}
 
-	_, err = io.Copy(w, barReader)
+	written, err := io.Copy(w, exportReader)
 	if err != nil {
-		return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
+		return fmt.Errorf("failed to write into %q: %s", p.dst.Addr, err)
 	}
 
+	p.s.Lock()
+	p.s.bytes += uint64(written)
+	p.s.requests++
+	p.s.Unlock()
+
 	if err := pw.Close(); err != nil {
 		return err
 	}
-	<-sync
+	<-done
 
-	log.Println("Import finished!")
 	return nil
 }
 
-func (p *vmNativeProcessor) getSourceTenants(ctx context.Context, f filter) ([]string, error) {
-	u := fmt.Sprintf("%s/%s", p.src.addr, nativeTenantsAddr)
-	req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
+func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error {
+	exportAddr := nativeExportAddr
+	srcURL := fmt.Sprintf("%s/%s", p.src.Addr, exportAddr)
+
+	importAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.ExtraLabels)
 	if err != nil {
-		return nil, fmt.Errorf("cannot create request to %q: %s", u, err)
+		return fmt.Errorf("failed to add labels to import path: %s", err)
+	}
+	dstURL := fmt.Sprintf("%s/%s", p.dst.Addr, importAddr)
+
+	if p.interCluster {
+		srcURL = fmt.Sprintf("%s/select/%s/prometheus/%s", p.src.Addr, tenantID, exportAddr)
+		dstURL = fmt.Sprintf("%s/insert/%s/prometheus/%s", p.dst.Addr, tenantID, importAddr)
 	}
 
-	params := req.URL.Query()
-	if f.timeStart != "" {
-		params.Set("start", f.timeStart)
+	barPrefix := "Requests to make"
+	initMessage := "Initing import process from %q to %q with filter %s"
+	initParams := []interface{}{srcURL, dstURL, p.filter.String()}
+	if p.interCluster {
+		barPrefix = fmt.Sprintf("Requests to make for tenant %s", tenantID)
+		initMessage = "Initing import process from %q to %q with filter %s for tenant %s"
+		initParams = []interface{}{srcURL, dstURL, p.filter.String(), tenantID}
 	}
-	if f.timeEnd != "" {
-		params.Set("end", f.timeEnd)
-	}
-	req.URL.RawQuery = params.Encode()
 
-	resp, err := p.src.do(req, http.StatusOK)
+	fmt.Println("") // extra line for better output formatting
+	log.Printf(initMessage, initParams...)
+
+	log.Printf("Exploring metrics...")
+	metrics, err := p.src.Explore(ctx, p.filter, tenantID)
 	if err != nil {
-		return nil, fmt.Errorf("tenants request failed: %s", err)
+		return fmt.Errorf("cannot get metrics from source %s: %w", p.src.Addr, err)
 	}
 
-	var r struct {
-		Tenants []string `json:"data"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
-		return nil, fmt.Errorf("cannot decode tenants response: %s", err)
+	if len(metrics) == 0 {
+		return fmt.Errorf("no metrics found")
 	}
 
-	if err := resp.Body.Close(); err != nil {
-		return nil, fmt.Errorf("cannot close tenants response body: %s", err)
-	}
-
-	return r.Tenants, nil
-}
-
-func (p *vmNativeProcessor) exportPipe(ctx context.Context, url string, f filter) (io.ReadCloser, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create request to %q: %s", p.src.addr, err)
-	}
-
-	params := req.URL.Query()
-	params.Set("match[]", f.match)
-	if f.timeStart != "" {
-		params.Set("start", f.timeStart)
-	}
-	if f.timeEnd != "" {
-		params.Set("end", f.timeEnd)
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// disable compression since it is meaningless for native format
-	req.Header.Set("Accept-Encoding", "identity")
-	resp, err := p.src.do(req, http.StatusOK)
-	if err != nil {
-		return nil, fmt.Errorf("export request failed: %s", err)
-	}
-	return resp.Body, nil
-}
-
-func (c *vmNativeClient) do(req *http.Request, expSC int) (*http.Response, error) {
-	if c.user != "" {
-		req.SetBasicAuth(c.user, c.password)
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error when performing request: %s", err)
-	}
-
-	if resp.StatusCode != expSC {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read response body for status code %d: %s", resp.StatusCode, err)
+	foundSeriesMsg := fmt.Sprintf("Found %d metrics to import", len(metrics))
+	if !p.interCluster {
+		// do not prompt for intercluster because there could be many tenants,
+		// and we don't want to interrupt the process when moving to the next tenant.
+		question := foundSeriesMsg + ". Continue?"
+		if !silent && !prompt(question) {
+			return nil
 		}
-		return nil, fmt.Errorf("unexpected response code %d: %s", resp.StatusCode, string(body))
+	} else {
+		log.Print(foundSeriesMsg)
 	}
-	return resp, err
+
+	processingMsg := fmt.Sprintf("Requests to make: %d", len(metrics)*len(ranges))
+	if len(ranges) > 1 {
+		processingMsg = fmt.Sprintf("Selected time range will be split into %d ranges according to %q step. %s", len(ranges), p.filter.Chunk, processingMsg)
+	}
+	log.Print(processingMsg)
+
+	var bar *pb.ProgressBar
+	if !silent {
+		bar = pb.ProgressBarTemplate(fmt.Sprintf(nativeBarTpl, barPrefix)).New(len(metrics) * len(ranges))
+		bar.Start()
+		defer bar.Finish()
+	}
+
+	filterCh := make(chan native.Filter)
+	errCh := make(chan error, p.cc)
+
+	var wg sync.WaitGroup
+	for i := 0; i < p.cc; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for f := range filterCh {
+				if err := p.do(ctx, f, srcURL, dstURL); err != nil {
+					errCh <- err
+					return
+				}
+				if bar != nil {
+					bar.Increment()
+				}
+			}
+		}()
+	}
+
+	// any error breaks the import
+	for s := range metrics {
+
+		match, err := buildMatchWithFilter(p.filter.Match, s)
+		if err != nil {
+			logger.Errorf("failed to build export filters: %s", err)
+			continue
+		}
+
+		for _, times := range ranges {
+			select {
+			case <-ctx.Done():
+				return fmt.Errorf("context canceled")
+			case infErr := <-errCh:
+				return fmt.Errorf("native error: %s", infErr)
+			case filterCh <- native.Filter{
+				Match:     match,
+				TimeStart: times[0].Format(time.RFC3339),
+				TimeEnd:   times[1].Format(time.RFC3339),
+			}:
+			}
+		}
+	}
+
+	close(filterCh)
+	wg.Wait()
+	close(errCh)
+
+	for err := range errCh {
+		return fmt.Errorf("import process failed: %s", err)
+	}
+
+	return nil
+}
+
+// stats represents client statistic
+// when processing data
+type stats struct {
+	sync.Mutex
+	startTime time.Time
+	bytes     uint64
+	requests  uint64
+	retries   uint64
+}
+
+func (s *stats) String() string {
+	s.Lock()
+	defer s.Unlock()
+
+	totalImportDuration := time.Since(s.startTime)
+	totalImportDurationS := totalImportDuration.Seconds()
+	bytesPerS := byteCountSI(0)
+	if s.bytes > 0 && totalImportDurationS > 0 {
+		bytesPerS = byteCountSI(int64(float64(s.bytes) / totalImportDurationS))
+	}
+
+	return fmt.Sprintf("VictoriaMetrics importer stats:\n"+
+		"  time spent while importing: %v;\n"+
+		"  total bytes: %s;\n"+
+		"  bytes/s: %s;\n"+
+		"  requests: %d;\n"+
+		"  requests retries: %d;",
+		totalImportDuration,
+		byteCountSI(int64(s.bytes)), bytesPerS,
+		s.requests, s.retries)
+}
+
+func byteCountSI(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB",
+		float64(b)/float64(div), "kMGTPE"[exp])
+}
+
+func buildMatchWithFilter(filter string, metricName string) (string, error) {
+	labels, err := promutils.NewLabelsFromString(filter)
+	if err != nil {
+		return "", err
+	}
+	labels.Set("__name__", metricName)
+
+	return labels.String(), nil
 }

app/vmctl/vm_native_test.go

@@ -2,117 +2,360 @@ package main
 
 import (
 	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+	remote_read_integration "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-// If you want to run this test:
-// 1. run two instances of victoriametrics and define -httpListenAddr for both or just for second instance
-// 2. define srcAddr and dstAddr const with your victoriametrics addresses
-// 3. define matchFilter const with your importing data
-// 4. define timeStartFilter
-// 5. run each test one by one
-
 const (
-	matchFilter     = `{job="avalanche"}`
-	timeStartFilter = "2020-01-01T20:07:00Z"
-	timeEndFilter   = "2020-08-01T20:07:00Z"
-	srcAddr         = "http://127.0.0.1:8428"
-	dstAddr         = "http://127.0.0.1:8528"
+	storagePath     = "TestStorage"
+	retentionPeriod = "100y"
 )
 
-// This test simulates close process if user abort it
 func Test_vmNativeProcessor_run(t *testing.T) {
-	t.Skip()
+
+	processFlags()
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	defer func() {
+		vmstorage.Stop()
+		if err := os.RemoveAll(storagePath); err != nil {
+			log.Fatalf("cannot remove %q: %s", storagePath, err)
+		}
+	}()
+
 	type fields struct {
-		filter    filter
-		rateLimit int64
-		dst       *vmNativeClient
-		src       *vmNativeClient
+		filter       native.Filter
+		dst          *native.Client
+		src          *native.Client
+		backoff      *backoff.Backoff
+		s            *stats
+		rateLimit    int64
+		interCluster bool
+		cc           int
+		matchName    string
+		matchValue   string
 	}
+	type args struct {
+		ctx    context.Context
+		silent bool
+	}
+
 	tests := []struct {
-		name    string
-		fields  fields
-		closer  func(cancelFunc context.CancelFunc)
-		wantErr bool
+		name           string
+		fields         fields
+		args           args
+		vmSeries       func(start, end, numOfSeries, numOfSamples int64) []vm.TimeSeries
+		expectedSeries []vm.TimeSeries
+		start          string
+		end            string
+		numOfSamples   int64
+		numOfSeries    int64
+		chunk          string
+		wantErr        bool
 	}{
 		{
-			name: "simulate syscall.SIGINT",
+			name:         "step minute on minute time range",
+			start:        "2022-11-25T11:23:05+02:00",
+			end:          "2022-11-27T11:24:05+02:00",
+			numOfSamples: 2,
+			numOfSeries:  3,
+			chunk:        stepper.StepMinute,
 			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
+				filter:       native.Filter{},
+				backoff:      backoff.New(),
+				rateLimit:    0,
+				interCluster: false,
+				cc:           1,
+				matchName:    "__name__",
+				matchValue:   ".*",
+			},
+			args: args{
+				ctx:    context.Background(),
+				silent: true,
+			},
+			vmSeries: remote_read_integration.GenerateVNSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{0, 0},
 				},
-				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{100, 100},
 				},
-				src: &vmNativeClient{
-					addr: srcAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{200, 200},
 				},
 			},
-			closer: func(cancelFunc context.CancelFunc) {
-				time.Sleep(time.Second * 5)
-				cancelFunc()
-			},
-			wantErr: true,
-		},
-		{
-			name: "simulate correct work",
-			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
-				},
-				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
-				},
-				src: &vmNativeClient{
-					addr: srcAddr,
-				},
-			},
-			closer:  func(cancelFunc context.CancelFunc) {},
 			wantErr: false,
 		},
 		{
-			name: "simulate correct work with chunking",
+			name:         "step month on month time range",
+			start:        "2022-09-26T11:23:05+02:00",
+			end:          "2022-11-26T11:24:05+02:00",
+			numOfSamples: 2,
+			numOfSeries:  3,
+			chunk:        stepper.StepMonth,
 			fields: fields{
-				filter: filter{
-					match:     matchFilter,
-					timeStart: timeStartFilter,
-					timeEnd:   timeEndFilter,
-					chunk:     stepper.StepMonth,
+				filter:       native.Filter{},
+				backoff:      backoff.New(),
+				rateLimit:    0,
+				interCluster: false,
+				cc:           1,
+				matchName:    "__name__",
+				matchValue:   ".*",
+			},
+			args: args{
+				ctx:    context.Background(),
+				silent: true,
+			},
+			vmSeries: remote_read_integration.GenerateVNSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{0},
 				},
-				rateLimit: 0,
-				dst: &vmNativeClient{
-					addr: dstAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{0},
 				},
-				src: &vmNativeClient{
-					addr: srcAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{200},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{200},
 				},
 			},
-			closer:  func(cancelFunc context.CancelFunc) {},
 			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ctx, cancelFn := context.WithCancel(context.Background())
-			p := &vmNativeProcessor{
-				filter:    tt.fields.filter,
-				rateLimit: tt.fields.rateLimit,
-				dst:       tt.fields.dst,
-				src:       tt.fields.src,
+			src := remote_read_integration.NewRemoteWriteServer(t)
+			dst := remote_read_integration.NewRemoteWriteServer(t)
+
+			defer func() {
+				src.Close()
+				dst.Close()
+			}()
+
+			start, err := time.Parse(time.RFC3339, tt.start)
+			if err != nil {
+				t.Fatalf("Error parse start time: %s", err)
 			}
 
-			tt.closer(cancelFn)
+			end, err := time.Parse(time.RFC3339, tt.end)
+			if err != nil {
+				t.Fatalf("Error parse end time: %s", err)
+			}
 
-			if err := p.run(ctx); (err != nil) != tt.wantErr {
+			tt.fields.filter.Match = fmt.Sprintf("%s=%q", tt.fields.matchName, tt.fields.matchValue)
+			tt.fields.filter.TimeStart = tt.start
+			tt.fields.filter.TimeEnd = tt.end
+
+			rws := tt.vmSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+
+			src.Series(rws)
+			dst.ExpectedSeries(tt.expectedSeries)
+
+			if err := fillStorage(rws); err != nil {
+				t.Fatalf("error add series to storage: %s", err)
+			}
+
+			tt.fields.src = &native.Client{
+				AuthCfg:              nil,
+				Addr:                 src.URL(),
+				ExtraLabels:          []string{},
+				DisableHTTPKeepAlive: false,
+			}
+			tt.fields.dst = &native.Client{
+				AuthCfg:              nil,
+				Addr:                 dst.URL(),
+				ExtraLabels:          []string{},
+				DisableHTTPKeepAlive: false,
+			}
+
+			p := &vmNativeProcessor{
+				filter:       tt.fields.filter,
+				dst:          tt.fields.dst,
+				src:          tt.fields.src,
+				backoff:      tt.fields.backoff,
+				s:            tt.fields.s,
+				rateLimit:    tt.fields.rateLimit,
+				interCluster: tt.fields.interCluster,
+				cc:           tt.fields.cc,
+			}
+
+			if err := p.run(tt.args.ctx, tt.args.silent); (err != nil) != tt.wantErr {
 				t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr)
 			}
+			deleted, err := deleteSeries(tt.fields.matchName, tt.fields.matchValue)
+			if err != nil {
+				t.Fatalf("error delete series: %s", err)
+			}
+			if int64(deleted) != tt.numOfSeries {
+				t.Fatalf("expected deleted series %d; got deleted series %d", tt.numOfSeries, deleted)
+			}
+		})
+	}
+}
+
+func processFlags() {
+	flag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "retentionPeriod", value: retentionPeriod},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func fillStorage(series []vm.TimeSeries) error {
+	var mrs []storage.MetricRow
+	for _, series := range series {
+		var labels []prompb.Label
+		for _, lp := range series.LabelPairs {
+			labels = append(labels, prompb.Label{Name: []byte(lp.Name), Value: []byte(lp.Value)})
+		}
+		if series.Name != "" {
+			labels = append(labels, prompb.Label{Name: []byte("__name__"), Value: []byte(series.Name)})
+		}
+		mr := storage.MetricRow{}
+		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+
+		timestamps := series.Timestamps
+		values := series.Values
+		for i, value := range values {
+			mr.Timestamp = timestamps[i]
+			mr.Value = value
+			mrs = append(mrs, mr)
+		}
+	}
+
+	if err := vmstorage.AddRows(mrs); err != nil {
+		return fmt.Errorf("unexpected error in AddRows: %s", err)
+	}
+	vmstorage.Storage.DebugFlush()
+	return nil
+}
+
+func deleteSeries(name, value string) (int, error) {
+	tfs := storage.NewTagFilters()
+	if err := tfs.Add([]byte(name), []byte(value), false, true); err != nil {
+		return 0, fmt.Errorf("unexpected error in TagFilters.Add: %w", err)
+	}
+	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs})
+}
+
+func Test_buildMatchWithFilter(t *testing.T) {
+	tests := []struct {
+		name       string
+		filter     string
+		metricName string
+		want       string
+		wantErr    bool
+	}{
+		{
+			name:       "parsed metric with label",
+			filter:     `{__name__="http_request_count_total",cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "metric name with label",
+			filter:     `http_request_count_total{cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "parsed metric with regexp value",
+			filter:     `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "only label with regexp",
+			filter:     `{cluster~=".*"}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "many labels in filter with regexp",
+			filter:     `{cluster~=".*",job!=""}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",job!="",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "match with error",
+			filter:     `{cluster=~".*"}`,
+			metricName: "http_request_count_total",
+			want:       ``,
+			wantErr:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildMatchWithFilter(tt.filter, tt.metricName)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("buildMatchWithFilter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("buildMatchWithFilter() got = %v, want %v", got, tt.want)
+			}
 		})
 	}
 }

app/vmgateway/README.md

@@ -206,6 +206,56 @@ mwIDAQAB
 ```
 This command will result in 3 keys loaded: 2 keys from files and 1 from command line.
 
+### Using OpenID discovery endpoint for JWT signature verification
+
+`vmgateway` supports using OpenID discovery endpoint for JWKS keys discovery.
+
+In order to enable [OpenID discovery](https://openid.net/specs/openid-connect-discovery-1_0.html) endpoint for JWT signature verification, you need to specify OpenID discovery endpoint URLs by using `auth.oidcDiscoveryEndpoints` flag.
+When `auth.oidcDiscoveryEndpoints` is specified `vmageteway` will fetch JWKS keys from the specified endpoint and use them for JWT signature verification.
+
+Example usage for tokens issued by Azure Active Directory:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.oidcDiscoveryEndpoints=https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration
+```
+
+Example usage for tokens issued by Google:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.oidcDiscoveryEndpoints=https://accounts.google.com/.well-known/openid-configuration
+```
+
+### Using JWKS endpoint for JWT signature verification
+
+`vmgateway` supports using JWKS endpoint for JWT signature verification.
+
+In order to enable JWKS endpoint for JWT signature verification, you need to specify JWKS endpoint URL by using `auth.jwksEndpoints` flag.
+When `auth.jwksEndpoints` is specified `vmageteway` will fetch public keys from the specified endpoint and use them for JWT signature verification.
+
+Example usage for tokens issued by Azure Active Directory:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.jwksEndpoints=https://login.microsoftonline.com/common/discovery/v2.0/keys
+```
+
+Example usage for tokens issued by Google:
+```console
+/bin/vmgateway -eula \
+  -enable.auth \
+  -write.url=http://localhost:8480 \
+  -read.url=http://localhost:8481 \
+  -auth.jwksEndpoints=https://www.googleapis.com/oauth2/v3/certs
+```
+
 ## Configuration
 
 The shortlist of configuration flags include the following:
@@ -213,6 +263,12 @@ The shortlist of configuration flags include the following:
 ```console
   -auth.httpHeader string
      HTTP header name to look for JWT authorization token (default "Authorization")
+  -auth.jwksEndpoints array
+     JWKS endpoints to fetch keys for JWT tokens signature verification
+     Supports an array of values separated by comma or specified via multiple flags.
+  -auth.oidcDiscoveryEndpoints array
+     OpenID Connect discovery endpoints to fetch keys for JWT tokens signature verification
+     Supports an array of values separated by comma or specified via multiple flags.
   -auth.publicKeyFiles array
      Path file with public key to verify JWT token signature
      Supports an array of values separated by comma or specified via multiple flags.

app/vminsert/common/insert_ctx.go

@@ -137,7 +137,8 @@ func (ctx *InsertCtx) ApplyRelabeling() {
 
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
-	if sa != nil && !ctx.skipStreamAggr {
+	sas := sasGlobal.Load()
+	if sas != nil && !ctx.skipStreamAggr {
 		ctx.streamAggrCtx.push(ctx.mrs)
 		if !*streamAggrKeepInput {
 			ctx.Reset(0)
@@ -146,7 +147,7 @@ func (ctx *InsertCtx) FlushBufs() error {
 	}
 	// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
 	// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
-	// used at every ParseStream() call under lib/protoparser/*/streamparser.go
+	// used at every stream.Parse() call under lib/protoparser/*
 	err := vmstorage.AddRows(ctx.mrs)
 	ctx.Reset(0)
 	if err == nil {

app/vminsert/common/streamaggr.go

@@ -2,15 +2,20 @@ package common
 
 import (
 	"flag"
+	"fmt"
+	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -24,28 +29,97 @@ var (
 		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
+var (
+	saCfgReloaderStopCh = make(chan struct{})
+	saCfgReloaderWG     sync.WaitGroup
+
+	saCfgReloads   = metrics.NewCounter(`vminsert_streamagg_config_reloads_total`)
+	saCfgReloadErr = metrics.NewCounter(`vminsert_streamagg_config_reloads_errors_total`)
+	saCfgSuccess   = metrics.NewCounter(`vminsert_streamagg_config_last_reload_successful`)
+	saCfgTimestamp = metrics.NewCounter(`vminsert_streamagg_config_last_reload_success_timestamp_seconds`)
+
+	sasGlobal atomic.Pointer[streamaggr.Aggregators]
+)
+
+// CheckStreamAggrConfig checks config pointed by -stramaggr.config
+func CheckStreamAggrConfig() error {
+	if *streamAggrConfig == "" {
+		return nil
+	}
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
+	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushNoop, *streamAggrDedupInterval)
+	if err != nil {
+		return fmt.Errorf("error when loading -streamAggr.config=%q: %w", *streamAggrConfig, err)
+	}
+	sas.MustStop()
+	return nil
+}
+
 // InitStreamAggr must be called after flag.Parse and before using the common package.
 //
 // MustStopStreamAggr must be called when stream aggr is no longer needed.
 func InitStreamAggr() {
 	if *streamAggrConfig == "" {
-		// Nothing to initialize
 		return
 	}
-	a, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
+
+	sighupCh := procutil.NewSighupChan()
+
+	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
 	if err != nil {
 		logger.Fatalf("cannot load -streamAggr.config=%q: %s", *streamAggrConfig, err)
 	}
-	sa = a
+	sasGlobal.Store(sas)
+	saCfgSuccess.Set(1)
+	saCfgTimestamp.Set(fasttime.UnixTimestamp())
+
+	// Start config reloader.
+	saCfgReloaderWG.Add(1)
+	go func() {
+		defer saCfgReloaderWG.Done()
+		for {
+			select {
+			case <-sighupCh:
+			case <-saCfgReloaderStopCh:
+				return
+			}
+			reloadStreamAggrConfig()
+		}
+	}()
+}
+
+func reloadStreamAggrConfig() {
+	logger.Infof("reloading -streamAggr.config=%q", *streamAggrConfig)
+	saCfgReloads.Inc()
+
+	sasNew, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
+	if err != nil {
+		saCfgSuccess.Set(0)
+		saCfgReloadErr.Inc()
+		logger.Errorf("cannot reload -streamAggr.config=%q: use the previously loaded config; error: %s", *streamAggrConfig, err)
+		return
+	}
+	sas := sasGlobal.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := sasGlobal.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded stream aggregation config at -streamAggr.config=%q", *streamAggrConfig)
+	} else {
+		logger.Infof("nothing changed in -streamAggr.config=%q", *streamAggrConfig)
+		sasNew.MustStop()
+	}
+	saCfgSuccess.Set(1)
+	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 }
 
 // MustStopStreamAggr stops stream aggregators.
 func MustStopStreamAggr() {
-	sa.MustStop()
-	sa = nil
-}
+	close(saCfgReloaderStopCh)
+	saCfgReloaderWG.Wait()
 
-var sa *streamaggr.Aggregators
+	sas := sasGlobal.Swap(nil)
+	sas.MustStop()
+}
 
 type streamAggrCtx struct {
 	mn  storage.MetricName
@@ -64,6 +138,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow) {
 	ts := &tss[0]
 	labels := ts.Labels
 	samples := ts.Samples
+	sas := sasGlobal.Load()
 	for _, mr := range mrs {
 		if err := mn.UnmarshalRaw(mr.MetricNameRaw); err != nil {
 			logger.Panicf("BUG: cannot unmarshal recently marshaled MetricName: %s", err)
@@ -88,7 +163,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow) {
 		ts.Labels = labels
 		ts.Samples = samples
 
-		sa.Push(tss)
+		sas.Push(tss)
 	}
 }
 

app/vminsert/csvimport/request_handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -22,7 +23,7 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return parser.ParseStream(req, func(rows []parser.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	})
 }

app/vminsert/datadog/request_handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -25,7 +26,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(series, extraLabels)
 	})
 }

app/vminsert/graphite/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -18,7 +19,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/influx/request_handler.go

@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -34,7 +35,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader) error {
-	return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, false, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, nil)
 	})
 }
@@ -52,7 +53,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, extraLabels)
 	})
 }

app/vminsert/main.go

@@ -48,13 +48,13 @@ var (
 		"See also -influxListenAddr.useProxyProtocol")
 	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbListenAddr.useProxyProtocol")
 	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
@@ -157,6 +157,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write":
+		if common.HandleVMProtoServerHandshake(w, r) {
+			return true
+		}
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()

app/vminsert/native/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -27,12 +27,12 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(block, extraLabels)
 	})
 }
 
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 

app/vminsert/opentsdb/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -18,7 +19,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return parser.ParseStream(r, insertRows)
+	return stream.Parse(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/opentsdbhttp/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -27,7 +28,7 @@ func InsertHandler(req *http.Request) error {
 		if err != nil {
 			return err
 		}
-		return parser.ParseStream(req, func(rows []parser.Row) error {
+		return stream.Parse(req, func(rows []parser.Row) error {
 			return insertRows(rows, extraLabels)
 		})
 	default:

app/vminsert/prometheusimport/request_handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,7 +29,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)